From c0e57b363d5488e70f64708c61bfd2a0bf6ebf31 Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Tue, 21 Nov 2023 13:15:31 +0200 Subject: [PATCH] elastic manage service --- .../eu/eudat/authorization/Permission.java | 3 +- .../elastic/AppElasticProperties.java | 9 ++ .../data/DmpDescriptionTemplateEntity.java | 17 +--- .../eudat/service/elastic/ElasticService.java | 8 ++ .../service/elastic/ElasticServiceImpl.java | 97 ++++++++++++++++++- .../ReferenceTypeServiceImpl.java | 4 +- .../dao/entities/DmpDatasetProfileDao.java | 10 -- .../entities/DmpDatasetProfileDaoImpl.java | 53 ---------- .../operations/DatabaseRepository.java | 2 - .../operations/DatabaseRepositoryImpl.java | 12 --- .../eudat/models/rda/mapper/DmpRDAMapper.java | 20 ++-- .../src/main/resources/config/permissions.yml | 10 +- 12 files changed, 129 insertions(+), 116 deletions(-) delete mode 100644 dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDao.java delete mode 100644 dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDaoImpl.java diff --git a/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java b/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java index f4bbb3950..d5ecf6643 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java +++ b/dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java @@ -22,7 +22,8 @@ public final class Permission { public static String PublicBrowseReference = "PublicBrowseReference"; public static String PublicBrowseUser = "PublicBrowseUser"; - + //Elastic + public static String ManageElastic = "ManageElastic"; //Language public static String BrowseLanguage = "BrowseLanguage"; diff --git a/dmp-backend/core/src/main/java/eu/eudat/configurations/elastic/AppElasticProperties.java b/dmp-backend/core/src/main/java/eu/eudat/configurations/elastic/AppElasticProperties.java index 3d69932ae..7900411fd 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/configurations/elastic/AppElasticProperties.java +++ b/dmp-backend/core/src/main/java/eu/eudat/configurations/elastic/AppElasticProperties.java @@ -7,6 +7,7 @@ public class AppElasticProperties { private boolean enabled; private String dmpIndexName; private String descriptionIndexName; + private int resetBatchSize; private boolean enableIcuAnalysisPlugin; public String getDmpIndexName() { @@ -40,4 +41,12 @@ public class AppElasticProperties { public void setDescriptionIndexName(String descriptionIndexName) { this.descriptionIndexName = descriptionIndexName; } + + public int getResetBatchSize() { + return resetBatchSize; + } + + public void setResetBatchSize(int resetBatchSize) { + this.resetBatchSize = resetBatchSize; + } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/data/DmpDescriptionTemplateEntity.java b/dmp-backend/core/src/main/java/eu/eudat/data/DmpDescriptionTemplateEntity.java index 3fa48aed6..ee532bb7e 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/data/DmpDescriptionTemplateEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/data/DmpDescriptionTemplateEntity.java @@ -11,7 +11,7 @@ import java.util.UUID; @Entity @Table(name = "\"DmpDescriptionTemplate\"") -public class DmpDescriptionTemplateEntity implements DataEntity { +public class DmpDescriptionTemplateEntity { @Id @Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)") @@ -105,19 +105,4 @@ public class DmpDescriptionTemplateEntity implements DataEntity tuple, List fields, String base) { - return null; - } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticService.java b/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticService.java index fabacb500..092b531e9 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticService.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticService.java @@ -24,4 +24,12 @@ public interface ElasticService { void persistDescription(DescriptionEntity description) throws IOException; void deleteDescription(DescriptionEntity description) throws IOException; + + void deleteDmpIndex() throws IOException; + + void deleteDescriptionIndex() throws IOException; + + void resetDmpIndex() throws IOException; + + void resetDescriptionIndex() throws IOException; } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java index 85efb414e..1b80f78ed 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java @@ -4,6 +4,7 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.mapping.Property; import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; import co.elastic.clients.elasticsearch.indices.*; +import eu.eudat.authorization.Permission; import eu.eudat.commons.enums.IsActive; import eu.eudat.configurations.elastic.AppElasticProperties; import eu.eudat.data.DescriptionEntity; @@ -17,12 +18,19 @@ import eu.eudat.model.Description; import eu.eudat.model.Dmp; import eu.eudat.query.DescriptionQuery; import eu.eudat.query.DmpQuery; +import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl; +import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.data.builder.BuilderFactory; +import gr.cite.tools.data.query.Ordering; +import gr.cite.tools.data.query.Paging; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.elastic.ElasticConstants; import gr.cite.tools.exception.MyNotFoundException; import gr.cite.tools.fieldset.BaseFieldSet; +import gr.cite.tools.logging.LoggerService; +import gr.cite.tools.logging.MapLogEntry; import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.data.elasticsearch.annotations.FieldType; @@ -33,10 +41,10 @@ import java.util.*; import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.stereotype.Service; -import org.springframework.web.context.annotation.RequestScope; @Service public class ElasticServiceImpl implements ElasticService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpBlueprintServiceImpl.class)); public final AppElasticProperties appElasticProperties; private final ElasticsearchClient restHighLevelClient; private final ElasticsearchTemplate elasticsearchTemplate; @@ -44,8 +52,9 @@ public class ElasticServiceImpl implements ElasticService { private final BuilderFactory builderFactory; private final EntityManager entityManager; private final MessageSource messageSource; + private final AuthorizationService authorizationService; - public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource) { + public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource, AuthorizationService authorizationService) { this.appElasticProperties = appElasticProperties; this.restHighLevelClient = restHighLevelClient; this.elasticsearchTemplate = elasticsearchTemplate; @@ -53,6 +62,7 @@ public class ElasticServiceImpl implements ElasticService { this.builderFactory = builderFactory; this.entityManager = entityManager; this.messageSource = messageSource; + this.authorizationService = authorizationService; } @Override @@ -115,7 +125,7 @@ public class ElasticServiceImpl implements ElasticService { indexSettingsAnalysis.analyzer("icu_analyzer_text", ab -> ab.custom(x-> x.filter("icu_folding", "english_stop", "english_stemmer").tokenizer("standard"))); } indexSettings.analysis(indexSettingsAnalysis.build()); - CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build()); + restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build()); } @@ -251,6 +261,8 @@ public class ElasticServiceImpl implements ElasticService { } //endregion + + //region persist delete @Override public void persistDmp(DmpEntity dmp) throws IOException { @@ -303,4 +315,83 @@ public class ElasticServiceImpl implements ElasticService { this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); } } + + //endregion + + //region manage + + @Override + public void deleteDmpIndex() throws IOException { + logger.debug(new MapLogEntry("delete dmp index")); + this.authorizationService.authorizeForce(Permission.ManageElastic); + + if (!this.enabled()) return; + boolean exists = this.existsDmpIndex(); + if (exists) return ; + + this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).build()); + } + + @Override + public void deleteDescriptionIndex() throws IOException { + logger.debug(new MapLogEntry("delete description index")); + this.authorizationService.authorizeForce(Permission.ManageElastic); + + if (!this.enabled()) return; + boolean exists = this.existsDescriptionIndex(); + if (exists) return ; + this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDescriptionIndexName()).build()); + } + + @Override + public void resetDmpIndex() throws IOException { + logger.debug(new MapLogEntry("reset dmp index")); + this.authorizationService.authorizeForce(Permission.ManageElastic); + + if (!this.enabled()) return; + this.deleteDmpIndex(); + + int page = 0; + int pageSize = this.appElasticProperties.getResetBatchSize(); + List items; + do { + DmpQuery query = this.queryFactory.query(DmpQuery.class); + query.setOrder(new Ordering().addAscending(Dmp._createdAt)); + query.setPage(new Paging(page * pageSize, pageSize)); + + items = this.queryFactory.query(DmpQuery.class).collect(); + if (items != null && !items.isEmpty()) { + List elasticEntities = this.builderFactory.builder(DmpElasticBuilder.class).build(items); + elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); + page++; + } + } while (items != null && !items.isEmpty()); + } + + @Override + public void resetDescriptionIndex() throws IOException { + logger.debug(new MapLogEntry("reset description index")); + this.authorizationService.authorizeForce(Permission.ManageElastic); + + if (!this.enabled()) return; + this.deleteDescriptionIndex(); + + int page = 0; + int pageSize = this.appElasticProperties.getResetBatchSize(); + List items; + do { + DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class); + query.setOrder(new Ordering().addAscending(Description._createdAt)); + query.setPage(new Paging(page * pageSize, pageSize)); + + items = this.queryFactory.query(DescriptionQuery.class).collect(); + if (items != null && !items.isEmpty()) { + List elasticEntities = this.builderFactory.builder(DescriptionElasticBuilder.class).build(items); + elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); + page++; + } + } while (items != null && !items.isEmpty()); + } + + //endregion } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java index 094612d92..d90da72f7 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java @@ -52,14 +52,13 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService { private final BuilderFactory builderFactory; private final ConventionService conventionService; private final MessageSource messageSource; - private final QueryFactory queryFactory; private final XmlHandlingService xmlHandlingService; private final ErrorThesaurusProperties errors; public ReferenceTypeServiceImpl( EntityManager entityManager, AuthorizationService authorizationService, DeleterFactory deleterFactory, BuilderFactory builderFactory, - ConventionService conventionService, MessageSource messageSource, QueryFactory queryFactory, + ConventionService conventionService, MessageSource messageSource, XmlHandlingService xmlHandlingService, ErrorThesaurusProperties errors) { this.entityManager = entityManager; this.authorizationService = authorizationService; @@ -67,7 +66,6 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService { this.builderFactory = builderFactory; this.conventionService = conventionService; this.messageSource = messageSource; - this.queryFactory = queryFactory; this.xmlHandlingService = xmlHandlingService; this.errors = errors; } diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDao.java b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDao.java deleted file mode 100644 index 4f23ff7c0..000000000 --- a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDao.java +++ /dev/null @@ -1,10 +0,0 @@ -package eu.eudat.data.dao.entities; - -import eu.eudat.data.DmpDescriptionTemplateEntity; -import eu.eudat.data.dao.DatabaseAccessLayer; -import eu.eudat.model.DmpDescriptionTemplate; - -import java.util.UUID; - -public interface DmpDatasetProfileDao extends DatabaseAccessLayer { -} diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDaoImpl.java b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDaoImpl.java deleted file mode 100644 index 30336493b..000000000 --- a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDaoImpl.java +++ /dev/null @@ -1,53 +0,0 @@ -package eu.eudat.data.dao.entities; - -import eu.eudat.data.DmpDescriptionTemplateEntity; -import eu.eudat.data.dao.DatabaseAccess; -import eu.eudat.data.dao.databaselayer.service.DatabaseService; -import eu.eudat.model.DmpDescriptionTemplate; -import eu.eudat.queryable.QueryableList; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Service; - -import javax.management.InvalidApplicationException; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; - -@Service("dmpDatasetProfileDao") -public class DmpDatasetProfileDaoImpl extends DatabaseAccess implements DmpDatasetProfileDao { - @Autowired - public DmpDatasetProfileDaoImpl(DatabaseService databaseService) { - super(databaseService); - } - - @Override - public DmpDescriptionTemplateEntity createOrUpdate(DmpDescriptionTemplateEntity item) { - return this.getDatabaseService().createOrUpdate(item, DmpDescriptionTemplateEntity.class); - } - - @Override - @Async - public CompletableFuture createOrUpdateAsync(DmpDescriptionTemplateEntity item) { - return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item)); - } - - @Override - public DmpDescriptionTemplateEntity find(UUID id) throws InvalidApplicationException { - return this.getDatabaseService().getQueryable(DmpDescriptionTemplateEntity.class).where((builder, root) -> builder.equal(root.get("id"), id)).getSingle(); - } - - @Override - public DmpDescriptionTemplateEntity find(UUID id, String hint) { - throw new UnsupportedOperationException(); - } - - @Override - public void delete(DmpDescriptionTemplateEntity item) { - this.getDatabaseService().delete(item); - } - - @Override - public QueryableList asQueryable() { - return this.getDatabaseService().getQueryable(DmpDescriptionTemplateEntity.class); - } -} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepository.java b/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepository.java index d00343413..59cf96c93 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepository.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepository.java @@ -12,8 +12,6 @@ public interface DatabaseRepository { DMPDao getDmpDao(); - DmpDatasetProfileDao getDmpDatasetProfileDao(); - OrganisationDao getOrganisationDao(); GrantDao getGrantDao(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepositoryImpl.java b/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepositoryImpl.java index 476878e5f..d87baff0f 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepositoryImpl.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepositoryImpl.java @@ -16,8 +16,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository { private DMPDao dmpDao; - private DmpDatasetProfileDao dmpDatasetProfileDao; - private OrganisationDao organisationDao; private GrantDao GrantDao; @@ -68,11 +66,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository { this.dmpDao = dmpDao; } - @Autowired - private void setDmpDatasetProfileDao(DmpDatasetProfileDao dmpDatasetProfileDao) { - this.dmpDatasetProfileDao = dmpDatasetProfileDao; - } - @Autowired private void setOrganisationDao(OrganisationDao organisationDao) { this.organisationDao = organisationDao; @@ -119,11 +112,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository { return dmpDao; } - @Override - public DmpDatasetProfileDao getDmpDatasetProfileDao() { - return dmpDatasetProfileDao; - } - @Override public OrganisationDao getOrganisationDao() { return organisationDao; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java index 57105b220..88b19da0e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java @@ -1,28 +1,21 @@ package eu.eudat.models.rda.mapper; -import eu.eudat.commons.enums.IsActive; -import eu.eudat.data.DescriptionTemplateEntity; import eu.eudat.data.DmpDescriptionTemplateEntity; import eu.eudat.data.DmpEntity; -import eu.eudat.data.EntityDoiEntity; -import eu.eudat.data.old.*; import eu.eudat.logic.services.ApiContext; -import eu.eudat.logic.utilities.helpers.StreamDistinctBy; import eu.eudat.models.rda.Dmp; -import eu.eudat.models.rda.DmpId; -import net.minidev.json.JSONObject; +import eu.eudat.query.DmpDescriptionTemplateQuery; +import gr.cite.tools.data.query.QueryFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import jakarta.persistence.NoResultException; import jakarta.transaction.Transactional; import javax.management.InvalidApplicationException; import java.io.IOException; import java.util.*; -import java.util.stream.Collectors; @Component public class DmpRDAMapper { @@ -31,13 +24,15 @@ public class DmpRDAMapper { private DatasetRDAMapper datasetRDAMapper; private ApiContext apiContext; + private final QueryFactory queryFactory; @Autowired - public DmpRDAMapper(DatasetRDAMapper datasetRDAMapper, ApiContext apiContext) throws IOException { + public DmpRDAMapper(DatasetRDAMapper datasetRDAMapper, ApiContext apiContext, QueryFactory queryFactory) throws IOException { this.datasetRDAMapper = datasetRDAMapper; this.apiContext = apiContext; + this.queryFactory = queryFactory; } @Transactional @@ -190,9 +185,6 @@ public class DmpRDAMapper { } private DmpDescriptionTemplateEntity getProfile(String descriptionTemplateId, UUID dmpId) throws InvalidApplicationException { - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDatasetProfileDao().asQueryable().where(((builder, root) -> builder.and( - builder.equal(root.get("datasetprofile"), UUID.fromString(descriptionTemplateId)), - builder.equal(root.get("dmp"), dmpId)) - )).getSingleOrDefault(); + return this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(dmpId).descriptionTemplateIds(UUID.fromString(descriptionTemplateId)).first(); } } diff --git a/dmp-backend/web/src/main/resources/config/permissions.yml b/dmp-backend/web/src/main/resources/config/permissions.yml index 742d26662..259aaf5fd 100644 --- a/dmp-backend/web/src/main/resources/config/permissions.yml +++ b/dmp-backend/web/src/main/resources/config/permissions.yml @@ -80,8 +80,14 @@ permissions: clients: [ ] allowAnonymous: true allowAuthenticated: true - - + + # Elastic + ManageElastic: + roles: + - Admin + clients: [ ] + allowAnonymous: false + allowAuthenticated: false # Language BrowseLanguage: