package eu.eudat.service.elastic; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.mapping.Property; import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; import co.elastic.clients.elasticsearch.indices.*; import eu.eudat.authorization.Permission; import eu.eudat.commons.enums.IsActive; import eu.eudat.data.DescriptionEntity; import eu.eudat.data.DmpEntity; import eu.eudat.elastic.data.DescriptionElasticEntity; import eu.eudat.elastic.data.DmpElasticEntity; import eu.eudat.elastic.data.nested.*; import eu.eudat.elastic.elasticbuilder.DescriptionElasticBuilder; import eu.eudat.elastic.elasticbuilder.DmpElasticBuilder; import eu.eudat.model.Description; import eu.eudat.model.Dmp; import eu.eudat.query.DescriptionQuery; import eu.eudat.query.DmpQuery; import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.query.Ordering; import gr.cite.tools.data.query.Paging; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.elastic.ElasticConstants; import gr.cite.tools.exception.MyNotFoundException; import gr.cite.tools.fieldset.BaseFieldSet; import gr.cite.tools.logging.LoggerService; import gr.cite.tools.logging.MapLogEntry; import jakarta.persistence.EntityManager; import org.slf4j.LoggerFactory; import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.data.elasticsearch.annotations.FieldType; import java.io.IOException; import java.util.*; import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.stereotype.Service; @Service public class ElasticServiceImpl implements ElasticService { private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpBlueprintServiceImpl.class)); public final AppElasticProperties appElasticProperties; private final ElasticsearchClient restHighLevelClient; private final ElasticsearchTemplate elasticsearchTemplate; private final QueryFactory queryFactory; private final BuilderFactory builderFactory; private final EntityManager entityManager; private final MessageSource messageSource; private final AuthorizationService authorizationService; public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource, AuthorizationService authorizationService) { this.appElasticProperties = appElasticProperties; this.restHighLevelClient = restHighLevelClient; this.elasticsearchTemplate = elasticsearchTemplate; this.queryFactory = queryFactory; this.builderFactory = builderFactory; this.entityManager = entityManager; this.messageSource = messageSource; this.authorizationService = authorizationService; } @Override public boolean enabled() { return appElasticProperties.isEnabled(); } @Override public boolean existsDmpIndex() throws IOException { if (!this.enabled()) return false; return restHighLevelClient.indices().exists(new ExistsRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).includeDefaults(true).build()).value(); } @Override public boolean existsDescriptionIndex() throws IOException { if (!this.enabled()) return false; return restHighLevelClient.indices().exists(new ExistsRequest.Builder().index(this.appElasticProperties.getDescriptionIndexName()).includeDefaults(true).build()).value(); } //region ensure index @Override public void ensureDmpIndex() throws IOException { if (!this.enabled()) return ; boolean exists = this.existsDmpIndex(); if (exists) return ; this.ensureIndex(this.appElasticProperties.getDmpIndexName(), this.createDmpTemplatePropertyMap()); } @Override public void ensureDescriptionIndex() throws IOException { if (!this.enabled()) return ; boolean exists = this.existsDescriptionIndex(); if (exists) return ; this.ensureIndex(this.appElasticProperties.getDescriptionIndexName(), this.createDescriptionTemplatePropertyMap()); } @Override public void ensureIndexes() throws IOException { if (!this.enabled()) return ; this.ensureDmpIndex(); this.ensureDescriptionIndex(); } private void ensureIndex(String indexName, Map propertyMap) throws IOException { TypeMapping.Builder typeMapping = new TypeMapping.Builder(); typeMapping.properties(propertyMap); IndexSettings.Builder indexSettings = new IndexSettings.Builder(); IndexSettingsAnalysis.Builder indexSettingsAnalysis = new IndexSettingsAnalysis.Builder(); indexSettingsAnalysis.filter("english_stemmer", ((tf) -> tf.definition(tfdb -> tfdb.stemmer(stemmerBuilder -> stemmerBuilder.language("english"))))) .filter("english_stop", tf -> tf.definition(tfdb -> tfdb.stop(stopTokenBuilder -> stopTokenBuilder))); if (this.appElasticProperties.isEnableIcuAnalysisPlugin()){ indexSettingsAnalysis.analyzer("icu_analyzer_text", ab -> ab.custom(x-> x.filter("icu_folding", "english_stop", "english_stemmer").tokenizer("icu_tokenizer"))); } else { indexSettingsAnalysis.analyzer("icu_analyzer_text", ab -> ab.custom(x-> x.filter("icu_folding", "english_stop", "english_stemmer").tokenizer("standard"))); } indexSettings.analysis(indexSettingsAnalysis.build()); restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build()); } private Map createDescriptionTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(DescriptionElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(DescriptionElasticEntity._label, this.createElastic(FieldType.Keyword, false)); propertyMap.put(DescriptionElasticEntity._description, this.createElastic(FieldType.Text, true)); propertyMap.put(DescriptionElasticEntity._status, this.createElastic(FieldType.Short, false)); propertyMap.put(DescriptionElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false)); propertyMap.put(DescriptionElasticEntity._createdAt, this.createElastic(FieldType.Date, false)); propertyMap.put(DescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build()); propertyMap.put(DescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build()); propertyMap.put(DescriptionElasticEntity._descriptionTemplate, new Property.Builder().object(x -> x.properties(this.createNestedDescriptionTemplateTemplatePropertyMap())).build()); propertyMap.put(DescriptionElasticEntity._dmp, new Property.Builder().object(x -> x.properties(this.createNetsedDmpTemplatePropertyMap())).build()); return propertyMap; } private Map createDmpTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(DmpElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(DmpElasticEntity._label, this.createElastic(FieldType.Text, true)); propertyMap.put(DmpElasticEntity._description, this.createElastic(FieldType.Text, false)); propertyMap.put(DmpElasticEntity._status, this.createElastic(FieldType.Short, false)); propertyMap.put(DmpElasticEntity._version, this.createElastic(FieldType.Short, false)); propertyMap.put(DmpElasticEntity._language, this.createElastic(FieldType.Keyword, false)); propertyMap.put(DmpElasticEntity._blueprintId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(DmpElasticEntity._accessType, this.createElastic(FieldType.Short, false)); propertyMap.put(DmpElasticEntity._groupId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(DmpElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false)); propertyMap.put(DmpElasticEntity._versionStatus, this.createElastic(FieldType.Short, false)); propertyMap.put(DmpElasticEntity._descriptions, new Property.Builder().nested(x -> x.properties(this.createNestedDescriptionTemplatePropertyMap())).build()); propertyMap.put(DmpElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build()); propertyMap.put(DmpElasticEntity._collaborators, new Property.Builder().nested(x -> x.properties(this.createNestedCollaboratorTemplatePropertyMap())).build()); propertyMap.put(DmpElasticEntity._dois, new Property.Builder().nested(x -> x.properties(this.createNestedDoisTemplatePropertyMap())).build()); return propertyMap; } private Map createNestedDescriptionTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(NestedDescriptionElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDescriptionElasticEntity._label, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDescriptionElasticEntity._dmpId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDescriptionElasticEntity._description, this.createElastic(FieldType.Text, true)); propertyMap.put(NestedDescriptionElasticEntity._status, this.createElastic(FieldType.Short, false)); propertyMap.put(NestedDescriptionElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false)); propertyMap.put(NestedDescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build()); propertyMap.put(NestedDescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build()); propertyMap.put(NestedDescriptionElasticEntity._descriptionTemplate, new Property.Builder().object(x -> x.properties(this.createNestedDescriptionTemplateTemplatePropertyMap())).build()); return propertyMap; } private Map createNestedTagsTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(NestedTagElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedTagElasticEntity._label, this.createElastic(FieldType.Text, true)); return propertyMap; } private Map createNestedReferencesTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(NestedReferenceElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedReferenceElasticEntity._label, this.createElastic(FieldType.Text, true)); return propertyMap; } private Map createNestedDescriptionTemplateTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(NestedDescriptionTemplateElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDescriptionTemplateElasticEntity._label, this.createElastic(FieldType.Text, true)); propertyMap.put(NestedDescriptionTemplateElasticEntity._versionStatus, this.createElastic(FieldType.Short, true)); return propertyMap; } private Map createNetsedDmpTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(NestedDmpElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDmpElasticEntity._label, this.createElastic(FieldType.Text, true)); propertyMap.put(NestedDmpElasticEntity._description, this.createElastic(FieldType.Text, false)); propertyMap.put(NestedDmpElasticEntity._status, this.createElastic(FieldType.Short, false)); propertyMap.put(NestedDmpElasticEntity._version, this.createElastic(FieldType.Short, false)); propertyMap.put(NestedDmpElasticEntity._versionStatus, this.createElastic(FieldType.Short, false)); propertyMap.put(NestedDmpElasticEntity._language, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDmpElasticEntity._blueprintId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDmpElasticEntity._accessType, this.createElastic(FieldType.Short, false)); propertyMap.put(NestedDmpElasticEntity._groupId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDmpElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false)); propertyMap.put(NestedDmpElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build()); propertyMap.put(NestedDmpElasticEntity._collaborators, new Property.Builder().nested(x -> x.properties(this.createNestedCollaboratorTemplatePropertyMap())).build()); propertyMap.put(NestedDmpElasticEntity._dois, new Property.Builder().nested(x -> x.properties(this.createNestedDoisTemplatePropertyMap())).build()); return propertyMap; } private Map createNestedCollaboratorTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(NestedCollaboratorElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedCollaboratorElasticEntity._name, this.createElastic(FieldType.Text, true)); propertyMap.put(NestedCollaboratorElasticEntity._role, this.createElastic(FieldType.Short, false)); return propertyMap; } private Map createNestedDoisTemplatePropertyMap(){ Map propertyMap = new HashMap<>(); propertyMap.put(NestedDoiElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDoiElasticEntity._repositoryId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDoiElasticEntity._doi, this.createElastic(FieldType.Keyword, false)); return propertyMap; } private Property createElastic(FieldType fieldType, boolean hasKeywordSubField){ switch (fieldType){ case Keyword -> { return new Property.Builder().keyword(x -> x).build(); } case Text -> { return hasKeywordSubField ? new Property.Builder().text(x -> x.analyzer("icu_analyzer_text").fields(ElasticConstants.SubFields.keyword, y -> y.keyword(z-> z))).build() : new Property.Builder().text(x -> x).build(); } case Date -> { return new Property.Builder().date(x -> x).build(); } case Short -> { return new Property.Builder().short_(x -> x).build(); } case Boolean -> { return new Property.Builder().boolean_(x -> x).build(); } default -> throw new RuntimeException(); } } //endregion //region persist delete @Override public void persistDmp(DmpEntity dmp) throws IOException { if (!this.enabled()) return; this.ensureIndexes(); DmpElasticEntity dmpElasticEntity = this.builderFactory.builder(DmpElasticBuilder.class).build(dmp); this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); List descriptions = this.builderFactory.builder(DescriptionElasticBuilder.class).build(this.queryFactory.query(DescriptionQuery.class).isActive(IsActive.Active).dmpSubQuery(this.queryFactory.query(DmpQuery.class).ids(dmp.getId())).collect()); this.elasticsearchTemplate.save(descriptions, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); } @Override public void deleteDmp(DmpEntity dmp) throws IOException { if (!this.enabled()) return; this.ensureIndexes(); DmpElasticEntity dmpElasticEntity = this.elasticsearchTemplate.get(dmp.getId().toString(),DmpElasticEntity.class, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); if (dmpElasticEntity == null) return; this.elasticsearchTemplate.delete(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); List descriptions = this.queryFactory.query(DescriptionQuery.class).dmpSubQuery(this.queryFactory.query(DmpQuery.class).ids(dmp.getId())).collectAs(new BaseFieldSet().ensure(Description._id)); for (DescriptionEntity description: descriptions) { DescriptionElasticEntity descriptionElasticEntity = this.elasticsearchTemplate.get(description.getId().toString(), DescriptionElasticEntity.class, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); if (descriptionElasticEntity == null) continue; this.elasticsearchTemplate.delete(descriptionElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); } } @Override public void persistDescription(DescriptionEntity description) throws IOException { if (!this.enabled()) return; this.ensureIndexes(); DescriptionElasticEntity descriptionElasticEntity = this.builderFactory.builder(DescriptionElasticBuilder.class).build(description); this.elasticsearchTemplate.save(descriptionElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); DmpEntity dmpEntity = this.entityManager.find(DmpEntity.class, description.getDmpId()); if (dmpEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{description.getDmpId(), Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale())); if (dmpEntity.getIsActive().equals(IsActive.Active)) { DmpElasticEntity dmpElasticEntity = this.builderFactory.builder(DmpElasticBuilder.class).build(dmpEntity); this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); } } @Override public void deleteDescription(DescriptionEntity description) throws IOException { if (!this.enabled()) return; this.ensureIndexes(); DescriptionElasticEntity descriptionElasticEntity = this.elasticsearchTemplate.get(description.getId().toString(), DescriptionElasticEntity.class, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); if (descriptionElasticEntity == null) return; this.elasticsearchTemplate.delete(descriptionElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); DmpEntity dmpEntity = this.entityManager.find(DmpEntity.class, description.getDmpId()); if (dmpEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{description.getDmpId(), Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale())); if (dmpEntity.getIsActive().equals(IsActive.Active)) { DmpElasticEntity dmpElasticEntity = this.builderFactory.builder(DmpElasticBuilder.class).build(dmpEntity); this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); } } //endregion //region manage @Override public void deleteDmpIndex() throws IOException { logger.debug(new MapLogEntry("delete dmp index")); this.authorizationService.authorizeForce(Permission.ManageElastic); if (!this.enabled()) return; boolean exists = this.existsDmpIndex(); if (!exists) return ; this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).build()); } @Override public void deleteDescriptionIndex() throws IOException { logger.debug(new MapLogEntry("delete description index")); this.authorizationService.authorizeForce(Permission.ManageElastic); if (!this.enabled()) return; boolean exists = this.existsDescriptionIndex(); if (!exists) return ; this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDescriptionIndexName()).build()); } @Override public void resetDmpIndex() throws IOException { logger.debug(new MapLogEntry("reset dmp index")); this.authorizationService.authorizeForce(Permission.ManageElastic); if (!this.enabled()) return; this.deleteDmpIndex(); this.ensureDmpIndex(); int page = 0; int pageSize = this.appElasticProperties.getResetBatchSize(); List items; do { DmpQuery query = this.queryFactory.query(DmpQuery.class); query.setOrder(new Ordering().addAscending(Dmp._createdAt)); query.setPage(new Paging(page * pageSize, pageSize)); items = query.collect(); if (items != null && !items.isEmpty()) { List elasticEntities = this.builderFactory.builder(DmpElasticBuilder.class).build(items); elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); page++; } } while (items != null && !items.isEmpty()); } @Override public void resetDescriptionIndex() throws IOException { logger.debug(new MapLogEntry("reset description index")); this.authorizationService.authorizeForce(Permission.ManageElastic); if (!this.enabled()) return; this.deleteDescriptionIndex(); this.ensureDescriptionIndex(); int page = 0; int pageSize = this.appElasticProperties.getResetBatchSize(); List items; do { DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class); query.setOrder(new Ordering().addAscending(Description._createdAt)); query.setPage(new Paging(page * pageSize, pageSize)); items = query.collect(); if (items != null && !items.isEmpty()) { List elasticEntities = this.builderFactory.builder(DescriptionElasticBuilder.class).build(items); elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); page++; } } while (items != null && !items.isEmpty()); } //endregion }