argos/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java

307 lines
17 KiB
Java

package eu.eudat.service.elastic;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
import co.elastic.clients.elasticsearch.indices.*;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.configurations.elastic.AppElasticProperties;
import eu.eudat.data.DescriptionEntity;
import eu.eudat.data.DmpEntity;
import eu.eudat.elastic.data.DescriptionElasticEntity;
import eu.eudat.elastic.data.DmpElasticEntity;
import eu.eudat.elastic.data.nested.*;
import eu.eudat.elastic.elasticbuilder.DescriptionElasticBuilder;
import eu.eudat.elastic.elasticbuilder.DmpElasticBuilder;
import eu.eudat.model.Description;
import eu.eudat.model.Dmp;
import eu.eudat.query.DescriptionQuery;
import eu.eudat.query.DmpQuery;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.elastic.ElasticConstants;
import gr.cite.tools.exception.MyNotFoundException;
import gr.cite.tools.fieldset.BaseFieldSet;
import jakarta.persistence.EntityManager;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.data.elasticsearch.annotations.FieldType;
import java.io.IOException;
import java.util.*;
import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
@Service
public class ElasticServiceImpl implements ElasticService {
public final AppElasticProperties appElasticProperties;
private final ElasticsearchClient restHighLevelClient;
private final ElasticsearchTemplate elasticsearchTemplate;
private final QueryFactory queryFactory;
private final BuilderFactory builderFactory;
private final EntityManager entityManager;
private final MessageSource messageSource;
public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource) {
this.appElasticProperties = appElasticProperties;
this.restHighLevelClient = restHighLevelClient;
this.elasticsearchTemplate = elasticsearchTemplate;
this.queryFactory = queryFactory;
this.builderFactory = builderFactory;
this.entityManager = entityManager;
this.messageSource = messageSource;
}
@Override
public boolean enabled() {
return appElasticProperties.isEnabled();
}
@Override
public boolean existsDmpIndex() throws IOException {
if (!this.enabled()) return false;
return restHighLevelClient.indices().exists(new ExistsRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).includeDefaults(true).build()).value();
}
@Override
public boolean existsDescriptionIndex() throws IOException {
if (!this.enabled()) return false;
return restHighLevelClient.indices().exists(new ExistsRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).includeDefaults(true).build()).value();
}
//region ensure index
@Override
public void ensureDmpIndex() throws IOException {
if (!this.enabled()) return ;
boolean exists = this.existsDmpIndex();
if (exists) return ;
this.ensureIndex(this.appElasticProperties.getDmpIndexName(), this.createDmpTemplatePropertyMap());
}
@Override
public void ensureDescriptionIndex() throws IOException {
if (!this.enabled()) return ;
boolean exists = this.existsDescriptionIndex();
if (exists) return ;
this.ensureIndex(this.appElasticProperties.getDescriptionIndexName(), this.createDescriptionTemplatePropertyMap());
}
@Override
public void ensureIndexes() throws IOException {
if (!this.enabled()) return ;
this.ensureDmpIndex();
this.ensureDescriptionIndex();
}
private void ensureIndex(String indexName, Map<String, Property> propertyMap) throws IOException {
TypeMapping.Builder typeMapping = new TypeMapping.Builder();
typeMapping.properties(propertyMap);
IndexSettings.Builder indexSettings = new IndexSettings.Builder();
IndexSettingsAnalysis.Builder indexSettingsAnalysis = new IndexSettingsAnalysis.Builder();
indexSettingsAnalysis.filter("english_stemmer", ((tf) -> tf.definition(tfdb -> tfdb.stemmer(stemmerBuilder -> stemmerBuilder.language("english")))))
.filter("english_stop", tf -> tf.definition(tfdb -> tfdb.stop(stopTokenBuilder -> stopTokenBuilder)));
if (this.appElasticProperties.isEnableIcuAnalysisPlugin()){
indexSettingsAnalysis.analyzer("icu_analyzer_text", ab -> ab.custom(x-> x.filter("icu_folding", "english_stop", "english_stemmer").tokenizer("icu_tokenizer")));
} else {
indexSettingsAnalysis.analyzer("icu_analyzer_text", ab -> ab.custom(x-> x.filter("icu_folding", "english_stop", "english_stemmer").tokenizer("standard")));
}
indexSettings.analysis(indexSettingsAnalysis.build());
CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build());
}
private Map<String, Property> createDescriptionTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(DescriptionElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(DescriptionElasticEntity._label, this.createElastic(FieldType.Keyword, false));
propertyMap.put(DescriptionElasticEntity._description, this.createElastic(FieldType.Text, true));
propertyMap.put(DescriptionElasticEntity._status, this.createElastic(FieldType.Short, false));
propertyMap.put(DescriptionElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false));
propertyMap.put(DescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build());
propertyMap.put(DescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build());
propertyMap.put(DescriptionElasticEntity._descriptionTemplate, new Property.Builder().object(x -> x.properties(this.createNestedDescriptionTemplateTemplatePropertyMap())).build());
propertyMap.put(DescriptionElasticEntity._dmp, new Property.Builder().object(x -> x.properties(this.createNetsedDmpTemplatePropertyMap())).build());
return propertyMap;
}
private Map<String, Property> createDmpTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(DmpElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(DmpElasticEntity._label, this.createElastic(FieldType.Text, true));
propertyMap.put(DmpElasticEntity._description, this.createElastic(FieldType.Text, false));
propertyMap.put(DmpElasticEntity._status, this.createElastic(FieldType.Short, false));
propertyMap.put(DmpElasticEntity._version, this.createElastic(FieldType.Short, false));
propertyMap.put(DmpElasticEntity._language, this.createElastic(FieldType.Keyword, false));
propertyMap.put(DmpElasticEntity._blueprintId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(DmpElasticEntity._accessType, this.createElastic(FieldType.Short, false));
propertyMap.put(DmpElasticEntity._groupId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(DmpElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false));
propertyMap.put(DmpElasticEntity._descriptions, new Property.Builder().nested(x -> x.properties(this.createNestedDescriptionTemplatePropertyMap())).build());
propertyMap.put(DmpElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build());
propertyMap.put(DmpElasticEntity._collaborators, new Property.Builder().nested(x -> x.properties(this.createNestedCollaboratorTemplatePropertyMap())).build());
propertyMap.put(DmpElasticEntity._dois, new Property.Builder().nested(x -> x.properties(this.createNestedDoisTemplatePropertyMap())).build());
return propertyMap;
}
private Map<String, Property> createNestedDescriptionTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedDescriptionElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDescriptionElasticEntity._label, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDescriptionElasticEntity._dmpId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDescriptionElasticEntity._description, this.createElastic(FieldType.Text, true));
propertyMap.put(NestedDescriptionElasticEntity._status, this.createElastic(FieldType.Short, false));
propertyMap.put(NestedDescriptionElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false));
propertyMap.put(NestedDescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build());
propertyMap.put(NestedDescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build());
return propertyMap;
}
private Map<String, Property> createNestedTagsTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedTagElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedTagElasticEntity._label, this.createElastic(FieldType.Text, true));
return propertyMap;
}
private Map<String, Property> createNestedReferencesTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedReferenceElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedReferenceElasticEntity._label, this.createElastic(FieldType.Text, true));
return propertyMap;
}
private Map<String, Property> createNestedDescriptionTemplateTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedDescriptionTemplateElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDescriptionTemplateElasticEntity._label, this.createElastic(FieldType.Text, true));
return propertyMap;
}
private Map<String, Property> createNetsedDmpTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedDmpElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDmpElasticEntity._label, this.createElastic(FieldType.Text, true));
propertyMap.put(NestedDmpElasticEntity._description, this.createElastic(FieldType.Text, false));
propertyMap.put(NestedDmpElasticEntity._status, this.createElastic(FieldType.Short, false));
propertyMap.put(NestedDmpElasticEntity._version, this.createElastic(FieldType.Short, false));
propertyMap.put(NestedDmpElasticEntity._language, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDmpElasticEntity._blueprintId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDmpElasticEntity._accessType, this.createElastic(FieldType.Short, false));
propertyMap.put(NestedDmpElasticEntity._groupId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDmpElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false));
propertyMap.put(NestedDmpElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build());
propertyMap.put(NestedDmpElasticEntity._collaborators, new Property.Builder().nested(x -> x.properties(this.createNestedCollaboratorTemplatePropertyMap())).build());
propertyMap.put(NestedDmpElasticEntity._dois, new Property.Builder().nested(x -> x.properties(this.createNestedDoisTemplatePropertyMap())).build());
return propertyMap;
}
private Map<String, Property> createNestedCollaboratorTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedCollaboratorElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedCollaboratorElasticEntity._name, this.createElastic(FieldType.Text, true));
propertyMap.put(NestedCollaboratorElasticEntity._role, this.createElastic(FieldType.Short, false));
return propertyMap;
}
private Map<String, Property> createNestedDoisTemplatePropertyMap(){
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedDoiElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDoiElasticEntity._repositoryId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDoiElasticEntity._doi, this.createElastic(FieldType.Keyword, false));
return propertyMap;
}
private Property createElastic(FieldType fieldType, boolean hasKeywordSubField){
switch (fieldType){
case Keyword -> {
return new Property.Builder().keyword(x -> x).build();
}
case Text -> {
return hasKeywordSubField ? new Property.Builder().text(x -> x.analyzer("icu_analyzer_text").fields(ElasticConstants.SubFields.keyword, y -> y.keyword(z-> z))).build() : new Property.Builder().text(x -> x).build();
}
case Date -> {
return new Property.Builder().date(x -> x).build();
}
case Short -> {
return new Property.Builder().short_(x -> x).build();
}
case Boolean -> {
return new Property.Builder().boolean_(x -> x).build();
}
default -> throw new RuntimeException();
}
}
//endregion
@Override
public void persistDmp(DmpEntity dmp) throws IOException {
if (!this.enabled()) return;
this.ensureIndexes();
DmpElasticEntity dmpElasticEntity = this.builderFactory.builder(DmpElasticBuilder.class).build(dmp);
this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
List<DescriptionElasticEntity> descriptions = this.builderFactory.builder(DescriptionElasticBuilder.class).build(this.queryFactory.query(DescriptionQuery.class).isActive(IsActive.Active).dmpSubQuery(this.queryFactory.query(DmpQuery.class).ids(dmp.getId())).collect());
this.elasticsearchTemplate.save(descriptions, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName()));
}
@Override
public void deleteDmp(DmpEntity dmp) throws IOException {
if (!this.enabled()) return;
this.ensureIndexes();
this.elasticsearchTemplate.delete(dmp.getId(), IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
List<DescriptionEntity> descriptions = this.queryFactory.query(DescriptionQuery.class).dmpSubQuery(this.queryFactory.query(DmpQuery.class).ids(dmp.getId())).collectAs(new BaseFieldSet().ensure(Description._id));
for (DescriptionEntity description: descriptions) {
this.elasticsearchTemplate.delete(description.getId(), IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName()));
}
}
@Override
public void persistDescription(DescriptionEntity description) throws IOException {
if (!this.enabled()) return;
this.ensureIndexes();
DescriptionElasticEntity descriptionElasticEntity = this.builderFactory.builder(DescriptionElasticBuilder.class).build(description);
this.elasticsearchTemplate.save(descriptionElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName()));
DmpEntity dmpEntity = this.entityManager.find(DmpEntity.class, description.getDmpId());
if (dmpEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{description.getDmpId(), Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));
if (dmpEntity.getIsActive().equals(IsActive.Active)) {
DmpElasticEntity dmpElasticEntity = this.builderFactory.builder(DmpElasticBuilder.class).build(dmpEntity);
this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
}
}
@Override
public void deleteDescription(DescriptionEntity description) throws IOException {
if (!this.enabled()) return;
this.ensureIndexes();
this.elasticsearchTemplate.delete(description.getId(), IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName()));
DmpEntity dmpEntity = this.entityManager.find(DmpEntity.class, description.getDmpId());
if (dmpEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{description.getDmpId(), Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));
if (dmpEntity.getIsActive().equals(IsActive.Active)) {
DmpElasticEntity dmpElasticEntity = this.builderFactory.builder(DmpElasticBuilder.class).build(dmpEntity);
this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
}
}
}