elastic manage service

dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java

@@ -22,7 +22,8 @@ public final class Permission {
 	public static String PublicBrowseReference = "PublicBrowseReference";
 	public static String PublicBrowseUser = "PublicBrowseUser";
 	
-	
+	//Elastic
+	public static String ManageElastic = "ManageElastic";
 	
 	//Language
 	public static String BrowseLanguage = "BrowseLanguage";

dmp-backend/core/src/main/java/eu/eudat/configurations/elastic/AppElasticProperties.java

@@ -7,6 +7,7 @@ public class AppElasticProperties {
 	private boolean enabled;
 	private String dmpIndexName;
 	private String descriptionIndexName;
+	private int resetBatchSize;
 	private boolean enableIcuAnalysisPlugin;
 
 	public String getDmpIndexName() {
@@ -40,4 +41,12 @@ public class AppElasticProperties {
 	public void setDescriptionIndexName(String descriptionIndexName) {
 		this.descriptionIndexName = descriptionIndexName;
 	}
+
+	public int getResetBatchSize() {
+		return resetBatchSize;
+	}
+
+	public void setResetBatchSize(int resetBatchSize) {
+		this.resetBatchSize = resetBatchSize;
+	}
 }

dmp-backend/core/src/main/java/eu/eudat/data/DmpDescriptionTemplateEntity.java

@@ -11,7 +11,7 @@ import java.util.UUID;
 
 @Entity
 @Table(name = "\"DmpDescriptionTemplate\"")
-public class DmpDescriptionTemplateEntity implements DataEntity<DmpDescriptionTemplateEntity, UUID> {
+public class DmpDescriptionTemplateEntity  {
 
     @Id
     @Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
@@ -105,19 +105,4 @@ public class DmpDescriptionTemplateEntity implements DataEntity<DmpDescriptionTe
     public void setIsActive(IsActive isActive) {
         this.isActive = isActive;
     }
-
-    @Override
-    public void update(DmpDescriptionTemplateEntity entity) {
-        
-    }
-
-    @Override
-    public UUID getKeys() {
-        return null;
-    }
-
-    @Override
-    public DmpDescriptionTemplateEntity buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
-        return null;
-    }
 }

dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticService.java

@@ -24,4 +24,12 @@ public interface ElasticService {
     void persistDescription(DescriptionEntity description) throws IOException;
 
     void deleteDescription(DescriptionEntity description) throws IOException;
+
+	void deleteDmpIndex() throws IOException;
+
+    void deleteDescriptionIndex() throws IOException;
+
+	void resetDmpIndex() throws IOException;
+
+    void resetDescriptionIndex() throws IOException;
 }

dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java

@@ -4,6 +4,7 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.mapping.Property;
 import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
 import co.elastic.clients.elasticsearch.indices.*;
+import eu.eudat.authorization.Permission;
 import eu.eudat.commons.enums.IsActive;
 import eu.eudat.configurations.elastic.AppElasticProperties;
 import eu.eudat.data.DescriptionEntity;
@@ -17,12 +18,19 @@ import eu.eudat.model.Description;
 import eu.eudat.model.Dmp;
 import eu.eudat.query.DescriptionQuery;
 import eu.eudat.query.DmpQuery;
+import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl;
+import gr.cite.commons.web.authz.service.AuthorizationService;
 import gr.cite.tools.data.builder.BuilderFactory;
+import gr.cite.tools.data.query.Ordering;
+import gr.cite.tools.data.query.Paging;
 import gr.cite.tools.data.query.QueryFactory;
 import gr.cite.tools.elastic.ElasticConstants;
 import gr.cite.tools.exception.MyNotFoundException;
 import gr.cite.tools.fieldset.BaseFieldSet;
+import gr.cite.tools.logging.LoggerService;
+import gr.cite.tools.logging.MapLogEntry;
 import jakarta.persistence.EntityManager;
+import org.slf4j.LoggerFactory;
 import org.springframework.context.MessageSource;
 import org.springframework.context.i18n.LocaleContextHolder;
 import org.springframework.data.elasticsearch.annotations.FieldType;
@@ -33,10 +41,10 @@ import java.util.*;
 import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
 import org.springframework.stereotype.Service;
-import org.springframework.web.context.annotation.RequestScope;
 
 @Service
 public class ElasticServiceImpl implements ElasticService {
+	private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpBlueprintServiceImpl.class));
 	public final AppElasticProperties appElasticProperties;
 	private final ElasticsearchClient restHighLevelClient;
 	private final ElasticsearchTemplate elasticsearchTemplate;
@@ -44,8 +52,9 @@ public class ElasticServiceImpl implements ElasticService {
 	private final BuilderFactory builderFactory;
 	private final EntityManager entityManager;
 	private final MessageSource messageSource;
+	private final AuthorizationService authorizationService;
 
-	public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource) {
+	public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource, AuthorizationService authorizationService) {
 		this.appElasticProperties = appElasticProperties;
 		this.restHighLevelClient = restHighLevelClient;
 		this.elasticsearchTemplate = elasticsearchTemplate;
@@ -53,6 +62,7 @@ public class ElasticServiceImpl implements ElasticService {
 		this.builderFactory = builderFactory;
 		this.entityManager = entityManager;
 		this.messageSource = messageSource;
+		this.authorizationService = authorizationService;
 	}
 
 	@Override
@@ -115,7 +125,7 @@ public class ElasticServiceImpl implements ElasticService {
 			indexSettingsAnalysis.analyzer("icu_analyzer_text",  ab -> ab.custom(x-> x.filter("icu_folding", "english_stop", "english_stemmer").tokenizer("standard")));
 		}
 		indexSettings.analysis(indexSettingsAnalysis.build());
-		CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build());
+		restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build());
 
 	}
 
@@ -252,6 +262,8 @@ public class ElasticServiceImpl implements ElasticService {
 	
 	//endregion
 	
+	//region persist delete
+
 	@Override
 	public void persistDmp(DmpEntity dmp) throws IOException {
 		if (!this.enabled()) return;
@@ -303,4 +315,83 @@ public class ElasticServiceImpl implements ElasticService {
 			this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
 		}
 	}
+	
+	//endregion
+	
+	//region manage
+	
+	@Override
+	public void deleteDmpIndex() throws IOException {
+		logger.debug(new MapLogEntry("delete dmp index"));
+		this.authorizationService.authorizeForce(Permission.ManageElastic);
+		
+		if (!this.enabled()) return;
+		boolean exists = this.existsDmpIndex();
+		if (exists) return ;
+		
+		this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).build());
+	}
+
+	@Override
+	public void deleteDescriptionIndex() throws IOException {
+		logger.debug(new MapLogEntry("delete description index"));
+		this.authorizationService.authorizeForce(Permission.ManageElastic);
+		
+		if (!this.enabled()) return;
+		boolean exists = this.existsDescriptionIndex();
+		if (exists) return ;
+		this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDescriptionIndexName()).build());
+	}
+
+	@Override
+	public void resetDmpIndex() throws IOException {
+		logger.debug(new MapLogEntry("reset dmp index"));
+		this.authorizationService.authorizeForce(Permission.ManageElastic);
+		
+		if (!this.enabled()) return;
+		this.deleteDmpIndex();
+
+		int page = 0;
+		int pageSize = this.appElasticProperties.getResetBatchSize();
+		List<DmpEntity> items;
+		do {
+			DmpQuery query = this.queryFactory.query(DmpQuery.class);
+			query.setOrder(new Ordering().addAscending(Dmp._createdAt));
+			query.setPage(new Paging(page * pageSize, pageSize));
+
+			items = this.queryFactory.query(DmpQuery.class).collect();
+			if (items != null && !items.isEmpty()) {
+				List<DmpElasticEntity> elasticEntities = this.builderFactory.builder(DmpElasticBuilder.class).build(items);
+				elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
+				page++;
+			}
+		} while (items != null && !items.isEmpty());
+	}
+
+	@Override
+	public void resetDescriptionIndex() throws IOException {
+		logger.debug(new MapLogEntry("reset description index"));
+		this.authorizationService.authorizeForce(Permission.ManageElastic);
+		
+		if (!this.enabled()) return;
+		this.deleteDescriptionIndex();
+
+		int page = 0;
+		int pageSize = this.appElasticProperties.getResetBatchSize();
+		List<DescriptionEntity> items;
+		do {
+			DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class);
+			query.setOrder(new Ordering().addAscending(Description._createdAt));
+			query.setPage(new Paging(page * pageSize, pageSize));
+
+			items = this.queryFactory.query(DescriptionQuery.class).collect();
+			if (items != null && !items.isEmpty()) {
+				List<DescriptionElasticEntity> elasticEntities = this.builderFactory.builder(DescriptionElasticBuilder.class).build(items);
+				elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName()));
+				page++;
+			}
+		} while (items != null && !items.isEmpty());
+	}
+	
+	//endregion
 }

dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java

@@ -52,14 +52,13 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
     private final BuilderFactory builderFactory;
     private final ConventionService conventionService;
     private final MessageSource messageSource;
-    private final QueryFactory queryFactory;
     private final XmlHandlingService xmlHandlingService;
     private final ErrorThesaurusProperties errors;
 
 
     public ReferenceTypeServiceImpl(
             EntityManager entityManager, AuthorizationService authorizationService, DeleterFactory deleterFactory, BuilderFactory builderFactory,
-            ConventionService conventionService, MessageSource messageSource, QueryFactory queryFactory,
+            ConventionService conventionService, MessageSource messageSource,
             XmlHandlingService xmlHandlingService, ErrorThesaurusProperties errors) {
         this.entityManager = entityManager;
         this.authorizationService = authorizationService;
@@ -67,7 +66,6 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
         this.builderFactory = builderFactory;
         this.conventionService = conventionService;
         this.messageSource = messageSource;
-        this.queryFactory = queryFactory;
         this.xmlHandlingService = xmlHandlingService;
         this.errors = errors;
     }

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDao.java

@@ -1,10 +0,0 @@
-package eu.eudat.data.dao.entities;
-
-import eu.eudat.data.DmpDescriptionTemplateEntity;
-import eu.eudat.data.dao.DatabaseAccessLayer;
-import eu.eudat.model.DmpDescriptionTemplate;
-
-import java.util.UUID;
-
-public interface DmpDatasetProfileDao extends DatabaseAccessLayer<DmpDescriptionTemplateEntity, UUID> {
-}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDaoImpl.java

@@ -1,53 +0,0 @@
-package eu.eudat.data.dao.entities;
-
-import eu.eudat.data.DmpDescriptionTemplateEntity;
-import eu.eudat.data.dao.DatabaseAccess;
-import eu.eudat.data.dao.databaselayer.service.DatabaseService;
-import eu.eudat.model.DmpDescriptionTemplate;
-import eu.eudat.queryable.QueryableList;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.Async;
-import org.springframework.stereotype.Service;
-
-import javax.management.InvalidApplicationException;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-
-@Service("dmpDatasetProfileDao")
-public class DmpDatasetProfileDaoImpl extends DatabaseAccess<DmpDescriptionTemplateEntity> implements DmpDatasetProfileDao {
-    @Autowired
-    public DmpDatasetProfileDaoImpl(DatabaseService<DmpDescriptionTemplateEntity> databaseService) {
-        super(databaseService);
-    }
-
-    @Override
-    public DmpDescriptionTemplateEntity createOrUpdate(DmpDescriptionTemplateEntity item) {
-        return this.getDatabaseService().createOrUpdate(item, DmpDescriptionTemplateEntity.class);
-    }
-
-    @Override
-    @Async
-    public CompletableFuture<DmpDescriptionTemplateEntity> createOrUpdateAsync(DmpDescriptionTemplateEntity item) {
-        return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
-    }
-
-    @Override
-    public DmpDescriptionTemplateEntity find(UUID id) throws InvalidApplicationException {
-        return this.getDatabaseService().getQueryable(DmpDescriptionTemplateEntity.class).where((builder, root) -> builder.equal(root.get("id"), id)).getSingle();
-    }
-
-    @Override
-    public DmpDescriptionTemplateEntity find(UUID id, String hint) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void delete(DmpDescriptionTemplateEntity item) {
-        this.getDatabaseService().delete(item);
-    }
-
-    @Override
-    public QueryableList<DmpDescriptionTemplateEntity> asQueryable() {
-        return this.getDatabaseService().getQueryable(DmpDescriptionTemplateEntity.class);
-    }
-}

dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepository.java

@@ -12,8 +12,6 @@ public interface DatabaseRepository {
 
     DMPDao getDmpDao();
 
-    DmpDatasetProfileDao getDmpDatasetProfileDao();
-
     OrganisationDao getOrganisationDao();
 
     GrantDao getGrantDao();

dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepositoryImpl.java

@@ -16,8 +16,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
 
     private DMPDao dmpDao;
 
-    private DmpDatasetProfileDao dmpDatasetProfileDao;
-
     private OrganisationDao organisationDao;
 
     private GrantDao GrantDao;
@@ -68,11 +66,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
         this.dmpDao = dmpDao;
     }
 
-    @Autowired
-    private void setDmpDatasetProfileDao(DmpDatasetProfileDao dmpDatasetProfileDao) {
-        this.dmpDatasetProfileDao = dmpDatasetProfileDao;
-    }
-
     @Autowired
     private void setOrganisationDao(OrganisationDao organisationDao) {
         this.organisationDao = organisationDao;
@@ -119,11 +112,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
         return dmpDao;
     }
 
-    @Override
-    public DmpDatasetProfileDao getDmpDatasetProfileDao() {
-        return dmpDatasetProfileDao;
-    }
-
     @Override
     public OrganisationDao getOrganisationDao() {
         return organisationDao;

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java

@@ -1,28 +1,21 @@
 package eu.eudat.models.rda.mapper;
 
-import eu.eudat.commons.enums.IsActive;
-import eu.eudat.data.DescriptionTemplateEntity;
 import eu.eudat.data.DmpDescriptionTemplateEntity;
 import eu.eudat.data.DmpEntity;
-import eu.eudat.data.EntityDoiEntity;
-import eu.eudat.data.old.*;
 import eu.eudat.logic.services.ApiContext;
-import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
 import eu.eudat.models.rda.Dmp;
-import eu.eudat.models.rda.DmpId;
+import eu.eudat.query.DmpDescriptionTemplateQuery;
-import net.minidev.json.JSONObject;
+import gr.cite.tools.data.query.QueryFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import jakarta.persistence.NoResultException;
 import jakarta.transaction.Transactional;
 
 import javax.management.InvalidApplicationException;
 import java.io.IOException;
 import java.util.*;
-import java.util.stream.Collectors;
 
 @Component
 public class DmpRDAMapper {
@@ -31,13 +24,15 @@ public class DmpRDAMapper {
 
 	private DatasetRDAMapper datasetRDAMapper;
 	private ApiContext apiContext;
+	private final QueryFactory queryFactory;
 
 
 	@Autowired
-	public DmpRDAMapper(DatasetRDAMapper datasetRDAMapper, ApiContext apiContext) throws IOException {
+	public DmpRDAMapper(DatasetRDAMapper datasetRDAMapper, ApiContext apiContext, QueryFactory queryFactory) throws IOException {
 		this.datasetRDAMapper = datasetRDAMapper;
 		this.apiContext = apiContext;
 
+		this.queryFactory = queryFactory;
 	}
 
 	@Transactional
@@ -190,9 +185,6 @@ public class DmpRDAMapper {
 	}
 
 	private DmpDescriptionTemplateEntity getProfile(String descriptionTemplateId, UUID dmpId) throws InvalidApplicationException {
-		return  apiContext.getOperationsContext().getDatabaseRepository().getDmpDatasetProfileDao().asQueryable().where(((builder, root) -> builder.and(
+		return  this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(dmpId).descriptionTemplateIds(UUID.fromString(descriptionTemplateId)).first();
-				builder.equal(root.get("datasetprofile"), UUID.fromString(descriptionTemplateId)),
-				builder.equal(root.get("dmp"), dmpId))
-		)).getSingleOrDefault();
 	}
 }

dmp-backend/web/src/main/resources/config/permissions.yml

@@ -81,7 +81,13 @@ permissions:
       allowAnonymous: true
       allowAuthenticated: true
       
-    
+    # Elastic
+    ManageElastic:
+      roles:
+        - Admin
+      clients: [ ]
+      allowAnonymous: false
+      allowAuthenticated: false
     
     # Language
     BrowseLanguage: