MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

elastic manage service

This commit is contained in:
Efstratios Giannopoulos 2023-11-21 13:15:31 +02:00
parent e62d765796
commit c0e57b363d
12 changed files with 129 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dmp-backend/core/src/main/java/eu/eudat/authorization/Permission.java
View File

@ -22,7 +22,8 @@ public final class Permission {
public static String PublicBrowseReference = "PublicBrowseReference";
public static String PublicBrowseUser = "PublicBrowseUser";
//Elastic
public static String ManageElastic = "ManageElastic";
//Language
public static String BrowseLanguage = "BrowseLanguage";

9
dmp-backend/core/src/main/java/eu/eudat/configurations/elastic/AppElasticProperties.java
View File

@ -7,6 +7,7 @@ public class AppElasticProperties {
private boolean enabled;
private String dmpIndexName;
private String descriptionIndexName;
private int resetBatchSize;
private boolean enableIcuAnalysisPlugin;
public String getDmpIndexName() {
@ -40,4 +41,12 @@ public class AppElasticProperties {
public void setDescriptionIndexName(String descriptionIndexName) {
this.descriptionIndexName = descriptionIndexName;
}
public int getResetBatchSize() {
return resetBatchSize;
}
public void setResetBatchSize(int resetBatchSize) {
this.resetBatchSize = resetBatchSize;
}
}

17
dmp-backend/core/src/main/java/eu/eudat/data/DmpDescriptionTemplateEntity.java
View File

@ -11,7 +11,7 @@ import java.util.UUID;
@Entity
@Table(name = "\"DmpDescriptionTemplate\"")
public class DmpDescriptionTemplateEntity implements DataEntity<DmpDescriptionTemplateEntity, UUID> {
public class DmpDescriptionTemplateEntity {
@Id
@Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
@ -105,19 +105,4 @@ public class DmpDescriptionTemplateEntity implements DataEntity<DmpDescriptionTe
public void setIsActive(IsActive isActive) {
this.isActive = isActive;
}
@Override
public void update(DmpDescriptionTemplateEntity entity) {
}
@Override
public UUID getKeys() {
return null;
}
@Override
public DmpDescriptionTemplateEntity buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
return null;
}
}

8
dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticService.java
View File

@ -24,4 +24,12 @@ public interface ElasticService {
void persistDescription(DescriptionEntity description) throws IOException;
void deleteDescription(DescriptionEntity description) throws IOException;
void deleteDmpIndex() throws IOException;
void deleteDescriptionIndex() throws IOException;
void resetDmpIndex() throws IOException;
void resetDescriptionIndex() throws IOException;
}

97
dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java
View File

@ -4,6 +4,7 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.mapping.Property;
import co.elastic.clients.elasticsearch._types.mapping.TypeMapping;
import co.elastic.clients.elasticsearch.indices.*;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.configurations.elastic.AppElasticProperties;
import eu.eudat.data.DescriptionEntity;
@ -17,12 +18,19 @@ import eu.eudat.model.Description;
import eu.eudat.model.Dmp;
import eu.eudat.query.DescriptionQuery;
import eu.eudat.query.DmpQuery;
import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.Paging;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.elastic.ElasticConstants;
import gr.cite.tools.exception.MyNotFoundException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry;
import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.data.elasticsearch.annotations.FieldType;
@ -33,10 +41,10 @@ import java.util.*;
import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.stereotype.Service;
import org.springframework.web.context.annotation.RequestScope;
@Service
public class ElasticServiceImpl implements ElasticService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpBlueprintServiceImpl.class));
public final AppElasticProperties appElasticProperties;
private final ElasticsearchClient restHighLevelClient;
private final ElasticsearchTemplate elasticsearchTemplate;
@ -44,8 +52,9 @@ public class ElasticServiceImpl implements ElasticService {
private final BuilderFactory builderFactory;
private final EntityManager entityManager;
private final MessageSource messageSource;
private final AuthorizationService authorizationService;
public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource) {
public ElasticServiceImpl(AppElasticProperties appElasticProperties, ElasticsearchClient restHighLevelClient, ElasticsearchTemplate elasticsearchTemplate, QueryFactory queryFactory, BuilderFactory builderFactory, EntityManager entityManager, MessageSource messageSource, AuthorizationService authorizationService) {
this.appElasticProperties = appElasticProperties;
this.restHighLevelClient = restHighLevelClient;
this.elasticsearchTemplate = elasticsearchTemplate;
@ -53,6 +62,7 @@ public class ElasticServiceImpl implements ElasticService {
this.builderFactory = builderFactory;
this.entityManager = entityManager;
this.messageSource = messageSource;
this.authorizationService = authorizationService;
}
@Override
@ -115,7 +125,7 @@ public class ElasticServiceImpl implements ElasticService {
indexSettingsAnalysis.analyzer("icu_analyzer_text", ab -> ab.custom(x-> x.filter("icu_folding", "english_stop", "english_stemmer").tokenizer("standard")));
}
indexSettings.analysis(indexSettingsAnalysis.build());
CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build());
restHighLevelClient.indices().create(new CreateIndexRequest.Builder().index(indexName).mappings(typeMapping.build()).settings(indexSettings.build()).build());
}
@ -252,6 +262,8 @@ public class ElasticServiceImpl implements ElasticService {
//endregion
//region persist delete
@Override
public void persistDmp(DmpEntity dmp) throws IOException {
if (!this.enabled()) return;
@ -303,4 +315,83 @@ public class ElasticServiceImpl implements ElasticService {
this.elasticsearchTemplate.save(dmpElasticEntity, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
}
}
//endregion
//region manage
@Override
public void deleteDmpIndex() throws IOException {
logger.debug(new MapLogEntry("delete dmp index"));
this.authorizationService.authorizeForce(Permission.ManageElastic);
if (!this.enabled()) return;
boolean exists = this.existsDmpIndex();
if (exists) return ;
this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).build());
}
@Override
public void deleteDescriptionIndex() throws IOException {
logger.debug(new MapLogEntry("delete description index"));
this.authorizationService.authorizeForce(Permission.ManageElastic);
if (!this.enabled()) return;
boolean exists = this.existsDescriptionIndex();
if (exists) return ;
this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDescriptionIndexName()).build());
}
@Override
public void resetDmpIndex() throws IOException {
logger.debug(new MapLogEntry("reset dmp index"));
this.authorizationService.authorizeForce(Permission.ManageElastic);
if (!this.enabled()) return;
this.deleteDmpIndex();
int page = 0;
int pageSize = this.appElasticProperties.getResetBatchSize();
List<DmpEntity> items;
do {
DmpQuery query = this.queryFactory.query(DmpQuery.class);
query.setOrder(new Ordering().addAscending(Dmp._createdAt));
query.setPage(new Paging(page * pageSize, pageSize));
items = this.queryFactory.query(DmpQuery.class).collect();
if (items != null && !items.isEmpty()) {
List<DmpElasticEntity> elasticEntities = this.builderFactory.builder(DmpElasticBuilder.class).build(items);
elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
page++;
}
} while (items != null && !items.isEmpty());
}
@Override
public void resetDescriptionIndex() throws IOException {
logger.debug(new MapLogEntry("reset description index"));
this.authorizationService.authorizeForce(Permission.ManageElastic);
if (!this.enabled()) return;
this.deleteDescriptionIndex();
int page = 0;
int pageSize = this.appElasticProperties.getResetBatchSize();
List<DescriptionEntity> items;
do {
DescriptionQuery query = this.queryFactory.query(DescriptionQuery.class);
query.setOrder(new Ordering().addAscending(Description._createdAt));
query.setPage(new Paging(page * pageSize, pageSize));
items = this.queryFactory.query(DescriptionQuery.class).collect();
if (items != null && !items.isEmpty()) {
List<DescriptionElasticEntity> elasticEntities = this.builderFactory.builder(DescriptionElasticBuilder.class).build(items);
elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName()));
page++;
}
} while (items != null && !items.isEmpty());
}
//endregion
}

4
dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java
View File

@ -52,14 +52,13 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
private final BuilderFactory builderFactory;
private final ConventionService conventionService;
private final MessageSource messageSource;
private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService;
private final ErrorThesaurusProperties errors;
public ReferenceTypeServiceImpl(
EntityManager entityManager, AuthorizationService authorizationService, DeleterFactory deleterFactory, BuilderFactory builderFactory,
ConventionService conventionService, MessageSource messageSource, QueryFactory queryFactory,
ConventionService conventionService, MessageSource messageSource,
XmlHandlingService xmlHandlingService, ErrorThesaurusProperties errors) {
this.entityManager = entityManager;
this.authorizationService = authorizationService;
@ -67,7 +66,6 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
this.builderFactory = builderFactory;
this.conventionService = conventionService;
this.messageSource = messageSource;
this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService;
this.errors = errors;
}

10
dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDao.java
View File

@ -1,10 +0,0 @@
package eu.eudat.data.dao.entities;
import eu.eudat.data.DmpDescriptionTemplateEntity;
import eu.eudat.data.dao.DatabaseAccessLayer;
import eu.eudat.model.DmpDescriptionTemplate;
import java.util.UUID;
public interface DmpDatasetProfileDao extends DatabaseAccessLayer<DmpDescriptionTemplateEntity, UUID> {
}

53
dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DmpDatasetProfileDaoImpl.java
View File

@ -1,53 +0,0 @@
package eu.eudat.data.dao.entities;
import eu.eudat.data.DmpDescriptionTemplateEntity;
import eu.eudat.data.dao.DatabaseAccess;
import eu.eudat.data.dao.databaselayer.service.DatabaseService;
import eu.eudat.model.DmpDescriptionTemplate;
import eu.eudat.queryable.QueryableList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.management.InvalidApplicationException;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@Service("dmpDatasetProfileDao")
public class DmpDatasetProfileDaoImpl extends DatabaseAccess<DmpDescriptionTemplateEntity> implements DmpDatasetProfileDao {
@Autowired
public DmpDatasetProfileDaoImpl(DatabaseService<DmpDescriptionTemplateEntity> databaseService) {
super(databaseService);
}
@Override
public DmpDescriptionTemplateEntity createOrUpdate(DmpDescriptionTemplateEntity item) {
return this.getDatabaseService().createOrUpdate(item, DmpDescriptionTemplateEntity.class);
}
@Override
@Async
public CompletableFuture<DmpDescriptionTemplateEntity> createOrUpdateAsync(DmpDescriptionTemplateEntity item) {
return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
}
@Override
public DmpDescriptionTemplateEntity find(UUID id) throws InvalidApplicationException {
return this.getDatabaseService().getQueryable(DmpDescriptionTemplateEntity.class).where((builder, root) -> builder.equal(root.get("id"), id)).getSingle();
}
@Override
public DmpDescriptionTemplateEntity find(UUID id, String hint) {
throw new UnsupportedOperationException();
}
@Override
public void delete(DmpDescriptionTemplateEntity item) {
this.getDatabaseService().delete(item);
}
@Override
public QueryableList<DmpDescriptionTemplateEntity> asQueryable() {
return this.getDatabaseService().getQueryable(DmpDescriptionTemplateEntity.class);
}
}

2
dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepository.java
View File

@ -12,8 +12,6 @@ public interface DatabaseRepository {
DMPDao getDmpDao();
DmpDatasetProfileDao getDmpDatasetProfileDao();
OrganisationDao getOrganisationDao();
GrantDao getGrantDao();

12
dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/DatabaseRepositoryImpl.java
View File

@ -16,8 +16,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
private DMPDao dmpDao;
private DmpDatasetProfileDao dmpDatasetProfileDao;
private OrganisationDao organisationDao;
private GrantDao GrantDao;
@ -68,11 +66,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
this.dmpDao = dmpDao;
}
@Autowired
private void setDmpDatasetProfileDao(DmpDatasetProfileDao dmpDatasetProfileDao) {
this.dmpDatasetProfileDao = dmpDatasetProfileDao;
}
@Autowired
private void setOrganisationDao(OrganisationDao organisationDao) {
this.organisationDao = organisationDao;
@ -119,11 +112,6 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
return dmpDao;
}
@Override
public DmpDatasetProfileDao getDmpDatasetProfileDao() {
return dmpDatasetProfileDao;
}
@Override
public OrganisationDao getOrganisationDao() {
return organisationDao;

20
dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java
View File

@ -1,28 +1,21 @@
package eu.eudat.models.rda.mapper;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.DescriptionTemplateEntity;
import eu.eudat.data.DmpDescriptionTemplateEntity;
import eu.eudat.data.DmpEntity;
import eu.eudat.data.EntityDoiEntity;
import eu.eudat.data.old.*;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
import eu.eudat.models.rda.Dmp;
import eu.eudat.models.rda.DmpId;
import net.minidev.json.JSONObject;
import eu.eudat.query.DmpDescriptionTemplateQuery;
import gr.cite.tools.data.query.QueryFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import jakarta.persistence.NoResultException;
import jakarta.transaction.Transactional;
import javax.management.InvalidApplicationException;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
@Component
public class DmpRDAMapper {
@ -31,13 +24,15 @@ public class DmpRDAMapper {
private DatasetRDAMapper datasetRDAMapper;
private ApiContext apiContext;
private final QueryFactory queryFactory;
@Autowired
public DmpRDAMapper(DatasetRDAMapper datasetRDAMapper, ApiContext apiContext) throws IOException {
public DmpRDAMapper(DatasetRDAMapper datasetRDAMapper, ApiContext apiContext, QueryFactory queryFactory) throws IOException {
this.datasetRDAMapper = datasetRDAMapper;
this.apiContext = apiContext;
this.queryFactory = queryFactory;
}
@Transactional
@ -190,9 +185,6 @@ public class DmpRDAMapper {
}
private DmpDescriptionTemplateEntity getProfile(String descriptionTemplateId, UUID dmpId) throws InvalidApplicationException {
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDatasetProfileDao().asQueryable().where(((builder, root) -> builder.and(
builder.equal(root.get("datasetprofile"), UUID.fromString(descriptionTemplateId)),
builder.equal(root.get("dmp"), dmpId))
)).getSingleOrDefault();
return this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(dmpId).descriptionTemplateIds(UUID.fromString(descriptionTemplateId)).first();
}
}

8
dmp-backend/web/src/main/resources/config/permissions.yml
View File

@ -81,7 +81,13 @@ permissions:
allowAnonymous: true
allowAuthenticated: true
# Elastic
ManageElastic:
roles:
- Admin
clients: [ ]
allowAnonymous: false
allowAuthenticated: false
# Language
BrowseLanguage: