elastic changes

This commit is contained in:
Efstratios Giannopoulos 2024-01-09 14:53:48 +02:00
parent 3f85f1c9af
commit 1f92872158
10 changed files with 119 additions and 112 deletions

View File

@ -37,6 +37,7 @@ public class AuditableAction {
public static final EventId Dmp_RemoveUser = new EventId(5007, "Dmp_RemoveUser");
public static final EventId Dmp_Invite_Users = new EventId(5008, "Dmp_Invite_Users");
public static final EventId Dmp_Invite_Accept = new EventId(5009, "Dmp_Invite_Accept");
public static final EventId Dmp_PublicQuery = new EventId(5010, "Dmp_PublicQuery");
public static final EventId Description_Query = new EventId(6000, "Description_Query");
public static final EventId Description_Lookup = new EventId(6001, "Description_Lookup");
@ -133,5 +134,7 @@ public class AuditableAction {
public static final EventId Prefilling_Generate = new EventId(210001, "Prefilling_Generate");
public static final EventId Prefilling_GenerateWithData = new EventId(210002, "Prefilling_GenerateWithData");
public static final EventId Maintenance_GenerateElastic = new EventId(220000, "Maintenance_GenerateElastic");
public static final EventId Maintenance_ClearElastic = new EventId(230000, "Maintenance_ClearElastic");
}

View File

@ -1,5 +1,6 @@
package eu.eudat.elastic.query;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.enums.DescriptionStatus;
@ -196,9 +197,26 @@ public class DescriptionElasticQuery extends ElasticQuery<DescriptionElasticEnti
if (!like.startsWith("*")) like = "*" + like;
if (!like.endsWith("*")) like = like + "*";
ElasticFields elasticFields = this.elasticFieldsOf();
elasticFields.add(DescriptionElasticEntity._label);
elasticFields.add(DescriptionElasticEntity._description);
predicates.add(this.like(elasticFields, List.of(like))._toQuery());
elasticFields.add("*", null, true);
predicates.add(this.or(
this.like(elasticFields, List.of(like))._toQuery(),
QueryBuilders.nested().path(DescriptionElasticEntity._tags).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DescriptionElasticEntity._references).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DescriptionElasticEntity._dmp + "." + NestedDmpElasticEntity._references).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DescriptionElasticEntity._dmp + "." + NestedDmpElasticEntity._collaborators).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DescriptionElasticEntity._dmp + "." + NestedDmpElasticEntity._dois).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery()
)._toQuery());
}
if (ids != null) {
predicates.add(this.containsUUID(this.elasticFieldOf(DescriptionElasticEntity._id), ids)._toQuery());

View File

@ -1,5 +1,6 @@
package eu.eudat.elastic.query;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.enums.DmpAccessType;
@ -9,7 +10,9 @@ import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.configurations.elastic.AppElasticProperties;
import eu.eudat.data.DmpEntity;
import eu.eudat.elastic.data.DescriptionElasticEntity;
import eu.eudat.elastic.data.DmpElasticEntity;
import eu.eudat.elastic.data.nested.NestedDescriptionElasticEntity;
import eu.eudat.query.DmpQuery;
import eu.eudat.query.utils.QueryUtilsService;
import eu.eudat.service.elastic.ElasticService;
@ -215,12 +218,31 @@ public class DmpElasticQuery extends ElasticQuery<DmpElasticEntity, UUID> {
List<Query> predicates = new ArrayList<>();
if (like != null && !like.isBlank()) {
if (!like.startsWith("*")) like = "*" + like;
if (!like.endsWith("*")) like = like + "*";
ElasticFields elasticFields = this.elasticFieldsOf();
elasticFields.add(DmpElasticEntity._label);
elasticFields.add(DmpElasticEntity._descriptions);
predicates.add(this.like(elasticFields, List.of(like))._toQuery());
elasticFields.add("*", null, true);
predicates.add(this.or(
this.like(elasticFields, List.of(like))._toQuery(),
QueryBuilders.nested().path(DmpElasticEntity._collaborators).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DmpElasticEntity._references).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DmpElasticEntity._descriptions + "." + NestedDescriptionElasticEntity._references).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DmpElasticEntity._descriptions + "." + NestedDescriptionElasticEntity._tags).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery(),
QueryBuilders.nested().path(DmpElasticEntity._descriptions).query(
this.like(elasticFields, List.of(like))._toQuery()
).build()._toQuery()
)._toQuery());
}
if (ids != null) {
predicates.add(this.containsUUID(this.elasticFieldOf(DmpElasticEntity._id), ids)._toQuery());

View File

@ -210,10 +210,9 @@ public class EntityDoiQuery extends QueryBase<EntityDoiEntity> {
if (this.entityIds != null) {
CriteriaBuilder.In<UUID> inClause = queryContext.CriteriaBuilder.in(queryContext.Root.get(EntityDoiEntity._entityId));
for (UUID item: this.entityIds) {
for (UUID item: this.entityIds)
inClause.value(item);
predicates.add(inClause);
}
predicates.add(inClause);
}
if (!predicates.isEmpty()) {
Predicate[] predicatesArray = predicates.toArray(new Predicate[0]);

View File

@ -136,6 +136,7 @@ public class ElasticServiceImpl implements ElasticService {
propertyMap.put(DescriptionElasticEntity._description, this.createElastic(FieldType.Text, true));
propertyMap.put(DescriptionElasticEntity._status, this.createElastic(FieldType.Short, false));
propertyMap.put(DescriptionElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false));
propertyMap.put(DescriptionElasticEntity._createdAt, this.createElastic(FieldType.Date, false));
propertyMap.put(DescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build());
propertyMap.put(DescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build());
@ -156,6 +157,7 @@ public class ElasticServiceImpl implements ElasticService {
propertyMap.put(DmpElasticEntity._accessType, this.createElastic(FieldType.Short, false));
propertyMap.put(DmpElasticEntity._groupId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(DmpElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false));
propertyMap.put(DmpElasticEntity._versionStatus, this.createElastic(FieldType.Short, false));
propertyMap.put(DmpElasticEntity._descriptions, new Property.Builder().nested(x -> x.properties(this.createNestedDescriptionTemplatePropertyMap())).build());
propertyMap.put(DmpElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build());
@ -175,6 +177,7 @@ public class ElasticServiceImpl implements ElasticService {
propertyMap.put(NestedDescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build());
propertyMap.put(NestedDescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build());
propertyMap.put(NestedDescriptionElasticEntity._descriptionTemplate, new Property.Builder().object(x -> x.properties(this.createNestedDescriptionTemplateTemplatePropertyMap())).build());
return propertyMap;
}
@ -199,6 +202,7 @@ public class ElasticServiceImpl implements ElasticService {
Map<String, Property> propertyMap = new HashMap<>();
propertyMap.put(NestedDescriptionTemplateElasticEntity._id, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDescriptionTemplateElasticEntity._label, this.createElastic(FieldType.Text, true));
propertyMap.put(NestedDescriptionTemplateElasticEntity._versionStatus, this.createElastic(FieldType.Short, true));
return propertyMap;
}
@ -210,6 +214,7 @@ public class ElasticServiceImpl implements ElasticService {
propertyMap.put(NestedDmpElasticEntity._description, this.createElastic(FieldType.Text, false));
propertyMap.put(NestedDmpElasticEntity._status, this.createElastic(FieldType.Short, false));
propertyMap.put(NestedDmpElasticEntity._version, this.createElastic(FieldType.Short, false));
propertyMap.put(NestedDmpElasticEntity._versionStatus, this.createElastic(FieldType.Short, false));
propertyMap.put(NestedDmpElasticEntity._language, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDmpElasticEntity._blueprintId, this.createElastic(FieldType.Keyword, false));
propertyMap.put(NestedDmpElasticEntity._accessType, this.createElastic(FieldType.Short, false));
@ -327,7 +332,7 @@ public class ElasticServiceImpl implements ElasticService {
if (!this.enabled()) return;
boolean exists = this.existsDmpIndex();
if (exists) return ;
if (!exists) return ;
this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).build());
}
@ -339,7 +344,7 @@ public class ElasticServiceImpl implements ElasticService {
if (!this.enabled()) return;
boolean exists = this.existsDescriptionIndex();
if (exists) return ;
if (!exists) return ;
this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDescriptionIndexName()).build());
}
@ -350,6 +355,7 @@ public class ElasticServiceImpl implements ElasticService {
if (!this.enabled()) return;
this.deleteDmpIndex();
this.ensureDmpIndex();
int page = 0;
int pageSize = this.appElasticProperties.getResetBatchSize();
@ -359,7 +365,7 @@ public class ElasticServiceImpl implements ElasticService {
query.setOrder(new Ordering().addAscending(Dmp._createdAt));
query.setPage(new Paging(page * pageSize, pageSize));
items = this.queryFactory.query(DmpQuery.class).collect();
items = query.collect();
if (items != null && !items.isEmpty()) {
List<DmpElasticEntity> elasticEntities = this.builderFactory.builder(DmpElasticBuilder.class).build(items);
elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName()));
@ -375,7 +381,8 @@ public class ElasticServiceImpl implements ElasticService {
if (!this.enabled()) return;
this.deleteDescriptionIndex();
this.ensureDescriptionIndex();
int page = 0;
int pageSize = this.appElasticProperties.getResetBatchSize();
List<DescriptionEntity> items;
@ -384,7 +391,7 @@ public class ElasticServiceImpl implements ElasticService {
query.setOrder(new Ordering().addAscending(Description._createdAt));
query.setPage(new Paging(page * pageSize, pageSize));
items = this.queryFactory.query(DescriptionQuery.class).collect();
items = query.collect();
if (items != null && !items.isEmpty()) {
List<DescriptionElasticEntity> elasticEntities = this.builderFactory.builder(DescriptionElasticBuilder.class).build(items);
elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName()));

View File

@ -1,52 +0,0 @@
package eu.eudat.service.maintance;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.DmpEntity;
import eu.eudat.query.DmpQuery;
import eu.eudat.service.dmp.DmpService;
import eu.eudat.service.elastic.ElasticService;
import gr.cite.tools.data.query.QueryFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.List;
@Service
public class MaintenanceService {
private static final Logger logger = LoggerFactory.getLogger(MaintenanceService.class);
private final QueryFactory queryFactory;
private final ElasticService elasticService;
@Autowired
public MaintenanceService(DmpService dmpService, QueryFactory queryFactory, ElasticService elasticService) {
this.queryFactory = queryFactory;
this.elasticService = elasticService;
}
public void generateElasticIndex() {
List<DmpEntity> dmpEntities = queryFactory.query(DmpQuery.class).isActive(IsActive.Active, IsActive.Inactive).collect();
dmpEntities.forEach(dmpEntity -> {
try {
elasticService.persistDmp(dmpEntity);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
});
}
public void clearElasticIndex() {
List<DmpEntity> dmpEntities = queryFactory.query(DmpQuery.class).isActive(IsActive.Active, IsActive.Inactive).collect();
dmpEntities.forEach(dmpEntity -> {
try {
elasticService.deleteDmp(dmpEntity);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
});
}
}

View File

@ -1,28 +1,17 @@
package eu.eudat;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.util.StdDateFormat;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import eu.eudat.data.BaseEntity;
import org.jetbrains.annotations.NotNull;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Primary;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.scheduling.annotation.EnableAsync;
import java.util.List;
@SpringBootApplication(scanBasePackages = {
"eu.eudat",
"eu.eudat.depositinterface",
@ -35,7 +24,6 @@ import java.util.List;
"eu.eudat.data"})
@EnableAsync
public class EuDatApplication extends SpringBootServletInitializer {
@Bean
@Primary
public ObjectMapper primaryObjectMapper(Jackson2ObjectMapperBuilder builder) {

View File

@ -63,7 +63,6 @@ public class DescriptionController {
private final MessageSource messageSource;
private final ElasticQueryHelperService elasticQueryHelperService;
private final FileTransformerService fileTransformerService;
public DescriptionController(
BuilderFactory builderFactory,
AuditService auditService,
@ -71,7 +70,7 @@ public class DescriptionController {
CensorFactory censorFactory,
QueryFactory queryFactory,
MessageSource messageSource,
ElasticQueryHelperService elasticQueryHelperService, FileTransformerService fileTransformerService) {
ElasticQueryHelperService elasticQueryHelperService) {
this.builderFactory = builderFactory;
this.auditService = auditService;
this.descriptionService = descriptionService;
@ -79,7 +78,6 @@ public class DescriptionController {
this.queryFactory = queryFactory;
this.messageSource = messageSource;
this.elasticQueryHelperService = elasticQueryHelperService;
this.fileTransformerService = fileTransformerService;
}
@PostMapping("public/query")

View File

@ -5,16 +5,19 @@ import eu.eudat.audit.AuditableAction;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.commons.validation.ValidationFilterAnnotation;
import eu.eudat.data.DmpEntity;
import eu.eudat.model.Dmp;
import eu.eudat.model.DmpUser;
import eu.eudat.model.*;
import eu.eudat.model.builder.DmpBuilder;
import eu.eudat.model.censorship.DmpCensor;
import eu.eudat.model.censorship.PublicDescriptionCensor;
import eu.eudat.model.censorship.PublicDmpCensor;
import eu.eudat.model.persist.*;
import eu.eudat.model.result.QueryResult;
import eu.eudat.models.data.helpers.responses.ResponseItem;
import eu.eudat.query.DmpQuery;
import eu.eudat.query.lookup.DescriptionLookup;
import eu.eudat.query.lookup.DmpLookup;
import eu.eudat.service.dmp.DmpService;
import eu.eudat.service.elastic.ElasticQueryHelperService;
import eu.eudat.service.transformer.FileTransformerService;
import eu.eudat.types.ApiMessageCode;
import gr.cite.tools.auditing.AuditService;
@ -42,6 +45,8 @@ import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.util.*;
import static eu.eudat.authorization.AuthorizationFlags.Public;
@RestController
@RequestMapping(path = "api/dmp")
public class DmpController {
@ -59,23 +64,38 @@ public class DmpController {
private final QueryFactory queryFactory;
private final MessageSource messageSource;
private final ElasticQueryHelperService elasticQueryHelperService;
private final FileTransformerService fileTransformerService;
public DmpController(
BuilderFactory builderFactory,
AuditService auditService,
DmpService dmpService,
CensorFactory censorFactory,
QueryFactory queryFactory,
MessageSource messageSource, FileTransformerService fileTransformerService) {
BuilderFactory builderFactory,
AuditService auditService,
DmpService dmpService,
CensorFactory censorFactory,
QueryFactory queryFactory,
MessageSource messageSource,
ElasticQueryHelperService elasticQueryHelperService) {
this.builderFactory = builderFactory;
this.auditService = auditService;
this.dmpService = dmpService;
this.censorFactory = censorFactory;
this.queryFactory = queryFactory;
this.messageSource = messageSource;
this.fileTransformerService = fileTransformerService;
this.elasticQueryHelperService = elasticQueryHelperService;
}
@PostMapping("public/query")
public QueryResult<PublicDmp> publicQuery(@RequestBody DmpLookup lookup) throws MyApplicationException, MyForbiddenException {
logger.debug("querying {}", Dmp.class.getSimpleName());
this.censorFactory.censor(PublicDmpCensor.class).censor(lookup.getProject());
//DmpQuery query = lookup.enrich(this.queryFactory).authorize(EnumSet.of(Public)).dmpSubQuery(this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public));
QueryResult<PublicDmp> queryResult = this.elasticQueryHelperService.collectPublic(lookup, EnumSet.of(Public), null);
this.auditService.track(AuditableAction.Dmp_PublicQuery, "lookup", lookup);
return queryResult;
}
@PostMapping("query")
@ -84,15 +104,12 @@ public class DmpController {
this.censorFactory.censor(DmpCensor.class).censor(lookup.getProject(), null);
DmpQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic);
List<DmpEntity> data = query.collectAs(lookup.getProject());
List<Dmp> models = this.builderFactory.builder(DmpBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data);
long count = (lookup.getMetadata() != null && lookup.getMetadata().getCountAll()) ? query.count() : models.size();
QueryResult<Dmp> queryResult = this.elasticQueryHelperService.collect(lookup, AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic, null);
this.auditService.track(AuditableAction.Dmp_Query, "lookup", lookup);
return new QueryResult<>(models, count);
return queryResult;
}
@GetMapping("{id}")

View File

@ -1,11 +1,16 @@
package eu.eudat.controllers.v2;
import eu.eudat.audit.AuditableAction;
import eu.eudat.authorization.Permission;
import eu.eudat.data.DescriptionEntity;
import eu.eudat.model.Lock;
import eu.eudat.models.data.helpers.responses.ResponseItem;
import eu.eudat.service.maintance.MaintenanceService;
import eu.eudat.service.elastic.ElasticService;
import eu.eudat.types.ApiMessageCode;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.auditing.AuditService;
import gr.cite.tools.logging.LoggerService;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
@ -18,34 +23,36 @@ import org.springframework.web.bind.annotation.RestController;
@RequestMapping(path = "api/maintenance")
public class MaintenanceController {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(MaintenanceController.class));
private final AuthorizationService authorizationService;
private final MaintenanceService maintenanceService;
private final ElasticService elasticService;
private final AuditService auditService;
@Autowired
public MaintenanceController(AuthorizationService authorizationService, MaintenanceService maintenanceService) {
public MaintenanceController(AuthorizationService authorizationService, ElasticService elasticService, AuditService auditService) {
this.authorizationService = authorizationService;
this.maintenanceService = maintenanceService;
this.elasticService = elasticService;
this.auditService = auditService;
}
/*
* Data Index
* */
@RequestMapping(method = RequestMethod.POST, value = {"/index/elastic"})
public @ResponseBody
ResponseEntity<ResponseItem<DescriptionEntity>> generateIndex() throws Exception {
public void generateIndex() throws Exception {
logger.debug("generate elastic ");
this.authorizationService.authorizeForce(Permission.ManageElastic);
this.maintenanceService.generateElasticIndex();
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<DescriptionEntity>().status(ApiMessageCode.SUCCESS_MESSAGE).message("Generated").payload(null));
elasticService.resetDmpIndex();
elasticService.resetDescriptionIndex();
this.auditService.track(AuditableAction.Maintenance_GenerateElastic);
}
@RequestMapping(method = RequestMethod.DELETE, value = {"/index/elastic"})
public @ResponseBody
ResponseEntity<ResponseItem<DescriptionEntity>> clearIndex() throws Exception {
public void clearIndex() throws Exception {
logger.debug("clear elastic");
this.authorizationService.authorizeForce(Permission.ManageElastic);
this.maintenanceService.clearElasticIndex();
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<DescriptionEntity>().status(ApiMessageCode.SUCCESS_MESSAGE).message("Cleared").payload(null));
elasticService.deleteDescriptionIndex();
elasticService.deleteDmpIndex();
this.auditService.track(AuditableAction.Maintenance_ClearElastic);
}
}