diff --git a/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java b/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java index 3d614c4c1..a85699e91 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java +++ b/dmp-backend/core/src/main/java/eu/eudat/audit/AuditableAction.java @@ -37,6 +37,7 @@ public class AuditableAction { public static final EventId Dmp_RemoveUser = new EventId(5007, "Dmp_RemoveUser"); public static final EventId Dmp_Invite_Users = new EventId(5008, "Dmp_Invite_Users"); public static final EventId Dmp_Invite_Accept = new EventId(5009, "Dmp_Invite_Accept"); + public static final EventId Dmp_PublicQuery = new EventId(5010, "Dmp_PublicQuery"); public static final EventId Description_Query = new EventId(6000, "Description_Query"); public static final EventId Description_Lookup = new EventId(6001, "Description_Lookup"); @@ -133,5 +134,7 @@ public class AuditableAction { public static final EventId Prefilling_Generate = new EventId(210001, "Prefilling_Generate"); public static final EventId Prefilling_GenerateWithData = new EventId(210002, "Prefilling_GenerateWithData"); + public static final EventId Maintenance_GenerateElastic = new EventId(220000, "Maintenance_GenerateElastic"); + public static final EventId Maintenance_ClearElastic = new EventId(230000, "Maintenance_ClearElastic"); } diff --git a/dmp-backend/core/src/main/java/eu/eudat/elastic/query/DescriptionElasticQuery.java b/dmp-backend/core/src/main/java/eu/eudat/elastic/query/DescriptionElasticQuery.java index d2c50d363..4fb97c821 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/elastic/query/DescriptionElasticQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/elastic/query/DescriptionElasticQuery.java @@ -1,5 +1,6 @@ package eu.eudat.elastic.query; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.Permission; import eu.eudat.commons.enums.DescriptionStatus; @@ -196,9 +197,26 @@ public class DescriptionElasticQuery extends ElasticQuery { List predicates = new ArrayList<>(); if (like != null && !like.isBlank()) { + if (!like.startsWith("*")) like = "*" + like; if (!like.endsWith("*")) like = like + "*"; ElasticFields elasticFields = this.elasticFieldsOf(); - elasticFields.add(DmpElasticEntity._label); - elasticFields.add(DmpElasticEntity._descriptions); - predicates.add(this.like(elasticFields, List.of(like))._toQuery()); + elasticFields.add("*", null, true); + + predicates.add(this.or( + this.like(elasticFields, List.of(like))._toQuery(), + QueryBuilders.nested().path(DmpElasticEntity._collaborators).query( + this.like(elasticFields, List.of(like))._toQuery() + ).build()._toQuery(), + QueryBuilders.nested().path(DmpElasticEntity._references).query( + this.like(elasticFields, List.of(like))._toQuery() + ).build()._toQuery(), + QueryBuilders.nested().path(DmpElasticEntity._descriptions + "." + NestedDescriptionElasticEntity._references).query( + this.like(elasticFields, List.of(like))._toQuery() + ).build()._toQuery(), + QueryBuilders.nested().path(DmpElasticEntity._descriptions + "." + NestedDescriptionElasticEntity._tags).query( + this.like(elasticFields, List.of(like))._toQuery() + ).build()._toQuery(), + QueryBuilders.nested().path(DmpElasticEntity._descriptions).query( + this.like(elasticFields, List.of(like))._toQuery() + ).build()._toQuery() + + )._toQuery()); } if (ids != null) { predicates.add(this.containsUUID(this.elasticFieldOf(DmpElasticEntity._id), ids)._toQuery()); diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java index ece0bab11..da22a63a3 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java @@ -210,10 +210,9 @@ public class EntityDoiQuery extends QueryBase { if (this.entityIds != null) { CriteriaBuilder.In inClause = queryContext.CriteriaBuilder.in(queryContext.Root.get(EntityDoiEntity._entityId)); - for (UUID item: this.entityIds) { + for (UUID item: this.entityIds) inClause.value(item); - predicates.add(inClause); - } + predicates.add(inClause); } if (!predicates.isEmpty()) { Predicate[] predicatesArray = predicates.toArray(new Predicate[0]); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java index 1b80f78ed..c35428ca5 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/elastic/ElasticServiceImpl.java @@ -136,6 +136,7 @@ public class ElasticServiceImpl implements ElasticService { propertyMap.put(DescriptionElasticEntity._description, this.createElastic(FieldType.Text, true)); propertyMap.put(DescriptionElasticEntity._status, this.createElastic(FieldType.Short, false)); propertyMap.put(DescriptionElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false)); + propertyMap.put(DescriptionElasticEntity._createdAt, this.createElastic(FieldType.Date, false)); propertyMap.put(DescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build()); propertyMap.put(DescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build()); @@ -156,6 +157,7 @@ public class ElasticServiceImpl implements ElasticService { propertyMap.put(DmpElasticEntity._accessType, this.createElastic(FieldType.Short, false)); propertyMap.put(DmpElasticEntity._groupId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(DmpElasticEntity._finalizedAt, this.createElastic(FieldType.Date, false)); + propertyMap.put(DmpElasticEntity._versionStatus, this.createElastic(FieldType.Short, false)); propertyMap.put(DmpElasticEntity._descriptions, new Property.Builder().nested(x -> x.properties(this.createNestedDescriptionTemplatePropertyMap())).build()); propertyMap.put(DmpElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build()); @@ -175,6 +177,7 @@ public class ElasticServiceImpl implements ElasticService { propertyMap.put(NestedDescriptionElasticEntity._tags, new Property.Builder().nested(x -> x.properties(this.createNestedTagsTemplatePropertyMap())).build()); propertyMap.put(NestedDescriptionElasticEntity._references, new Property.Builder().nested(x -> x.properties(this.createNestedReferencesTemplatePropertyMap())).build()); + propertyMap.put(NestedDescriptionElasticEntity._descriptionTemplate, new Property.Builder().object(x -> x.properties(this.createNestedDescriptionTemplateTemplatePropertyMap())).build()); return propertyMap; } @@ -199,6 +202,7 @@ public class ElasticServiceImpl implements ElasticService { Map propertyMap = new HashMap<>(); propertyMap.put(NestedDescriptionTemplateElasticEntity._id, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDescriptionTemplateElasticEntity._label, this.createElastic(FieldType.Text, true)); + propertyMap.put(NestedDescriptionTemplateElasticEntity._versionStatus, this.createElastic(FieldType.Short, true)); return propertyMap; } @@ -210,6 +214,7 @@ public class ElasticServiceImpl implements ElasticService { propertyMap.put(NestedDmpElasticEntity._description, this.createElastic(FieldType.Text, false)); propertyMap.put(NestedDmpElasticEntity._status, this.createElastic(FieldType.Short, false)); propertyMap.put(NestedDmpElasticEntity._version, this.createElastic(FieldType.Short, false)); + propertyMap.put(NestedDmpElasticEntity._versionStatus, this.createElastic(FieldType.Short, false)); propertyMap.put(NestedDmpElasticEntity._language, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDmpElasticEntity._blueprintId, this.createElastic(FieldType.Keyword, false)); propertyMap.put(NestedDmpElasticEntity._accessType, this.createElastic(FieldType.Short, false)); @@ -327,7 +332,7 @@ public class ElasticServiceImpl implements ElasticService { if (!this.enabled()) return; boolean exists = this.existsDmpIndex(); - if (exists) return ; + if (!exists) return ; this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDmpIndexName()).build()); } @@ -339,7 +344,7 @@ public class ElasticServiceImpl implements ElasticService { if (!this.enabled()) return; boolean exists = this.existsDescriptionIndex(); - if (exists) return ; + if (!exists) return ; this.restHighLevelClient.indices().delete(new DeleteIndexRequest.Builder().index(this.appElasticProperties.getDescriptionIndexName()).build()); } @@ -350,6 +355,7 @@ public class ElasticServiceImpl implements ElasticService { if (!this.enabled()) return; this.deleteDmpIndex(); + this.ensureDmpIndex(); int page = 0; int pageSize = this.appElasticProperties.getResetBatchSize(); @@ -359,7 +365,7 @@ public class ElasticServiceImpl implements ElasticService { query.setOrder(new Ordering().addAscending(Dmp._createdAt)); query.setPage(new Paging(page * pageSize, pageSize)); - items = this.queryFactory.query(DmpQuery.class).collect(); + items = query.collect(); if (items != null && !items.isEmpty()) { List elasticEntities = this.builderFactory.builder(DmpElasticBuilder.class).build(items); elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDmpIndexName())); @@ -375,7 +381,8 @@ public class ElasticServiceImpl implements ElasticService { if (!this.enabled()) return; this.deleteDescriptionIndex(); - + this.ensureDescriptionIndex(); + int page = 0; int pageSize = this.appElasticProperties.getResetBatchSize(); List items; @@ -384,7 +391,7 @@ public class ElasticServiceImpl implements ElasticService { query.setOrder(new Ordering().addAscending(Description._createdAt)); query.setPage(new Paging(page * pageSize, pageSize)); - items = this.queryFactory.query(DescriptionQuery.class).collect(); + items = query.collect(); if (items != null && !items.isEmpty()) { List elasticEntities = this.builderFactory.builder(DescriptionElasticBuilder.class).build(items); elasticsearchTemplate.save(elasticEntities, IndexCoordinates.of(this.appElasticProperties.getDescriptionIndexName())); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/maintance/MaintenanceService.java b/dmp-backend/core/src/main/java/eu/eudat/service/maintance/MaintenanceService.java deleted file mode 100644 index 3a4794b6e..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/maintance/MaintenanceService.java +++ /dev/null @@ -1,52 +0,0 @@ -package eu.eudat.service.maintance; - -import eu.eudat.commons.enums.IsActive; -import eu.eudat.data.DmpEntity; -import eu.eudat.query.DmpQuery; -import eu.eudat.service.dmp.DmpService; -import eu.eudat.service.elastic.ElasticService; -import gr.cite.tools.data.query.QueryFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.util.List; - -@Service -public class MaintenanceService { - private static final Logger logger = LoggerFactory.getLogger(MaintenanceService.class); - - private final QueryFactory queryFactory; - private final ElasticService elasticService; - - @Autowired - public MaintenanceService(DmpService dmpService, QueryFactory queryFactory, ElasticService elasticService) { - this.queryFactory = queryFactory; - this.elasticService = elasticService; - } - - - public void generateElasticIndex() { - List dmpEntities = queryFactory.query(DmpQuery.class).isActive(IsActive.Active, IsActive.Inactive).collect(); - dmpEntities.forEach(dmpEntity -> { - try { - elasticService.persistDmp(dmpEntity); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - }); - } - - public void clearElasticIndex() { - List dmpEntities = queryFactory.query(DmpQuery.class).isActive(IsActive.Active, IsActive.Inactive).collect(); - dmpEntities.forEach(dmpEntity -> { - try { - elasticService.deleteDmp(dmpEntity); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - }); - } -} diff --git a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java index 3015b3784..ba1dac251 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java +++ b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java @@ -1,28 +1,17 @@ package eu.eudat; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.json.JsonMapper; -import com.fasterxml.jackson.databind.util.StdDateFormat; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import eu.eudat.data.BaseEntity; -import org.jetbrains.annotations.NotNull; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; -import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; -import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.scheduling.annotation.EnableAsync; -import java.util.List; - @SpringBootApplication(scanBasePackages = { "eu.eudat", "eu.eudat.depositinterface", @@ -35,7 +24,6 @@ import java.util.List; "eu.eudat.data"}) @EnableAsync public class EuDatApplication extends SpringBootServletInitializer { - @Bean @Primary public ObjectMapper primaryObjectMapper(Jackson2ObjectMapperBuilder builder) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DescriptionController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DescriptionController.java index 757a2d9d1..b48c96379 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DescriptionController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DescriptionController.java @@ -63,7 +63,6 @@ public class DescriptionController { private final MessageSource messageSource; private final ElasticQueryHelperService elasticQueryHelperService; - private final FileTransformerService fileTransformerService; public DescriptionController( BuilderFactory builderFactory, AuditService auditService, @@ -71,7 +70,7 @@ public class DescriptionController { CensorFactory censorFactory, QueryFactory queryFactory, MessageSource messageSource, - ElasticQueryHelperService elasticQueryHelperService, FileTransformerService fileTransformerService) { + ElasticQueryHelperService elasticQueryHelperService) { this.builderFactory = builderFactory; this.auditService = auditService; this.descriptionService = descriptionService; @@ -79,7 +78,6 @@ public class DescriptionController { this.queryFactory = queryFactory; this.messageSource = messageSource; this.elasticQueryHelperService = elasticQueryHelperService; - this.fileTransformerService = fileTransformerService; } @PostMapping("public/query") diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DmpController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DmpController.java index bc2e64c43..05345a88a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DmpController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/DmpController.java @@ -5,16 +5,19 @@ import eu.eudat.audit.AuditableAction; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.commons.validation.ValidationFilterAnnotation; import eu.eudat.data.DmpEntity; -import eu.eudat.model.Dmp; -import eu.eudat.model.DmpUser; +import eu.eudat.model.*; import eu.eudat.model.builder.DmpBuilder; import eu.eudat.model.censorship.DmpCensor; +import eu.eudat.model.censorship.PublicDescriptionCensor; +import eu.eudat.model.censorship.PublicDmpCensor; import eu.eudat.model.persist.*; import eu.eudat.model.result.QueryResult; import eu.eudat.models.data.helpers.responses.ResponseItem; import eu.eudat.query.DmpQuery; +import eu.eudat.query.lookup.DescriptionLookup; import eu.eudat.query.lookup.DmpLookup; import eu.eudat.service.dmp.DmpService; +import eu.eudat.service.elastic.ElasticQueryHelperService; import eu.eudat.service.transformer.FileTransformerService; import eu.eudat.types.ApiMessageCode; import gr.cite.tools.auditing.AuditService; @@ -42,6 +45,8 @@ import javax.xml.transform.TransformerException; import java.io.IOException; import java.util.*; +import static eu.eudat.authorization.AuthorizationFlags.Public; + @RestController @RequestMapping(path = "api/dmp") public class DmpController { @@ -59,23 +64,38 @@ public class DmpController { private final QueryFactory queryFactory; private final MessageSource messageSource; + private final ElasticQueryHelperService elasticQueryHelperService; - private final FileTransformerService fileTransformerService; public DmpController( - BuilderFactory builderFactory, - AuditService auditService, - DmpService dmpService, - CensorFactory censorFactory, - QueryFactory queryFactory, - MessageSource messageSource, FileTransformerService fileTransformerService) { + BuilderFactory builderFactory, + AuditService auditService, + DmpService dmpService, + CensorFactory censorFactory, + QueryFactory queryFactory, + MessageSource messageSource, + ElasticQueryHelperService elasticQueryHelperService) { this.builderFactory = builderFactory; this.auditService = auditService; this.dmpService = dmpService; this.censorFactory = censorFactory; this.queryFactory = queryFactory; this.messageSource = messageSource; - this.fileTransformerService = fileTransformerService; + this.elasticQueryHelperService = elasticQueryHelperService; + } + + @PostMapping("public/query") + public QueryResult publicQuery(@RequestBody DmpLookup lookup) throws MyApplicationException, MyForbiddenException { + logger.debug("querying {}", Dmp.class.getSimpleName()); + + this.censorFactory.censor(PublicDmpCensor.class).censor(lookup.getProject()); + + //DmpQuery query = lookup.enrich(this.queryFactory).authorize(EnumSet.of(Public)).dmpSubQuery(this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public)); + QueryResult queryResult = this.elasticQueryHelperService.collectPublic(lookup, EnumSet.of(Public), null); + + this.auditService.track(AuditableAction.Dmp_PublicQuery, "lookup", lookup); + + return queryResult; } @PostMapping("query") @@ -84,15 +104,12 @@ public class DmpController { this.censorFactory.censor(DmpCensor.class).censor(lookup.getProject(), null); - DmpQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic); - List data = query.collectAs(lookup.getProject()); - List models = this.builderFactory.builder(DmpBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data); - long count = (lookup.getMetadata() != null && lookup.getMetadata().getCountAll()) ? query.count() : models.size(); + QueryResult queryResult = this.elasticQueryHelperService.collect(lookup, AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic, null); this.auditService.track(AuditableAction.Dmp_Query, "lookup", lookup); - return new QueryResult<>(models, count); + return queryResult; } @GetMapping("{id}") diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/MaintenanceController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/MaintenanceController.java index d64d13867..644fa84b5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/MaintenanceController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/v2/MaintenanceController.java @@ -1,11 +1,16 @@ package eu.eudat.controllers.v2; +import eu.eudat.audit.AuditableAction; import eu.eudat.authorization.Permission; import eu.eudat.data.DescriptionEntity; +import eu.eudat.model.Lock; import eu.eudat.models.data.helpers.responses.ResponseItem; -import eu.eudat.service.maintance.MaintenanceService; +import eu.eudat.service.elastic.ElasticService; import eu.eudat.types.ApiMessageCode; import gr.cite.commons.web.authz.service.AuthorizationService; +import gr.cite.tools.auditing.AuditService; +import gr.cite.tools.logging.LoggerService; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -18,34 +23,36 @@ import org.springframework.web.bind.annotation.RestController; @RequestMapping(path = "api/maintenance") public class MaintenanceController { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(MaintenanceController.class)); private final AuthorizationService authorizationService; - private final MaintenanceService maintenanceService; + private final ElasticService elasticService; + private final AuditService auditService; @Autowired - public MaintenanceController(AuthorizationService authorizationService, MaintenanceService maintenanceService) { + public MaintenanceController(AuthorizationService authorizationService, ElasticService elasticService, AuditService auditService) { this.authorizationService = authorizationService; - this.maintenanceService = maintenanceService; + this.elasticService = elasticService; + this.auditService = auditService; } - /* - * Data Index - * */ @RequestMapping(method = RequestMethod.POST, value = {"/index/elastic"}) - public @ResponseBody - ResponseEntity> generateIndex() throws Exception { + public void generateIndex() throws Exception { + logger.debug("generate elastic "); this.authorizationService.authorizeForce(Permission.ManageElastic); - this.maintenanceService.generateElasticIndex(); - return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.SUCCESS_MESSAGE).message("Generated").payload(null)); + elasticService.resetDmpIndex(); + elasticService.resetDescriptionIndex(); + this.auditService.track(AuditableAction.Maintenance_GenerateElastic); } @RequestMapping(method = RequestMethod.DELETE, value = {"/index/elastic"}) - public @ResponseBody - ResponseEntity> clearIndex() throws Exception { + public void clearIndex() throws Exception { + logger.debug("clear elastic"); this.authorizationService.authorizeForce(Permission.ManageElastic); - - this.maintenanceService.clearElasticIndex(); - return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.SUCCESS_MESSAGE).message("Cleared").payload(null)); + + elasticService.deleteDescriptionIndex(); + elasticService.deleteDmpIndex(); + this.auditService.track(AuditableAction.Maintenance_ClearElastic); } }