diff --git a/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java b/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java index f34c2ec8a..34a314bb1 100644 --- a/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java +++ b/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java @@ -2,46 +2,64 @@ package eu.eudat.elastic.repository; import eu.eudat.elastic.criteria.DatasetCriteria; import eu.eudat.elastic.entities.Dataset; +import eu.eudat.elastic.entities.Dmp; import eu.eudat.elastic.entities.Tag; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; +import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.InnerHitBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters; +import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.springframework.beans.factory.annotation.Autowired; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @Service("datasetRepository") public class DatasetRepository extends ElasticRepository { - @Autowired - public DatasetRepository(RestHighLevelClient client) { + private final DmpRepository dmpRepository; + + public DatasetRepository(RestHighLevelClient client, DmpRepository dmpRepository) { super(client); + this.dmpRepository = dmpRepository; } @Override public Dataset createOrUpdate(Dataset entity) throws IOException { if (this.getClient() != null) { XContentBuilder builder = XContentFactory.jsonBuilder(); - IndexRequest request = new IndexRequest("datasets").id(entity.getId()).source(entity.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder)); + Dmp dmp = this.dmpRepository.findDocument(entity.getDmp().toString()); + boolean found = false; + for(int i = 0; i < dmp.getDatasets().size(); i++) { + if (dmp.getDatasets().get(i).getId().equals(entity.getId())) { + dmp.getDatasets().set(i, entity); + found = true; + break; + } + } + if (!found) { + dmp.getDatasets().add(entity); + } + IndexRequest request = new IndexRequest("dmps").id(dmp.getId().toString()).source(dmp.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder)); this.getClient().index(request, RequestOptions.DEFAULT); return entity; } @@ -51,9 +69,20 @@ public class DatasetRepository extends ElasticRepository)Arrays.stream(response.getHits().getHits()) + .map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream) + .map(SearchHits::getHits).flatMap(Arrays::stream) + .map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).findFirst().orElse(null); +// GetRequest request = new GetRequest("datasets", id); +// GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT); +// return new Dataset().fromElasticEntity(response.getSourceAsMap()); } return null; } @@ -61,70 +90,81 @@ public class DatasetRepository extends ElasticRepository query(DatasetCriteria criteria) throws IOException { if (this.getClient() != null) { - SearchRequest searchRequest = new SearchRequest("datasets"); + SearchRequest searchRequest = new SearchRequest("dmps"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - CountRequest countRequest = new CountRequest("datasets"); - countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())))); + /*CountRequest countRequest = new CountRequest("dmps").routing("datasets").routing("id"); + countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())))); CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT); - Long count = countResponse.getCount(); + Long count = countResponse.getCount();*/ + + SearchRequest countRequest = new SearchRequest("dmps"); + NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets"); + FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())))); + nestedAggregationBuilder.subAggregation(filtersAggregationBuilder); + SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder(); + countSourceBuilder.aggregation(nestedAggregationBuilder); + countRequest.source(countSourceBuilder); + SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT); + Long count = ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount(); + searchSourceBuilder.size(count.intValue()); - BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))); + BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))); if (criteria.isPublic()) { - boolQuery = boolQuery.should(QueryBuilders.termQuery("public.keyword", "true")); - boolQuery = boolQuery.should(QueryBuilders.termQuery("status.keyword", Dataset.Status.FINALISED.getValue())); - boolQuery = boolQuery.should(QueryBuilders.termQuery("lastPublicVersion.keyword", "true")); + boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true")); + boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue())); + boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastPublicVersion", "true")); } if (criteria.getLike() != null && !criteria.getLike().isEmpty()) { - boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{ - {"label", 1.0f}, - {"description", 1.0f}, - {"formData", 1.0f} + boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).allowLeadingWildcard(true).fields(Stream.of(new Object[][]{ + {"datasets.label", 1.0f}, + {"datasets.description", 1.0f}, + {"datasets.formData", 1.0f} }).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1])))); } if (criteria.getDatasetTemplates() != null && criteria.getDatasetTemplates().size() > 0) { - boolQuery = boolQuery.should(QueryBuilders.termsQuery("template.keyword", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList()))); + boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.template", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getStatus() != null) { - boolQuery = boolQuery.should(QueryBuilders.termQuery("status.keyword", criteria.getStatus().toString())); + boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", criteria.getStatus().toString())); } if (criteria.getDmps() != null && criteria.getDmps().size() > 0) { - boolQuery = boolQuery.should(QueryBuilders.termsQuery("dmp.keyword", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList()))); + boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.dmp", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) { - boolQuery = boolQuery.should(QueryBuilders.termsQuery("group.keyword", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList()))); + boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.group", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getGrants() != null && criteria.getGrants().size() > 0) { - boolQuery = boolQuery.should(QueryBuilders.termsQuery("grant.keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList()))); + boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.grant", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getGrantStatus() != null) { - boolQuery = boolQuery.should(QueryBuilders.termQuery("grantStatus.keyword", criteria.getGrantStatus().toString())); + boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.grantStatus", criteria.getGrantStatus().toString())); } if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) { - boolQuery = boolQuery.should(QueryBuilders.termsQuery("collaborators.id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList()))); + boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList()))); } if (!criteria.isPublic()) { if (criteria.getAllowAllVersions() != null && !criteria.getAllowAllVersions()) { - boolQuery = boolQuery.should(QueryBuilders.termQuery("lastVersion.keyword", "true")); + boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastVersion", "true")); } } if (criteria.getOrganiztions() != null && criteria.getOrganiztions().size() > 0) { - boolQuery = boolQuery.should(QueryBuilders.termsQuery("organizations.id.keyword", criteria.getOrganiztions())); + boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.organizations.id", criteria.getOrganiztions())); } if (criteria.getTags() != null && criteria.getTags().size() > 0) { - boolQuery = boolQuery.should(QueryBuilders.termsQuery("tags.name.keyword", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList()))); + boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList()))); } if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) { @@ -132,10 +172,15 @@ public class DatasetRepository extends ElasticRepository this.transformFromString(x.getSourceAsString(), Dataset.class)).collect(Collectors.toList()); + return ((Stream)Arrays.stream(response.getHits().getHits()) + .map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream) + .map(SearchHits::getHits).flatMap(Arrays::stream) + .map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList()); } return null; } @@ -143,7 +188,7 @@ public class DatasetRepository extends ElasticRepository { + private static final Logger logger = LoggerFactory.getLogger(DmpRepository.class); @Autowired public DmpRepository(RestHighLevelClient client) { super(client); } + private void generateMapping() throws IOException { + if (this.getClient() != null) { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("properties"); + builder.startObject("datasets"); + builder.field("type", "nested"); + builder.endObject(); + builder.endObject(); + builder.endObject(); + PutMappingRequest putMappingRequest = new PutMappingRequest("dmps"); + putMappingRequest.source(builder); + this.getClient().indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); + } + } + @Override public Dmp createOrUpdate(Dmp entity) throws IOException { if (this.getClient() != null) { @@ -129,6 +149,20 @@ public class DmpRepository extends ElasticRepository { return null; } + public boolean createIndex() { + try { + if (!this.exists()) { + CreateIndexRequest createIndexRequest = new CreateIndexRequest("dmps"); + this.getClient().indices().create(createIndexRequest, RequestOptions.DEFAULT); + this.generateMapping(); + } + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } + @Override public boolean exists() throws IOException { if (this.getClient() != null) { @@ -144,6 +178,8 @@ public class DmpRepository extends ElasticRepository { DeleteByQueryRequest delete = new DeleteByQueryRequest("dmps"); delete.setQuery(QueryBuilders.matchAllQuery()); this.getClient().deleteByQuery(delete, RequestOptions.DEFAULT); + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("dmps"); + this.getClient().indices().delete(deleteIndexRequest, RequestOptions.DEFAULT); } } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index e15b43edb..ede74e290 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -1738,14 +1738,16 @@ public class DataManagementPlanManager { public void generateIndex(Principal principal) { if (principal.getAuthorities().contains(Authorities.ADMIN.getValue())) { - List dmps = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList(); - dmps.forEach(dmp -> { - try { - this.updateIndex(dmp); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - }); + if (apiContext.getOperationsContext().getElasticRepository().getDmpRepository().createIndex()) { + List dmps = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList(); + dmps.forEach(dmp -> { + try { + this.updateIndex(dmp); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + }); + } } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index a30f89ab1..e82bc1801 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -134,7 +134,7 @@ public class DatasetManager { datasets = datasetRepository.exists() ? datasetRepository.query(datasetCriteria) : null; } catch (Exception ex) { - logger.warn(ex.getMessage()); + logger.warn(ex.getMessage(), ex); datasets = null; } diff --git a/dmp-frontend/src/app/ui/admin/index-managment/index-managment.component.ts b/dmp-frontend/src/app/ui/admin/index-managment/index-managment.component.ts index 699dd0902..51865c403 100644 --- a/dmp-frontend/src/app/ui/admin/index-managment/index-managment.component.ts +++ b/dmp-frontend/src/app/ui/admin/index-managment/index-managment.component.ts @@ -30,16 +30,16 @@ export class IndexManagmentComponent extends BaseComponent implements OnInit { generateIndex(ev: Event) { (ev.srcElement as HTMLButtonElement).disabled = true; - this.datasetService.generateIndex().pipe(takeUntil(this._destroyed)).subscribe( - response => { - (ev.srcElement as HTMLButtonElement).disabled = false; - this.onCallbackSuccess(); - }, - error => { - (ev.srcElement as HTMLButtonElement).disabled = false; - this.onCallbackError(error); - } - ); + // this.datasetService.generateIndex().pipe(takeUntil(this._destroyed)).subscribe( + // response => { + // (ev.srcElement as HTMLButtonElement).disabled = false; + // this.onCallbackSuccess(); + // }, + // error => { + // (ev.srcElement as HTMLButtonElement).disabled = false; + // this.onCallbackError(error); + // } + // ); this.dmpService.generateIndex().pipe(takeUntil(this._destroyed)).subscribe( response => { (ev.srcElement as HTMLButtonElement).disabled = false; @@ -54,16 +54,16 @@ export class IndexManagmentComponent extends BaseComponent implements OnInit { clearIndex(ev: Event) { (ev.srcElement as HTMLButtonElement).disabled = true; - this.datasetService.clearIndex().pipe(takeUntil(this._destroyed)).subscribe( - response => { - (ev.srcElement as HTMLButtonElement).disabled = false; - this.onCallbackSuccess(); - }, - error => { - (ev.srcElement as HTMLButtonElement).disabled = false; - this.onCallbackError(error); - } - ); + // this.datasetService.clearIndex().pipe(takeUntil(this._destroyed)).subscribe( + // response => { + // (ev.srcElement as HTMLButtonElement).disabled = false; + // this.onCallbackSuccess(); + // }, + // error => { + // (ev.srcElement as HTMLButtonElement).disabled = false; + // this.onCallbackError(error); + // } + // ); this.dmpService.clearIndex().pipe(takeUntil(this._destroyed)).subscribe( response => { (ev.srcElement as HTMLButtonElement).disabled = false;