Vastly improve and optimize DMP and Dataset Queries (and with elastic)
This commit is contained in:
parent
ce8b49deb0
commit
5de811a76d
|
@ -6,22 +6,22 @@ import java.util.List;
|
||||||
|
|
||||||
public class ColumnOrderings {
|
public class ColumnOrderings {
|
||||||
|
|
||||||
private String[] fields;
|
private List<String> fields;
|
||||||
|
|
||||||
public String[] getFields() {
|
public List<String> getFields() {
|
||||||
return fields;
|
return fields;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setFields(String[] fields) {
|
public void setFields(List<String> fields) {
|
||||||
this.fields = fields;
|
this.fields = fields;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Ordering[] getFieldOrderings() {
|
public List<Ordering> getFieldOrderings() {
|
||||||
List<Ordering> orderings = new LinkedList<>();
|
List<Ordering> orderings = new LinkedList<>();
|
||||||
for (String field : fields) {
|
for (String field : fields) {
|
||||||
orderings.add(this.orderingFromString(field));
|
orderings.add(this.orderingFromString(field));
|
||||||
}
|
}
|
||||||
return orderings.toArray(new Ordering[orderings.size()]);
|
return orderings;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Ordering orderingFromString(String field) {
|
private Ordering orderingFromString(String field) {
|
||||||
|
|
|
@ -18,6 +18,7 @@ public class DatasetCriteria extends Criteria {
|
||||||
private List<UUID> collaborators;
|
private List<UUID> collaborators;
|
||||||
private Boolean allowAllVersions;
|
private Boolean allowAllVersions;
|
||||||
private List<String> organiztions;
|
private List<String> organiztions;
|
||||||
|
private Boolean hasTags;
|
||||||
private List<Tag> tags;
|
private List<Tag> tags;
|
||||||
private boolean isPublic;
|
private boolean isPublic;
|
||||||
private Short grantStatus;
|
private Short grantStatus;
|
||||||
|
@ -144,4 +145,12 @@ public class DatasetCriteria extends Criteria {
|
||||||
public void setSortCriteria(List<SortCriteria> sortCriteria) {
|
public void setSortCriteria(List<SortCriteria> sortCriteria) {
|
||||||
this.sortCriteria = sortCriteria;
|
this.sortCriteria = sortCriteria;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Boolean getHasTags() {
|
||||||
|
return hasTags;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setHasTags(Boolean hasTags) {
|
||||||
|
this.hasTags = hasTags;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -283,6 +283,13 @@ public class Dataset implements ElasticEntity<Dataset> {
|
||||||
@Override
|
@Override
|
||||||
public Dataset fromElasticEntity(Map<String, Object> fields) {
|
public Dataset fromElasticEntity(Map<String, Object> fields) {
|
||||||
if (fields != null) {
|
if (fields != null) {
|
||||||
|
if (fields.size() == 1) {
|
||||||
|
if (fields.containsKey("id")) {
|
||||||
|
this.id = (String) fields.get("id");
|
||||||
|
} else if (fields.containsKey("tags")) {
|
||||||
|
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
}else if (fields.size() > 1) {
|
||||||
this.id = (String) fields.get("id");
|
this.id = (String) fields.get("id");
|
||||||
if (fields.get("tags") != null) {
|
if (fields.get("tags") != null) {
|
||||||
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
|
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
|
||||||
|
@ -316,6 +323,7 @@ public class Dataset implements ElasticEntity<Dataset> {
|
||||||
}
|
}
|
||||||
this.formData = (String) fields.get("formData");
|
this.formData = (String) fields.get("formData");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
|
||||||
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
|
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
|
||||||
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
|
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
|
||||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||||
|
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
|
||||||
import org.elasticsearch.search.sort.SortBuilder;
|
import org.elasticsearch.search.sort.SortBuilder;
|
||||||
import org.elasticsearch.search.sort.SortBuilders;
|
import org.elasticsearch.search.sort.SortBuilders;
|
||||||
import org.elasticsearch.search.sort.SortOrder;
|
import org.elasticsearch.search.sort.SortOrder;
|
||||||
|
@ -122,8 +123,128 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
|
||||||
|
|
||||||
searchSourceBuilder.size(count.intValue());
|
searchSourceBuilder.size(count.intValue());
|
||||||
|
|
||||||
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
|
|
||||||
List<SortBuilder> sortBuilders = new ArrayList<>();
|
List<SortBuilder> sortBuilders = new ArrayList<>();
|
||||||
|
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
|
||||||
|
|
||||||
|
|
||||||
|
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
|
||||||
|
criteria.getSortCriteria().forEach(sortCriteria -> {
|
||||||
|
switch(sortCriteria.getColumnType()) {
|
||||||
|
case COLUMN:
|
||||||
|
sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
|
||||||
|
break;
|
||||||
|
case JOIN_COLUMN:
|
||||||
|
List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
|
||||||
|
fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
|
||||||
|
sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.tags"}, null)));
|
||||||
|
searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).fetchSource("datasets.tags", null);
|
||||||
|
if (criteria.getSize() > 0) {
|
||||||
|
searchSourceBuilder.size(criteria.getSize());
|
||||||
|
}
|
||||||
|
sortBuilders.forEach(searchSourceBuilder::sort);
|
||||||
|
searchRequest.source(searchSourceBuilder);
|
||||||
|
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
|
||||||
|
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
|
||||||
|
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
|
||||||
|
.map(SearchHits::getHits).flatMap(Arrays::stream)
|
||||||
|
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Dataset> queryIds(DatasetCriteria criteria) throws IOException {
|
||||||
|
if (this.getClient() != null) {
|
||||||
|
SearchRequest searchRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
|
||||||
|
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||||
|
|
||||||
|
/*CountRequest countRequest = new CountRequest("dmps").routing("datasets").routing("id");
|
||||||
|
countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
|
||||||
|
CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT);
|
||||||
|
Long count = countResponse.getCount();*/
|
||||||
|
|
||||||
|
SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
|
||||||
|
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
|
||||||
|
FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
|
||||||
|
nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
|
||||||
|
SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
|
||||||
|
countSourceBuilder.aggregation(nestedAggregationBuilder);
|
||||||
|
countRequest.source(countSourceBuilder);
|
||||||
|
SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
|
||||||
|
Long count = ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
|
||||||
|
|
||||||
|
|
||||||
|
searchSourceBuilder.size(count.intValue());
|
||||||
|
|
||||||
|
List<SortBuilder> sortBuilders = new ArrayList<>();
|
||||||
|
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
|
||||||
|
|
||||||
|
|
||||||
|
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
|
||||||
|
criteria.getSortCriteria().forEach(sortCriteria -> {
|
||||||
|
switch(sortCriteria.getColumnType()) {
|
||||||
|
case COLUMN:
|
||||||
|
sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
|
||||||
|
break;
|
||||||
|
case JOIN_COLUMN:
|
||||||
|
List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
|
||||||
|
fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
|
||||||
|
sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.id"}, null)));
|
||||||
|
searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).size(criteria.getSize()).fetchSource("datasets.id", null);
|
||||||
|
sortBuilders.forEach(searchSourceBuilder::sort);
|
||||||
|
searchRequest.source(searchSourceBuilder);
|
||||||
|
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
|
||||||
|
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
|
||||||
|
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
|
||||||
|
.map(SearchHits::getHits).flatMap(Arrays::stream)
|
||||||
|
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Long count(DatasetCriteria criteria) throws IOException {
|
||||||
|
if (this.getClient() != null) {
|
||||||
|
//CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
|
||||||
|
|
||||||
|
SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
|
||||||
|
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
|
||||||
|
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
|
||||||
|
FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", boolQuery);
|
||||||
|
nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
|
||||||
|
SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
|
||||||
|
countSourceBuilder.aggregation(nestedAggregationBuilder);
|
||||||
|
countRequest.source(countSourceBuilder);
|
||||||
|
SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
|
||||||
|
return ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder());
|
||||||
|
countRequest.query(nestedQueryBuilder);
|
||||||
|
CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
|
||||||
|
return response.getCount();*/
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private BoolQueryBuilder createBoolQuery(DatasetCriteria criteria) {
|
||||||
|
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
|
||||||
if (criteria.isPublic()) {
|
if (criteria.isPublic()) {
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
|
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
|
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
|
||||||
|
@ -162,7 +283,7 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
|
||||||
}
|
}
|
||||||
|
|
||||||
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
|
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
|
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!criteria.isPublic()) {
|
if (!criteria.isPublic()) {
|
||||||
|
@ -179,40 +300,19 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
|
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (criteria.getHasTags() != null) {
|
||||||
|
boolQuery = criteria.getHasTags() == true ? boolQuery.should(QueryBuilders.existsQuery("datasets.tags.id")) : boolQuery.mustNot(QueryBuilders.existsQuery("datasets.tags.id"));
|
||||||
|
}
|
||||||
|
|
||||||
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
|
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
|
||||||
boolQuery.should(QueryBuilders.matchAllQuery());
|
boolQuery.should(QueryBuilders.matchAllQuery());
|
||||||
} else {
|
} else {
|
||||||
boolQuery.minimumShouldMatch(boolQuery.should().size());
|
boolQuery.minimumShouldMatch(boolQuery.should().size());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
|
|
||||||
criteria.getSortCriteria().forEach(sortCriteria -> {
|
|
||||||
switch(sortCriteria.getColumnType()) {
|
|
||||||
case COLUMN:
|
|
||||||
sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
|
|
||||||
break;
|
|
||||||
case JOIN_COLUMN:
|
|
||||||
List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
|
|
||||||
fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
|
|
||||||
sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
|
|
||||||
});
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
|
return boolQuery;
|
||||||
searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).size(criteria.getSize());
|
|
||||||
sortBuilders.forEach(searchSourceBuilder::sort);
|
|
||||||
searchRequest.source(searchSourceBuilder);
|
|
||||||
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
|
|
||||||
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
|
|
||||||
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
|
|
||||||
.map(SearchHits::getHits).flatMap(Arrays::stream)
|
|
||||||
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -99,60 +99,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
|
||||||
|
|
||||||
searchSourceBuilder.size(count.intValue());
|
searchSourceBuilder.size(count.intValue());
|
||||||
|
|
||||||
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
|
|
||||||
List<SortBuilder> sortBuilders = new ArrayList<>();
|
List<SortBuilder> sortBuilders = new ArrayList<>();
|
||||||
if (criteria.isPublic()) {
|
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
|
|
||||||
}
|
|
||||||
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
|
|
||||||
{Dmp.MapKey.LABEL.getName(), 1.0f},
|
|
||||||
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
|
|
||||||
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getStatus() != null) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getRoles() != null && criteria.getRoles().size() > 0) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".role.keyword", criteria.getRoles()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!criteria.isAllowAllVersions()) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getGrantStatus() != null) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
|
|
||||||
boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
|
|
||||||
} else {
|
|
||||||
boolQuery.minimumShouldMatch(boolQuery.should().size());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
|
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
|
||||||
criteria.getSortCriteria().forEach(sortCriteria -> {
|
criteria.getSortCriteria().forEach(sortCriteria -> {
|
||||||
|
@ -170,8 +118,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
|
||||||
});
|
});
|
||||||
|
|
||||||
}
|
}
|
||||||
searchSourceBuilder.query(boolQuery).from(criteria.getOffset());
|
searchSourceBuilder.query(boolQuery).from(criteria.getOffset()).fetchSource("id", null);
|
||||||
if (criteria.getSize() != null) {
|
if (criteria.getSize() > 0) {
|
||||||
searchSourceBuilder.size(criteria.getSize());
|
searchSourceBuilder.size(criteria.getSize());
|
||||||
}
|
}
|
||||||
sortBuilders.forEach(searchSourceBuilder::sort);
|
sortBuilders.forEach(searchSourceBuilder::sort);
|
||||||
|
@ -182,10 +130,21 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public Long count(DmpCriteria criteria) throws IOException {
|
public Long count(DmpCriteria criteria) throws IOException {
|
||||||
if (this.getClient() != null) {
|
if (this.getClient() != null) {
|
||||||
CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
|
CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
|
||||||
|
|
||||||
|
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
|
||||||
|
|
||||||
|
countRequest.query(boolQuery);
|
||||||
|
CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
|
||||||
|
return response.getCount();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private BoolQueryBuilder createBoolQuery(DmpCriteria criteria) {
|
||||||
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
|
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
|
||||||
if (criteria.isPublic()) {
|
if (criteria.isPublic()) {
|
||||||
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
|
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
|
||||||
|
@ -235,12 +194,7 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
|
||||||
} else {
|
} else {
|
||||||
boolQuery.minimumShouldMatch(boolQuery.should().size());
|
boolQuery.minimumShouldMatch(boolQuery.should().size());
|
||||||
}
|
}
|
||||||
|
return boolQuery;
|
||||||
countRequest.query(boolQuery);
|
|
||||||
CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
|
|
||||||
return response.getCount();
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean createIndex() {
|
public boolean createIndex() {
|
||||||
|
|
|
@ -18,6 +18,8 @@ public interface Repository<ET extends ElasticEntity, C extends Criteria> {
|
||||||
|
|
||||||
List<ET> query(C criteria) throws ExecutionException, InterruptedException, IOException;
|
List<ET> query(C criteria) throws ExecutionException, InterruptedException, IOException;
|
||||||
|
|
||||||
|
Long count(C criteria) throws ExecutionException, InterruptedException, IOException;
|
||||||
|
|
||||||
boolean exists() throws IOException;
|
boolean exists() throws IOException;
|
||||||
|
|
||||||
void clear() throws IOException;
|
void clear() throws IOException;
|
||||||
|
|
|
@ -1,10 +1,8 @@
|
||||||
package eu.eudat.controllers;
|
package eu.eudat.controllers;
|
||||||
|
|
||||||
import eu.eudat.elastic.criteria.DatasetCriteria;
|
import eu.eudat.elastic.criteria.DatasetCriteria;
|
||||||
import eu.eudat.elastic.criteria.TagCriteria;
|
|
||||||
import eu.eudat.elastic.entities.Dataset;
|
import eu.eudat.elastic.entities.Dataset;
|
||||||
import eu.eudat.elastic.entities.Tag;
|
import eu.eudat.elastic.entities.Tag;
|
||||||
import eu.eudat.elastic.repository.Repository;
|
|
||||||
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
|
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
|
||||||
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
|
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
|
||||||
import eu.eudat.logic.services.ApiContext;
|
import eu.eudat.logic.services.ApiContext;
|
||||||
|
@ -49,7 +47,9 @@ public class TagController extends BaseController {
|
||||||
/*List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type);
|
/*List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type);
|
||||||
TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);*/
|
TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);*/
|
||||||
if (this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().exists()) {
|
if (this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().exists()) {
|
||||||
List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
|
DatasetCriteria criteria = new DatasetCriteria();
|
||||||
|
criteria.setHasTags(true);
|
||||||
|
List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tag.getName().startsWith(query)).collect(Collectors.toList());
|
||||||
|
|
||||||
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<Tag>>().payload(tags).status(ApiMessageCode.NO_MESSAGE));
|
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<Tag>>().payload(tags).status(ApiMessageCode.NO_MESSAGE));
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -8,13 +8,12 @@ import eu.eudat.data.dao.entities.DMPDao;
|
||||||
import eu.eudat.data.dao.entities.DatasetDao;
|
import eu.eudat.data.dao.entities.DatasetDao;
|
||||||
import eu.eudat.data.dao.entities.GrantDao;
|
import eu.eudat.data.dao.entities.GrantDao;
|
||||||
import eu.eudat.data.dao.entities.OrganisationDao;
|
import eu.eudat.data.dao.entities.OrganisationDao;
|
||||||
import eu.eudat.data.entities.DMP;
|
import eu.eudat.data.entities.*;
|
||||||
import eu.eudat.data.entities.Dataset;
|
|
||||||
import eu.eudat.data.entities.Grant;
|
|
||||||
import eu.eudat.data.entities.UserInfo;
|
|
||||||
import eu.eudat.data.query.PaginationService;
|
import eu.eudat.data.query.PaginationService;
|
||||||
|
import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
|
||||||
import eu.eudat.elastic.entities.Dmp;
|
import eu.eudat.elastic.entities.Dmp;
|
||||||
import eu.eudat.logic.builders.model.models.RecentActivityDataBuilder;
|
import eu.eudat.logic.builders.model.models.RecentActivityDataBuilder;
|
||||||
|
import eu.eudat.logic.mapper.elastic.criteria.DmpCriteriaMapper;
|
||||||
import eu.eudat.logic.services.ApiContext;
|
import eu.eudat.logic.services.ApiContext;
|
||||||
import eu.eudat.logic.services.operations.DatabaseRepository;
|
import eu.eudat.logic.services.operations.DatabaseRepository;
|
||||||
import eu.eudat.models.HintedModelFactory;
|
import eu.eudat.models.HintedModelFactory;
|
||||||
|
@ -26,6 +25,8 @@ import eu.eudat.models.data.dashboard.recent.model.RecentDmpModel;
|
||||||
import eu.eudat.models.data.dashboard.recent.tablerequest.RecentActivityTableRequest;
|
import eu.eudat.models.data.dashboard.recent.tablerequest.RecentActivityTableRequest;
|
||||||
import eu.eudat.models.data.dashboard.searchbar.SearchBarItem;
|
import eu.eudat.models.data.dashboard.searchbar.SearchBarItem;
|
||||||
import eu.eudat.models.data.dashboard.statistics.DashBoardStatistics;
|
import eu.eudat.models.data.dashboard.statistics.DashBoardStatistics;
|
||||||
|
import eu.eudat.models.data.listingmodels.DataManagementPlanListingModel;
|
||||||
|
import eu.eudat.models.data.listingmodels.DatasetListingModel;
|
||||||
import eu.eudat.models.data.security.Principal;
|
import eu.eudat.models.data.security.Principal;
|
||||||
import eu.eudat.queryable.QueryableList;
|
import eu.eudat.queryable.QueryableList;
|
||||||
import eu.eudat.types.searchbar.SearchBarItemType;
|
import eu.eudat.types.searchbar.SearchBarItemType;
|
||||||
|
@ -97,8 +98,8 @@ public class DashBoardManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
public DashBoardStatistics getMeStatistics(Principal principal) throws IOException {
|
public DashBoardStatistics getMeStatistics(Principal principal) throws IOException {
|
||||||
List<eu.eudat.elastic.entities.Dataset> datasets = null;
|
Long datasets = 0L;
|
||||||
List<eu.eudat.elastic.entities.Dmp> dmps = null;
|
Long dmps = 0L;
|
||||||
DashBoardStatistics statistics = new DashBoardStatistics();
|
DashBoardStatistics statistics = new DashBoardStatistics();
|
||||||
DMPDao dataManagementPlanRepository = databaseRepository.getDmpDao();
|
DMPDao dataManagementPlanRepository = databaseRepository.getDmpDao();
|
||||||
DatasetDao datasetRepository = databaseRepository.getDatasetDao();
|
DatasetDao datasetRepository = databaseRepository.getDatasetDao();
|
||||||
|
@ -112,7 +113,8 @@ public class DashBoardManager {
|
||||||
eu.eudat.elastic.criteria.DatasetCriteria datasetElasticCriteria = new eu.eudat.elastic.criteria.DatasetCriteria();
|
eu.eudat.elastic.criteria.DatasetCriteria datasetElasticCriteria = new eu.eudat.elastic.criteria.DatasetCriteria();
|
||||||
datasetElasticCriteria.setAllowAllVersions(false);
|
datasetElasticCriteria.setAllowAllVersions(false);
|
||||||
datasetElasticCriteria.setPublic(false);
|
datasetElasticCriteria.setPublic(false);
|
||||||
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
|
datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
|
||||||
|
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().count(datasetElasticCriteria);
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
logger.warn(e.getMessage(), e);
|
logger.warn(e.getMessage(), e);
|
||||||
datasets = null;
|
datasets = null;
|
||||||
|
@ -126,33 +128,40 @@ public class DashBoardManager {
|
||||||
eu.eudat.elastic.criteria.DmpCriteria dmpElasticCriteria = new eu.eudat.elastic.criteria.DmpCriteria();
|
eu.eudat.elastic.criteria.DmpCriteria dmpElasticCriteria = new eu.eudat.elastic.criteria.DmpCriteria();
|
||||||
dmpElasticCriteria.setAllowAllVersions(false);
|
dmpElasticCriteria.setAllowAllVersions(false);
|
||||||
dmpElasticCriteria.setPublic(false);
|
dmpElasticCriteria.setPublic(false);
|
||||||
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
|
dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
|
||||||
|
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().count(dmpElasticCriteria);
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
logger.warn(e.getMessage(), e);
|
logger.warn(e.getMessage(), e);
|
||||||
dmps = null;
|
dmps = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
dataManagementPlanCriteria.setAllVersions(false);
|
dataManagementPlanCriteria.setAllVersions(false);
|
||||||
GrantCriteria grantCriteria = new GrantCriteria();
|
dataManagementPlanCriteria.setOnlyPublic(false);
|
||||||
|
dataManagementPlanCriteria.setIsPublic(false);
|
||||||
|
|
||||||
|
GrantCriteria grantCriteria = new GrantCriteria();
|
||||||
grantCriteria.setActive(true);
|
grantCriteria.setActive(true);
|
||||||
|
|
||||||
OrganisationCriteria organisationCriteria = new OrganisationCriteria();
|
OrganisationCriteria organisationCriteria = new OrganisationCriteria();
|
||||||
organisationCriteria.setActive(true);
|
organisationCriteria.setActive(true);
|
||||||
|
|
||||||
List<Integer> roles = new LinkedList<>();
|
List<Integer> roles = new LinkedList<>();
|
||||||
List<Dmp> finalDmps = dmps;
|
if ((dmps == null || dmps == 0L) && (datasets == null || datasets == 0L)) {
|
||||||
CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated((dmps != null && !dmps.isEmpty()) ? dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))) : dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
|
CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated(dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
|
||||||
.whenComplete((dmpsStats, throwable) -> statistics.setTotalDataManagementPlanCount(dmpsStats));
|
.whenComplete((dmpsStats, throwable) -> statistics.setTotalDataManagementPlanCount(dmpsStats));
|
||||||
List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
|
CompletableFuture datasetFuture = datasetRepository.getAuthenticated( datasetRepository.getWithCriteria(datasetCriteria), user, roles).countAsync()
|
||||||
CompletableFuture datasetFuture = datasetRepository.getAuthenticated((datasets != null && !datasets.isEmpty()) ? datasetRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))) : datasetRepository.getWithCriteria(datasetCriteria), user, roles).countAsync()
|
|
||||||
.whenComplete((datasetsStats, throwable) -> statistics.setTotalDataSetCount(datasetsStats));
|
.whenComplete((datasetsStats, throwable) -> statistics.setTotalDataSetCount(datasetsStats));
|
||||||
|
CompletableFuture.allOf(dmpFuture, datasetFuture).join();
|
||||||
|
} else {
|
||||||
|
statistics.setTotalDataManagementPlanCount(dmps);
|
||||||
|
statistics.setTotalDataSetCount(datasets);
|
||||||
|
}
|
||||||
CompletableFuture grantFuture = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user).countAsync()
|
CompletableFuture grantFuture = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user).countAsync()
|
||||||
.whenComplete((grantsStats, throwable) -> statistics.setTotalGrantCount(grantsStats));
|
.whenComplete((grantsStats, throwable) -> statistics.setTotalGrantCount(grantsStats));
|
||||||
CompletableFuture orgnanisationFuture = organisationRepository.getAuthenticated(organisationRepository.getWithCriteria(organisationCriteria).withHint("organisationRecentActivity"), user).countAsync()
|
CompletableFuture orgnanisationFuture = organisationRepository.getAuthenticated(organisationRepository.getWithCriteria(organisationCriteria).withHint("organisationRecentActivity"), user).countAsync()
|
||||||
.whenComplete((organisationStats, throwable) -> statistics.setTotalOrganisationCount(organisationStats));
|
.whenComplete((organisationStats, throwable) -> statistics.setTotalOrganisationCount(organisationStats));
|
||||||
|
|
||||||
CompletableFuture.allOf(dmpFuture, datasetFuture, grantFuture, orgnanisationFuture).join();
|
CompletableFuture.allOf( grantFuture, orgnanisationFuture).join();
|
||||||
return statistics;
|
return statistics;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -185,14 +194,14 @@ public class DashBoardManager {
|
||||||
.selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
|
.selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
|
||||||
.whenComplete((datasetActivities, throwable) -> activity.setRecentDatasetActivities(datasetActivities));
|
.whenComplete((datasetActivities, throwable) -> activity.setRecentDatasetActivities(datasetActivities));
|
||||||
|
|
||||||
/*CompletableFuture<List<RecentActivityData>> grants = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user)
|
CompletableFuture<List<RecentActivityData>> grants = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user)
|
||||||
.withHint("grantRecentActivity")
|
.withHint("grantRecentActivity")
|
||||||
.orderBy((builder, root) -> builder.desc(root.get("modified")))
|
.orderBy((builder, root) -> builder.desc(root.get("modified")))
|
||||||
.take(numberofactivities)
|
.take(numberofactivities)
|
||||||
.selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
|
.selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
|
||||||
.whenComplete((grantActivities, throwable) -> activity.setRecentGrantActivities(grantActivities));*/
|
.whenComplete((grantActivities, throwable) -> activity.setRecentGrantActivities(grantActivities));
|
||||||
|
|
||||||
CompletableFuture.allOf(/*grants, */dmps, datasets).join();
|
CompletableFuture.allOf(grants, dmps, datasets).join();
|
||||||
return activity;
|
return activity;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -228,7 +237,13 @@ public class DashBoardManager {
|
||||||
datasetElasticCriteria.setLike(tableRequest.getCriteria().getLike());
|
datasetElasticCriteria.setLike(tableRequest.getCriteria().getLike());
|
||||||
datasetElasticCriteria.setAllowAllVersions(false);
|
datasetElasticCriteria.setAllowAllVersions(false);
|
||||||
datasetElasticCriteria.setPublic(!isAuthenticated);
|
datasetElasticCriteria.setPublic(!isAuthenticated);
|
||||||
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
|
datasetElasticCriteria.setOffset(tableRequest.getDatasetOffset());
|
||||||
|
datasetElasticCriteria.setSize(tableRequest.getLength());
|
||||||
|
if (isAuthenticated) {
|
||||||
|
datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
|
||||||
|
}
|
||||||
|
datasetElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
|
||||||
|
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().queryIds(datasetElasticCriteria);
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
logger.warn(e.getMessage(), e);
|
logger.warn(e.getMessage(), e);
|
||||||
datasets = null;
|
datasets = null;
|
||||||
|
@ -241,10 +256,16 @@ public class DashBoardManager {
|
||||||
dmpElasticCriteria.setLike(tableRequest.getCriteria().getLike());
|
dmpElasticCriteria.setLike(tableRequest.getCriteria().getLike());
|
||||||
dmpElasticCriteria.setAllowAllVersions(false);
|
dmpElasticCriteria.setAllowAllVersions(false);
|
||||||
dmpElasticCriteria.setPublic(!isAuthenticated);
|
dmpElasticCriteria.setPublic(!isAuthenticated);
|
||||||
|
dmpElasticCriteria.setOffset(tableRequest.getDmpOffset());
|
||||||
|
dmpElasticCriteria.setSize(tableRequest.getLength());
|
||||||
|
if (isAuthenticated) {
|
||||||
|
dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
|
||||||
|
}
|
||||||
|
dmpElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
|
||||||
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
|
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
|
||||||
}catch (Exception e) {
|
}catch (Exception e) {
|
||||||
logger.warn(e.getMessage(), e);
|
logger.warn(e.getMessage(), e);
|
||||||
datasets = null;
|
dmps = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -253,6 +274,8 @@ public class DashBoardManager {
|
||||||
dmpList = dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))).distinct();
|
dmpList = dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))).distinct();
|
||||||
} else {
|
} else {
|
||||||
dmpList = dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria).distinct();
|
dmpList = dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria).distinct();
|
||||||
|
PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
|
||||||
|
dmpList.skip(tableRequest.getDmpOffset()).take(tableRequest.getLength());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (datasets != null && !datasets.isEmpty()) {
|
if (datasets != null && !datasets.isEmpty()) {
|
||||||
|
@ -260,50 +283,50 @@ public class DashBoardManager {
|
||||||
datasetList = datasetRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
|
datasetList = datasetRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
|
||||||
} else {
|
} else {
|
||||||
datasetList = datasetRepository.getWithCriteria(datasetCriteria);
|
datasetList = datasetRepository.getWithCriteria(datasetCriteria);
|
||||||
|
for (int i = 0; i< tableRequest.getOrderings().getFields().size(); i++) {
|
||||||
|
if (tableRequest.getOrderings().getFields().get(i).contains("publishedAt")) {
|
||||||
|
String newField = tableRequest.getOrderings().getFields().get(i).toCharArray()[0] + "dmp:publishedAt|join|";
|
||||||
|
tableRequest.getOrderings().getFields().set(i, newField);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
if (tableRequest.getOrderings().getFields().get(0).contains("publishedAt")) {
|
||||||
|
tableRequest.getOrderings().getFields().set(0, tableRequest.getOrderings().getFields().get(0).charAt(0) + "dmp:" + tableRequest.getOrderings().getFields().get(0).substring(1) + "|join|");
|
||||||
|
}
|
||||||
|
PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
|
||||||
|
datasetList.skip(tableRequest.getDatasetOffset()).take(tableRequest.getLength());
|
||||||
|
|
||||||
if (isAuthenticated) {
|
if (isAuthenticated) {
|
||||||
|
|
||||||
List<Integer> roles = new LinkedList<>();
|
List<Integer> roles = new LinkedList<>();
|
||||||
|
roles.add(UserDMP.UserDMPRoles.USER.getValue());
|
||||||
|
roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
|
||||||
dmpList = dataManagementPlanRepository.getAuthenticated(dmpList, principal.getId(), roles);
|
dmpList = dataManagementPlanRepository.getAuthenticated(dmpList, principal.getId(), roles);
|
||||||
datasetList = datasetRepository.getAuthenticated(datasetList, user, roles);
|
datasetList = datasetRepository.getAuthenticated(datasetList, user, roles);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
|
|
||||||
for (int i = 0; i< tableRequest.getOrderings().getFields().length; i++) {
|
|
||||||
if (tableRequest.getOrderings().getFields()[i].contains("publishedAt")) {
|
|
||||||
String newField = tableRequest.getOrderings().getFields()[i].toCharArray()[0] + "dmp:publishedAt|join|";
|
|
||||||
tableRequest.getOrderings().getFields()[i] = newField;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
|
|
||||||
|
|
||||||
/*CompletableFuture future = CompletableFuture.runAsync(() -> */{
|
/*CompletableFuture future = CompletableFuture.runAsync(() -> */{
|
||||||
recentActivityModels.addAll(dmpList
|
List<DMP> dmps1 = dmpList.withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))
|
||||||
.withHint(HintedModelFactory.getHint(RecentDmpModel.class))
|
.distinct().toList();
|
||||||
// .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
|
recentActivityModels.addAll(dmps1.stream().map(dmp -> {
|
||||||
.skip(tableRequest.getDmpOffset())
|
DatasetCriteria datasetCriteria1 = new DatasetCriteria();
|
||||||
.take(tableRequest.getLength())
|
datasetCriteria1.setDmpIds(Collections.singletonList(dmp.getId()));
|
||||||
.select(item -> {
|
if (isAuthenticated) {
|
||||||
item.setDataset(
|
dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1, principal.getId()));
|
||||||
item.getDataset().stream()
|
} else {
|
||||||
.filter(dataset -> !dataset.getStatus().equals(Dataset.Status.DELETED.getValue()) && !dataset.getStatus().equals(Dataset.Status.CANCELED.getValue())).collect(Collectors.toList()).stream()
|
dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1));
|
||||||
.filter(dataset -> dataset.getDmp().getUsers().stream()
|
}
|
||||||
.filter(x -> x.getUser().getId().equals(principal.getId()))
|
return new RecentDmpModel().fromDataModel(dmp);
|
||||||
.collect(Collectors.toList()).size() > 0)
|
}).collect(Collectors.toList()));
|
||||||
.collect(Collectors.toSet()));
|
|
||||||
return new RecentDmpModel().fromEntity(item);
|
|
||||||
}));
|
|
||||||
|
|
||||||
recentActivityModels.addAll(datasetList
|
List<RecentActivityModel> recentDatasetModels = datasetList
|
||||||
.withHint(HintedModelFactory.getHint(RecentDatasetModel.class))
|
.withHint(HintedModelFactory.getHint(DatasetListingModel.class))
|
||||||
// .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
|
.select(item -> new RecentDatasetModel().fromEntity(item));
|
||||||
.skip(tableRequest.getDatasetOffset())
|
recentActivityModels.addAll(recentDatasetModels);
|
||||||
.take(tableRequest.getLength())
|
|
||||||
.select(item -> {
|
|
||||||
return new RecentDatasetModel().fromEntity(item);
|
|
||||||
}));
|
|
||||||
}/*);*/
|
}/*);*/
|
||||||
|
|
||||||
//GK: Shuffle the deck otherwise we will summon the DMPodia when sorting with status
|
//GK: Shuffle the deck otherwise we will summon the DMPodia when sorting with status
|
||||||
|
@ -322,7 +345,7 @@ public class DashBoardManager {
|
||||||
//CompletableFuture.allOf(future).join();
|
//CompletableFuture.allOf(future).join();
|
||||||
// CompletableFuture.allOf(dmps, datasets).join();
|
// CompletableFuture.allOf(dmps, datasets).join();
|
||||||
|
|
||||||
return recentActivityModels.stream().sorted(this.comparators.get(tableRequest.getCriteria().getOrder())).limit(tableRequest.getLength()).collect(Collectors.toList());
|
return recentActivityModels;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<SearchBarItem> searchUserData(String like, Principal principal) {
|
public List<SearchBarItem> searchUserData(String like, Principal principal) {
|
||||||
|
@ -383,4 +406,38 @@ public class DashBoardManager {
|
||||||
|
|
||||||
return searchBarItems;
|
return searchBarItems;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Set<Dataset> retrieveRelevantDatasets(DatasetCriteria datasetCriteria) {
|
||||||
|
return retrieveRelevantDatasets(datasetCriteria, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Set<Dataset> retrieveRelevantDatasets (DatasetCriteria datasetCriteria, UUID principal) {
|
||||||
|
QueryableList<Dataset> datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria);
|
||||||
|
if (principal != null) {
|
||||||
|
UserInfo userInfo = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal);
|
||||||
|
List<Integer> roles = new ArrayList<>();
|
||||||
|
roles.add(0);
|
||||||
|
roles.add(1);
|
||||||
|
datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getAuthenticated(datasetItems, userInfo, roles);
|
||||||
|
}
|
||||||
|
Long maxDatasets = datasetItems.count();
|
||||||
|
DatasetTableRequest datasetTableRequest = new DatasetTableRequest();
|
||||||
|
datasetTableRequest.setOffset(0);
|
||||||
|
datasetTableRequest.setLength(3);
|
||||||
|
Set<Dataset> datasetsSet = new LinkedHashSet<>();
|
||||||
|
try {
|
||||||
|
datasetItems = PaginationManager.applyPaging(datasetItems, datasetTableRequest);
|
||||||
|
List<Dataset> datasets = datasetItems.toList();
|
||||||
|
datasetsSet.addAll(datasets);
|
||||||
|
for (int i = 0; i < maxDatasets - datasets.size(); i++) {
|
||||||
|
Dataset fakedataset = new Dataset();
|
||||||
|
fakedataset.setId(UUID.randomUUID());
|
||||||
|
datasetsSet.add(fakedataset);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
|
||||||
|
return datasetsSet;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -88,7 +88,6 @@ import java.nio.file.Files;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.time.temporal.ChronoUnit;
|
import java.time.temporal.ChronoUnit;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.CompletableFuture;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
import java.util.zip.ZipEntry;
|
import java.util.zip.ZipEntry;
|
||||||
|
@ -157,7 +156,7 @@ public class DatasetManager {
|
||||||
List<eu.eudat.elastic.entities.Dataset> datasets;
|
List<eu.eudat.elastic.entities.Dataset> datasets;
|
||||||
try {
|
try {
|
||||||
datasets = datasetRepository.exists() ?
|
datasets = datasetRepository.exists() ?
|
||||||
datasetRepository.query(datasetCriteria) : null;
|
datasetRepository.queryIds(datasetCriteria) : null;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
logger.warn(ex.getMessage(), ex);
|
logger.warn(ex.getMessage(), ex);
|
||||||
datasets = null;
|
datasets = null;
|
||||||
|
@ -186,8 +185,12 @@ public class DatasetManager {
|
||||||
if (principal.getId() == null) {
|
if (principal.getId() == null) {
|
||||||
throw new UnauthorisedException("You are not allowed to access those datasets");
|
throw new UnauthorisedException("You are not allowed to access those datasets");
|
||||||
}
|
}
|
||||||
if (datasetTableRequest.getCriteria().getRole() != null)
|
if (datasetTableRequest.getCriteria().getRole() != null) {
|
||||||
roles.add(datasetTableRequest.getCriteria().getRole());
|
roles.add(datasetTableRequest.getCriteria().getRole());
|
||||||
|
} else {
|
||||||
|
roles.add(UserDMP.UserDMPRoles.USER.getValue());
|
||||||
|
roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
|
||||||
|
}
|
||||||
authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles);
|
authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles);
|
||||||
pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
|
pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
|
||||||
} else {
|
} else {
|
||||||
|
@ -215,6 +218,7 @@ public class DatasetManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
public DataTableData<DatasetListingModel> getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception {
|
public DataTableData<DatasetListingModel> getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception {
|
||||||
|
Long count = 0L;
|
||||||
DatasetCriteria datasetCriteria = new DatasetCriteria();
|
DatasetCriteria datasetCriteria = new DatasetCriteria();
|
||||||
datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
|
datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
|
||||||
datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile());
|
datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile());
|
||||||
|
@ -228,7 +232,8 @@ public class DatasetManager {
|
||||||
List<eu.eudat.elastic.entities.Dataset> datasets;
|
List<eu.eudat.elastic.entities.Dataset> datasets;
|
||||||
try {
|
try {
|
||||||
datasets = datasetRepository.exists() ?
|
datasets = datasetRepository.exists() ?
|
||||||
datasetRepository.query(datasetCriteria) : new LinkedList<>();
|
datasetRepository.queryIds(datasetCriteria) : new LinkedList<>();
|
||||||
|
count = datasetRepository.exists() ? datasetRepository.count(datasetCriteria) : 0L;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
logger.warn(ex.getMessage());
|
logger.warn(ex.getMessage());
|
||||||
datasets = null;
|
datasets = null;
|
||||||
|
@ -254,22 +259,22 @@ public class DatasetManager {
|
||||||
return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
|
return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
String[] strings = new String[1];
|
List<String> strings = new ArrayList<>();
|
||||||
strings[0] = "-dmp:publishedAt|join|";
|
strings.add("-dmp:publishedAt|join|");
|
||||||
datasetTableRequest.getOrderings().setFields(strings);
|
datasetTableRequest.getOrderings().setFields(strings);
|
||||||
|
if (count == 0L) {
|
||||||
|
count = items.count();
|
||||||
|
}
|
||||||
QueryableList<eu.eudat.data.entities.Dataset> pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
|
QueryableList<eu.eudat.data.entities.Dataset> pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
|
||||||
DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
|
DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
|
||||||
|
|
||||||
CompletableFuture<List<DatasetListingModel>> itemsFuture = pagedItems.
|
List<DatasetListingModel> datasetListis = pagedItems.
|
||||||
selectAsync(this::mapModel).whenComplete((resultList, throwable) -> {
|
select(this::mapModel);
|
||||||
dataTable.setData(resultList.stream().filter(Objects::nonNull).collect(Collectors.toList()));
|
|
||||||
});
|
dataTable.setData(datasetListis.stream().filter(Objects::nonNull).collect(Collectors.toList()));
|
||||||
|
|
||||||
CompletableFuture countFuture = pagedItems.countAsync().whenComplete((count, throwable) -> {
|
|
||||||
dataTable.setTotalCount(count);
|
dataTable.setTotalCount(count);
|
||||||
});
|
//CompletableFuture.allOf(countFuture).join();
|
||||||
|
|
||||||
CompletableFuture.allOf(itemsFuture, countFuture).join();
|
|
||||||
return dataTable;
|
return dataTable;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1079,7 +1084,9 @@ public class DatasetManager {
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition());
|
String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition());
|
||||||
JsonNode propertiesJson = mapper.readTree(json);
|
JsonNode propertiesJson = mapper.readTree(json);
|
||||||
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
|
DatasetCriteria criteria = new DatasetCriteria();
|
||||||
|
criteria.setHasTags(true);
|
||||||
|
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
|
||||||
Set<JsonNode> tagNodes = new HashSet<>();
|
Set<JsonNode> tagNodes = new HashSet<>();
|
||||||
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true));
|
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true));
|
||||||
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "rdaProperty", "dataset.keyword"));
|
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "rdaProperty", "dataset.keyword"));
|
||||||
|
|
|
@ -31,6 +31,7 @@ public class DatasetMapper {
|
||||||
if (tags != null && !tags.isEmpty()) {
|
if (tags != null && !tags.isEmpty()) {
|
||||||
DatasetCriteria criteria = new DatasetCriteria();
|
DatasetCriteria criteria = new DatasetCriteria();
|
||||||
criteria.setTags(tags);
|
criteria.setTags(tags);
|
||||||
|
criteria.setHasTags(true);
|
||||||
List<Tag> tags1 = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
|
List<Tag> tags1 = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
|
||||||
.filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tags.stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
|
.filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tags.stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
|
||||||
if (tags1.isEmpty()) {
|
if (tags1.isEmpty()) {
|
||||||
|
|
|
@ -45,7 +45,7 @@ public class DmpCriteriaMapper {
|
||||||
|
|
||||||
public static List<SortCriteria> toElasticSorting(ColumnOrderings columnOrderings) {
|
public static List<SortCriteria> toElasticSorting(ColumnOrderings columnOrderings) {
|
||||||
List<SortCriteria> sortCriteria = new ArrayList<>();
|
List<SortCriteria> sortCriteria = new ArrayList<>();
|
||||||
if (columnOrderings.getFieldOrderings() != null && columnOrderings.getFieldOrderings().length > 0) {
|
if (columnOrderings.getFieldOrderings() != null && !columnOrderings.getFieldOrderings().isEmpty()) {
|
||||||
for (Ordering ordering: columnOrderings.getFieldOrderings()) {
|
for (Ordering ordering: columnOrderings.getFieldOrderings()) {
|
||||||
SortCriteria sortCriteria1 = new SortCriteria();
|
SortCriteria sortCriteria1 = new SortCriteria();
|
||||||
sortCriteria1.setFieldName(ordering.getFieldName() + (ordering.getFieldName().contains("label") ?".keyword" : ""));
|
sortCriteria1.setFieldName(ordering.getFieldName() + (ordering.getFieldName().contains("label") ?".keyword" : ""));
|
||||||
|
|
|
@ -9,7 +9,7 @@ import java.util.Map;
|
||||||
* Created by ikalyvas on 3/5/2018.
|
* Created by ikalyvas on 3/5/2018.
|
||||||
*/
|
*/
|
||||||
public interface VisibilityRuleService {
|
public interface VisibilityRuleService {
|
||||||
Boolean isElementVisible(String id);
|
boolean isElementVisible(String id);
|
||||||
|
|
||||||
void buildVisibilityContext(List<Rule> sources);
|
void buildVisibilityContext(List<Rule> sources);
|
||||||
|
|
||||||
|
|
|
@ -16,7 +16,7 @@ public class VisibilityRuleServiceImpl implements VisibilityRuleService {
|
||||||
private VisibilityContext visibilityContext;
|
private VisibilityContext visibilityContext;
|
||||||
private Map<String, Object> properties;
|
private Map<String, Object> properties;
|
||||||
|
|
||||||
public Boolean isElementVisible(String id) {
|
public boolean isElementVisible(String id) {
|
||||||
if (!this.elementVisibility.containsKey(id) || this.elementVisibility.get(id)) return true;
|
if (!this.elementVisibility.containsKey(id) || this.elementVisibility.get(id)) return true;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue