MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Vastly improve and optimize DMP and Dataset Queries (and with elastic)

This commit is contained in:
George Kalampokis 2021-08-04 18:38:49 +03:00
parent ce8b49deb0
commit 5de811a76d
13 changed files with 414 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dmp-backend/data/src/main/java/eu/eudat/data/query/definition/helpers/ColumnOrderings.java
View File

@ -6,22 +6,22 @@ import java.util.List;
public class ColumnOrderings {
private String[] fields;
private List<String> fields;
public String[] getFields() {
public List<String> getFields() {
return fields;
}
public void setFields(String[] fields) {
public void setFields(List<String> fields) {
this.fields = fields;
}
public Ordering[] getFieldOrderings() {
public List<Ordering> getFieldOrderings() {
List<Ordering> orderings = new LinkedList<>();
for (String field : fields) {
orderings.add(this.orderingFromString(field));
}
return orderings.toArray(new Ordering[orderings.size()]);
return orderings;
}
private Ordering orderingFromString(String field) {

9
dmp-backend/elastic/src/main/java/eu/eudat/elastic/criteria/DatasetCriteria.java
View File

@ -18,6 +18,7 @@ public class DatasetCriteria extends Criteria {
private List<UUID> collaborators;
private Boolean allowAllVersions;
private List<String> organiztions;
private Boolean hasTags;
private List<Tag> tags;
private boolean isPublic;
private Short grantStatus;
@ -144,4 +145,12 @@ public class DatasetCriteria extends Criteria {
public void setSortCriteria(List<SortCriteria> sortCriteria) {
this.sortCriteria = sortCriteria;
}
public Boolean getHasTags() {
return hasTags;
}
public void setHasTags(Boolean hasTags) {
this.hasTags = hasTags;
}
}

70
dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java
View File

@ -283,38 +283,46 @@ public class Dataset implements ElasticEntity<Dataset> {
@Override
public Dataset fromElasticEntity(Map<String, Object> fields) {
if (fields != null) {
this.id = (String) fields.get("id");
if (fields.get("tags") != null) {
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
if (fields.size() == 1) {
if (fields.containsKey("id")) {
this.id = (String) fields.get("id");
} else if (fields.containsKey("tags")) {
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
}else if (fields.size() > 1) {
this.id = (String) fields.get("id");
if (fields.get("tags") != null) {
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
this.label = (String) fields.get("label");
this.description = (String) fields.get("description");
this.template = UUID.fromString((String) fields.get("template"));
this.status = Short.valueOf((String) fields.get("status"));
this.dmp = UUID.fromString((String) fields.get("dmp"));
this.group = UUID.fromString((String) fields.get("group"));
this.grant = UUID.fromString((String) fields.get("grant"));
if (fields.get("created") != null)
this.created = Date.from(Instant.parse((String) fields.get("created")));
if (fields.get("modified") != null)
this.modified = Date.from(Instant.parse((String) fields.get("modified")));
if (fields.get("finalizedAt") != null)
this.finalizedAt = Date.from(Instant.parse((String) fields.get("finalizedAt")));
if (fields.get("collaborators") != null) {
this.collaborators = ((List<HashMap>) fields.get("collaborators")).stream().map(hashMap -> new Collaborator().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
this.lastVersion = Boolean.parseBoolean((String) fields.get("lastVersion"));
this.lastPublicVersion = Boolean.parseBoolean((String) fields.get("lastPublicVersion"));
if (fields.get("organizations") != null) {
this.organizations = ((List<HashMap>) fields.get("organizations")).stream().map(hashMap -> new Organization().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
if (fields.get("public") != null) {
this.isPublic = Boolean.valueOf((String) fields.get("public"));
}
if (fields.get("grantStatus") != null) {
this.grantStatus = Short.valueOf((String) fields.get("grantStatus"));
}
this.formData = (String) fields.get("formData");
}
this.label = (String) fields.get("label");
this.description = (String) fields.get("description");
this.template = UUID.fromString((String) fields.get("template"));
this.status = Short.valueOf((String) fields.get("status"));
this.dmp = UUID.fromString((String) fields.get("dmp"));
this.group = UUID.fromString((String) fields.get("group"));
this.grant = UUID.fromString((String) fields.get("grant"));
if (fields.get("created") != null)
this.created = Date.from(Instant.parse((String) fields.get("created")));
if (fields.get("modified") != null)
this.modified = Date.from(Instant.parse((String) fields.get("modified")));
if (fields.get("finalizedAt") != null)
this.finalizedAt = Date.from(Instant.parse((String) fields.get("finalizedAt")));
if (fields.get("collaborators") != null) {
this.collaborators = ((List<HashMap>) fields.get("collaborators")).stream().map(hashMap -> new Collaborator().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
this.lastVersion = Boolean.parseBoolean((String) fields.get("lastVersion"));
this.lastPublicVersion = Boolean.parseBoolean((String) fields.get("lastPublicVersion"));
if (fields.get("organizations") != null) {
this.organizations = ((List<HashMap>) fields.get("organizations")).stream().map(hashMap -> new Organization().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
if (fields.get("public") != null) {
this.isPublic = Boolean.valueOf((String) fields.get("public"));
}
if (fields.get("grantStatus") != null) {
this.grantStatus = Short.valueOf((String) fields.get("grantStatus"));
}
this.formData = (String) fields.get("formData");
}
return this;
}

224
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
@ -122,68 +123,9 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
searchSourceBuilder.size(count.intValue());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
List<SortBuilder> sortBuilders = new ArrayList<>();
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastPublicVersion", "true"));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).allowLeadingWildcard(true).fields(Stream.of(new Object[][]{
{"datasets.label", 1.0f},
{"datasets.description", 1.0f},
{"datasets.formData", 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
if (criteria.getDatasetTemplates() != null && criteria.getDatasetTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.template", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", criteria.getStatus().toString()));
}
if (criteria.getDmps() != null && criteria.getDmps().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.dmp", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.group", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.grant", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.grantStatus", criteria.getGrantStatus().toString()));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isPublic()) {
if (criteria.getAllowAllVersions() != null && !criteria.getAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastVersion", "true"));
}
}
if (criteria.getOrganiztions() != null && criteria.getOrganiztions().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.organizations.id", criteria.getOrganiztions()));
}
if (criteria.getTags() != null && criteria.getTags().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
criteria.getSortCriteria().forEach(sortCriteria -> {
@ -202,8 +144,11 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
}
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).size(criteria.getSize());
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.tags"}, null)));
searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).fetchSource("datasets.tags", null);
if (criteria.getSize() > 0) {
searchSourceBuilder.size(criteria.getSize());
}
sortBuilders.forEach(searchSourceBuilder::sort);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
@ -215,6 +160,161 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
return null;
}
public List<Dataset> queryIds(DatasetCriteria criteria) throws IOException {
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
/*CountRequest countRequest = new CountRequest("dmps").routing("datasets").routing("id");
countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT);
Long count = countResponse.getCount();*/
SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
countSourceBuilder.aggregation(nestedAggregationBuilder);
countRequest.source(countSourceBuilder);
SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
Long count = ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
searchSourceBuilder.size(count.intValue());
List<SortBuilder> sortBuilders = new ArrayList<>();
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
criteria.getSortCriteria().forEach(sortCriteria -> {
switch(sortCriteria.getColumnType()) {
case COLUMN:
sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
break;
case JOIN_COLUMN:
List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
});
break;
}
});
}
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.id"}, null)));
searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).size(criteria.getSize()).fetchSource("datasets.id", null);
sortBuilders.forEach(searchSourceBuilder::sort);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
.map(SearchHits::getHits).flatMap(Arrays::stream)
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
}
return null;
}
@Override
public Long count(DatasetCriteria criteria) throws IOException {
if (this.getClient() != null) {
//CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", boolQuery);
nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
countSourceBuilder.aggregation(nestedAggregationBuilder);
countRequest.source(countSourceBuilder);
SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
return ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
/*NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder());
countRequest.query(nestedQueryBuilder);
CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
return response.getCount();*/
}
return null;
}
private BoolQueryBuilder createBoolQuery(DatasetCriteria criteria) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastPublicVersion", "true"));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).allowLeadingWildcard(true).fields(Stream.of(new Object[][]{
{"datasets.label", 1.0f},
{"datasets.description", 1.0f},
{"datasets.formData", 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getDatasetTemplates() != null && criteria.getDatasetTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.template", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", criteria.getStatus().toString()));
}
if (criteria.getDmps() != null && criteria.getDmps().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.dmp", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.group", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.grant", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.grantStatus", criteria.getGrantStatus().toString()));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isPublic()) {
if (criteria.getAllowAllVersions() != null && !criteria.getAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastVersion", "true"));
}
}
if (criteria.getOrganiztions() != null && criteria.getOrganiztions().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.organizations.id", criteria.getOrganiztions()));
}
if (criteria.getTags() != null && criteria.getTags().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
}
if (criteria.getHasTags() != null) {
boolQuery = criteria.getHasTags() == true ? boolQuery.should(QueryBuilders.existsQuery("datasets.tags.id")) : boolQuery.mustNot(QueryBuilders.existsQuery("datasets.tags.id"));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
return boolQuery;
}
@Override
public boolean exists() throws IOException {
if (this.getClient() != null) {

162
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DmpRepository.java
View File

@ -99,60 +99,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
searchSourceBuilder.size(count.intValue());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
List<SortBuilder> sortBuilders = new ArrayList<>();
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{Dmp.MapKey.LABEL.getName(), 1.0f},
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getRoles() != null && criteria.getRoles().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".role.keyword", criteria.getRoles()));
}
if (!criteria.isAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
}
if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
criteria.getSortCriteria().forEach(sortCriteria -> {
@ -170,8 +118,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
});
}
searchSourceBuilder.query(boolQuery).from(criteria.getOffset());
if (criteria.getSize() != null) {
searchSourceBuilder.query(boolQuery).from(criteria.getOffset()).fetchSource("id", null);
if (criteria.getSize() > 0) {
searchSourceBuilder.size(criteria.getSize());
}
sortBuilders.forEach(searchSourceBuilder::sort);
@ -182,59 +130,12 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
return null;
}
@Override
public Long count(DmpCriteria criteria) throws IOException {
if (this.getClient() != null) {
CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{Dmp.MapKey.LABEL.getName(), 1.0f},
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
}
if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
countRequest.query(boolQuery);
CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
@ -243,6 +144,59 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
return null;
}
private BoolQueryBuilder createBoolQuery(DmpCriteria criteria) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{Dmp.MapKey.LABEL.getName(), 1.0f},
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
}
if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
return boolQuery;
}
public boolean createIndex() {
try {
if (!this.exists()) {

2
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/Repository.java
View File

@ -18,6 +18,8 @@ public interface Repository<ET extends ElasticEntity, C extends Criteria> {
List<ET> query(C criteria) throws ExecutionException, InterruptedException, IOException;
Long count(C criteria) throws ExecutionException, InterruptedException, IOException;
boolean exists() throws IOException;
void clear() throws IOException;

6
dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java
View File

@ -1,10 +1,8 @@
package eu.eudat.controllers;
import eu.eudat.elastic.criteria.DatasetCriteria;
import eu.eudat.elastic.criteria.TagCriteria;
import eu.eudat.elastic.entities.Dataset;
import eu.eudat.elastic.entities.Tag;
import eu.eudat.elastic.repository.Repository;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
@ -49,7 +47,9 @@ public class TagController extends BaseController {
/*List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type);
TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);*/
if (this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().exists()) {
List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
DatasetCriteria criteria = new DatasetCriteria();
criteria.setHasTags(true);
List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tag.getName().startsWith(query)).collect(Collectors.toList());
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<Tag>>().payload(tags).status(ApiMessageCode.NO_MESSAGE));
} else {

163
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DashBoardManager.java
View File

@ -8,13 +8,12 @@ import eu.eudat.data.dao.entities.DMPDao;
import eu.eudat.data.dao.entities.DatasetDao;
import eu.eudat.data.dao.entities.GrantDao;
import eu.eudat.data.dao.entities.OrganisationDao;
import eu.eudat.data.entities.DMP;
import eu.eudat.data.entities.Dataset;
import eu.eudat.data.entities.Grant;
import eu.eudat.data.entities.UserInfo;
import eu.eudat.data.entities.*;
import eu.eudat.data.query.PaginationService;
import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
import eu.eudat.elastic.entities.Dmp;
import eu.eudat.logic.builders.model.models.RecentActivityDataBuilder;
import eu.eudat.logic.mapper.elastic.criteria.DmpCriteriaMapper;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.models.HintedModelFactory;
@ -26,6 +25,8 @@ import eu.eudat.models.data.dashboard.recent.model.RecentDmpModel;
import eu.eudat.models.data.dashboard.recent.tablerequest.RecentActivityTableRequest;
import eu.eudat.models.data.dashboard.searchbar.SearchBarItem;
import eu.eudat.models.data.dashboard.statistics.DashBoardStatistics;
import eu.eudat.models.data.listingmodels.DataManagementPlanListingModel;
import eu.eudat.models.data.listingmodels.DatasetListingModel;
import eu.eudat.models.data.security.Principal;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.searchbar.SearchBarItemType;
@ -97,8 +98,8 @@ public class DashBoardManager {
}
public DashBoardStatistics getMeStatistics(Principal principal) throws IOException {
List<eu.eudat.elastic.entities.Dataset> datasets = null;
List<eu.eudat.elastic.entities.Dmp> dmps = null;
Long datasets = 0L;
Long dmps = 0L;
DashBoardStatistics statistics = new DashBoardStatistics();
DMPDao dataManagementPlanRepository = databaseRepository.getDmpDao();
DatasetDao datasetRepository = databaseRepository.getDatasetDao();
@ -112,7 +113,8 @@ public class DashBoardManager {
eu.eudat.elastic.criteria.DatasetCriteria datasetElasticCriteria = new eu.eudat.elastic.criteria.DatasetCriteria();
datasetElasticCriteria.setAllowAllVersions(false);
datasetElasticCriteria.setPublic(false);
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().count(datasetElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
datasets = null;
@ -126,33 +128,40 @@ public class DashBoardManager {
eu.eudat.elastic.criteria.DmpCriteria dmpElasticCriteria = new eu.eudat.elastic.criteria.DmpCriteria();
dmpElasticCriteria.setAllowAllVersions(false);
dmpElasticCriteria.setPublic(false);
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().count(dmpElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
dmps = null;
}
}
dataManagementPlanCriteria.setAllVersions(false);
GrantCriteria grantCriteria = new GrantCriteria();
dataManagementPlanCriteria.setOnlyPublic(false);
dataManagementPlanCriteria.setIsPublic(false);
GrantCriteria grantCriteria = new GrantCriteria();
grantCriteria.setActive(true);
OrganisationCriteria organisationCriteria = new OrganisationCriteria();
organisationCriteria.setActive(true);
List<Integer> roles = new LinkedList<>();
List<Dmp> finalDmps = dmps;
CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated((dmps != null && !dmps.isEmpty()) ? dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))) : dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
.whenComplete((dmpsStats, throwable) -> statistics.setTotalDataManagementPlanCount(dmpsStats));
List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
CompletableFuture datasetFuture = datasetRepository.getAuthenticated((datasets != null && !datasets.isEmpty()) ? datasetRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))) : datasetRepository.getWithCriteria(datasetCriteria), user, roles).countAsync()
.whenComplete((datasetsStats, throwable) -> statistics.setTotalDataSetCount(datasetsStats));
if ((dmps == null || dmps == 0L) && (datasets == null || datasets == 0L)) {
CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated(dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
.whenComplete((dmpsStats, throwable) -> statistics.setTotalDataManagementPlanCount(dmpsStats));
CompletableFuture datasetFuture = datasetRepository.getAuthenticated( datasetRepository.getWithCriteria(datasetCriteria), user, roles).countAsync()
.whenComplete((datasetsStats, throwable) -> statistics.setTotalDataSetCount(datasetsStats));
CompletableFuture.allOf(dmpFuture, datasetFuture).join();
} else {
statistics.setTotalDataManagementPlanCount(dmps);
statistics.setTotalDataSetCount(datasets);
}
CompletableFuture grantFuture = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user).countAsync()
.whenComplete((grantsStats, throwable) -> statistics.setTotalGrantCount(grantsStats));
CompletableFuture orgnanisationFuture = organisationRepository.getAuthenticated(organisationRepository.getWithCriteria(organisationCriteria).withHint("organisationRecentActivity"), user).countAsync()
.whenComplete((organisationStats, throwable) -> statistics.setTotalOrganisationCount(organisationStats));
CompletableFuture.allOf(dmpFuture, datasetFuture, grantFuture, orgnanisationFuture).join();
CompletableFuture.allOf( grantFuture, orgnanisationFuture).join();
return statistics;
}
@ -185,14 +194,14 @@ public class DashBoardManager {
.selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
.whenComplete((datasetActivities, throwable) -> activity.setRecentDatasetActivities(datasetActivities));
/*CompletableFuture<List<RecentActivityData>> grants = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user)
CompletableFuture<List<RecentActivityData>> grants = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user)
.withHint("grantRecentActivity")
.orderBy((builder, root) -> builder.desc(root.get("modified")))
.take(numberofactivities)
.selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
.whenComplete((grantActivities, throwable) -> activity.setRecentGrantActivities(grantActivities));*/
.whenComplete((grantActivities, throwable) -> activity.setRecentGrantActivities(grantActivities));
CompletableFuture.allOf(/*grants, */dmps, datasets).join();
CompletableFuture.allOf(grants, dmps, datasets).join();
return activity;
}
@ -228,7 +237,13 @@ public class DashBoardManager {
datasetElasticCriteria.setLike(tableRequest.getCriteria().getLike());
datasetElasticCriteria.setAllowAllVersions(false);
datasetElasticCriteria.setPublic(!isAuthenticated);
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
datasetElasticCriteria.setOffset(tableRequest.getDatasetOffset());
datasetElasticCriteria.setSize(tableRequest.getLength());
if (isAuthenticated) {
datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
}
datasetElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().queryIds(datasetElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
datasets = null;
@ -241,10 +256,16 @@ public class DashBoardManager {
dmpElasticCriteria.setLike(tableRequest.getCriteria().getLike());
dmpElasticCriteria.setAllowAllVersions(false);
dmpElasticCriteria.setPublic(!isAuthenticated);
dmpElasticCriteria.setOffset(tableRequest.getDmpOffset());
dmpElasticCriteria.setSize(tableRequest.getLength());
if (isAuthenticated) {
dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
}
dmpElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
datasets = null;
dmps = null;
}
}
@ -253,6 +274,8 @@ public class DashBoardManager {
dmpList = dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))).distinct();
} else {
dmpList = dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria).distinct();
PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
dmpList.skip(tableRequest.getDmpOffset()).take(tableRequest.getLength());
}
if (datasets != null && !datasets.isEmpty()) {
@ -260,50 +283,50 @@ public class DashBoardManager {
datasetList = datasetRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
} else {
datasetList = datasetRepository.getWithCriteria(datasetCriteria);
for (int i = 0; i< tableRequest.getOrderings().getFields().size(); i++) {
if (tableRequest.getOrderings().getFields().get(i).contains("publishedAt")) {
String newField = tableRequest.getOrderings().getFields().get(i).toCharArray()[0] + "dmp:publishedAt|join|";
tableRequest.getOrderings().getFields().set(i, newField);
}
}
}
if (tableRequest.getOrderings().getFields().get(0).contains("publishedAt")) {
tableRequest.getOrderings().getFields().set(0, tableRequest.getOrderings().getFields().get(0).charAt(0) + "dmp:" + tableRequest.getOrderings().getFields().get(0).substring(1) + "|join|");
}
PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
datasetList.skip(tableRequest.getDatasetOffset()).take(tableRequest.getLength());
if (isAuthenticated) {
List<Integer> roles = new LinkedList<>();
roles.add(UserDMP.UserDMPRoles.USER.getValue());
roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
dmpList = dataManagementPlanRepository.getAuthenticated(dmpList, principal.getId(), roles);
datasetList = datasetRepository.getAuthenticated(datasetList, user, roles);
}
PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
for (int i = 0; i< tableRequest.getOrderings().getFields().length; i++) {
if (tableRequest.getOrderings().getFields()[i].contains("publishedAt")) {
String newField = tableRequest.getOrderings().getFields()[i].toCharArray()[0] + "dmp:publishedAt|join|";
tableRequest.getOrderings().getFields()[i] = newField;
}
}
PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
/*CompletableFuture future = CompletableFuture.runAsync(() -> */{
recentActivityModels.addAll(dmpList
.withHint(HintedModelFactory.getHint(RecentDmpModel.class))
// .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
.skip(tableRequest.getDmpOffset())
.take(tableRequest.getLength())
.select(item -> {
item.setDataset(
item.getDataset().stream()
.filter(dataset -> !dataset.getStatus().equals(Dataset.Status.DELETED.getValue()) && !dataset.getStatus().equals(Dataset.Status.CANCELED.getValue())).collect(Collectors.toList()).stream()
.filter(dataset -> dataset.getDmp().getUsers().stream()
.filter(x -> x.getUser().getId().equals(principal.getId()))
.collect(Collectors.toList()).size() > 0)
.collect(Collectors.toSet()));
return new RecentDmpModel().fromEntity(item);
}));
List<DMP> dmps1 = dmpList.withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))
.distinct().toList();
recentActivityModels.addAll(dmps1.stream().map(dmp -> {
DatasetCriteria datasetCriteria1 = new DatasetCriteria();
datasetCriteria1.setDmpIds(Collections.singletonList(dmp.getId()));
if (isAuthenticated) {
dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1, principal.getId()));
} else {
dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1));
}
return new RecentDmpModel().fromDataModel(dmp);
}).collect(Collectors.toList()));
recentActivityModels.addAll(datasetList
.withHint(HintedModelFactory.getHint(RecentDatasetModel.class))
// .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
.skip(tableRequest.getDatasetOffset())
.take(tableRequest.getLength())
.select(item -> {
return new RecentDatasetModel().fromEntity(item);
}));
List<RecentActivityModel> recentDatasetModels = datasetList
.withHint(HintedModelFactory.getHint(DatasetListingModel.class))
.select(item -> new RecentDatasetModel().fromEntity(item));
recentActivityModels.addAll(recentDatasetModels);
}/*);*/
//GK: Shuffle the deck otherwise we will summon the DMPodia when sorting with status
@ -322,7 +345,7 @@ public class DashBoardManager {
//CompletableFuture.allOf(future).join();
// CompletableFuture.allOf(dmps, datasets).join();
return recentActivityModels.stream().sorted(this.comparators.get(tableRequest.getCriteria().getOrder())).limit(tableRequest.getLength()).collect(Collectors.toList());
return recentActivityModels;
}
public List<SearchBarItem> searchUserData(String like, Principal principal) {
@ -383,4 +406,38 @@ public class DashBoardManager {
return searchBarItems;
}
private Set<Dataset> retrieveRelevantDatasets(DatasetCriteria datasetCriteria) {
return retrieveRelevantDatasets(datasetCriteria, null);
}
private Set<Dataset> retrieveRelevantDatasets (DatasetCriteria datasetCriteria, UUID principal) {
QueryableList<Dataset> datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria);
if (principal != null) {
UserInfo userInfo = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal);
List<Integer> roles = new ArrayList<>();
roles.add(0);
roles.add(1);
datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getAuthenticated(datasetItems, userInfo, roles);
}
Long maxDatasets = datasetItems.count();
DatasetTableRequest datasetTableRequest = new DatasetTableRequest();
datasetTableRequest.setOffset(0);
datasetTableRequest.setLength(3);
Set<Dataset> datasetsSet = new LinkedHashSet<>();
try {
datasetItems = PaginationManager.applyPaging(datasetItems, datasetTableRequest);
List<Dataset> datasets = datasetItems.toList();
datasetsSet.addAll(datasets);
for (int i = 0; i < maxDatasets - datasets.size(); i++) {
Dataset fakedataset = new Dataset();
fakedataset.setId(UUID.randomUUID());
datasetsSet.add(fakedataset);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return datasetsSet;
}
}

37
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java
View File

@ -88,7 +88,6 @@ import java.nio.file.Files;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
@ -157,7 +156,7 @@ public class DatasetManager {
List<eu.eudat.elastic.entities.Dataset> datasets;
try {
datasets = datasetRepository.exists() ?
datasetRepository.query(datasetCriteria) : null;
datasetRepository.queryIds(datasetCriteria) : null;
} catch (Exception ex) {
logger.warn(ex.getMessage(), ex);
datasets = null;
@ -186,8 +185,12 @@ public class DatasetManager {
if (principal.getId() == null) {
throw new UnauthorisedException("You are not allowed to access those datasets");
}
if (datasetTableRequest.getCriteria().getRole() != null)
if (datasetTableRequest.getCriteria().getRole() != null) {
roles.add(datasetTableRequest.getCriteria().getRole());
} else {
roles.add(UserDMP.UserDMPRoles.USER.getValue());
roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
}
authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles);
pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
} else {
@ -215,6 +218,7 @@ public class DatasetManager {
}
public DataTableData<DatasetListingModel> getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception {
Long count = 0L;
DatasetCriteria datasetCriteria = new DatasetCriteria();
datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile());
@ -228,7 +232,8 @@ public class DatasetManager {
List<eu.eudat.elastic.entities.Dataset> datasets;
try {
datasets = datasetRepository.exists() ?
datasetRepository.query(datasetCriteria) : new LinkedList<>();
datasetRepository.queryIds(datasetCriteria) : new LinkedList<>();
count = datasetRepository.exists() ? datasetRepository.count(datasetCriteria) : 0L;
} catch (Exception ex) {
logger.warn(ex.getMessage());
datasets = null;
@ -254,22 +259,22 @@ public class DatasetManager {
return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
});
}
String[] strings = new String[1];
strings[0] = "-dmp:publishedAt|join|";
List<String> strings = new ArrayList<>();
strings.add("-dmp:publishedAt|join|");
datasetTableRequest.getOrderings().setFields(strings);
if (count == 0L) {
count = items.count();
}
QueryableList<eu.eudat.data.entities.Dataset> pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
CompletableFuture<List<DatasetListingModel>> itemsFuture = pagedItems.
selectAsync(this::mapModel).whenComplete((resultList, throwable) -> {
dataTable.setData(resultList.stream().filter(Objects::nonNull).collect(Collectors.toList()));
});
List<DatasetListingModel> datasetListis = pagedItems.
select(this::mapModel);
CompletableFuture countFuture = pagedItems.countAsync().whenComplete((count, throwable) -> {
dataTable.setTotalCount(count);
});
dataTable.setData(datasetListis.stream().filter(Objects::nonNull).collect(Collectors.toList()));
CompletableFuture.allOf(itemsFuture, countFuture).join();
dataTable.setTotalCount(count);
//CompletableFuture.allOf(countFuture).join();
return dataTable;
}
@ -1079,7 +1084,9 @@ public class DatasetManager {
ObjectMapper mapper = new ObjectMapper();
String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition());
JsonNode propertiesJson = mapper.readTree(json);
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
DatasetCriteria criteria = new DatasetCriteria();
criteria.setHasTags(true);
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
Set<JsonNode> tagNodes = new HashSet<>();
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true));
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "rdaProperty", "dataset.keyword"));

1
dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/DatasetMapper.java
View File

@ -31,6 +31,7 @@ public class DatasetMapper {
if (tags != null && !tags.isEmpty()) {
DatasetCriteria criteria = new DatasetCriteria();
criteria.setTags(tags);
criteria.setHasTags(true);
List<Tag> tags1 = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
.filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tags.stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
if (tags1.isEmpty()) {

2
dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/criteria/DmpCriteriaMapper.java
View File

@ -45,7 +45,7 @@ public class DmpCriteriaMapper {
public static List<SortCriteria> toElasticSorting(ColumnOrderings columnOrderings) {
List<SortCriteria> sortCriteria = new ArrayList<>();
if (columnOrderings.getFieldOrderings() != null && columnOrderings.getFieldOrderings().length > 0) {
if (columnOrderings.getFieldOrderings() != null && !columnOrderings.getFieldOrderings().isEmpty()) {
for (Ordering ordering: columnOrderings.getFieldOrderings()) {
SortCriteria sortCriteria1 = new SortCriteria();
sortCriteria1.setFieldName(ordering.getFieldName() + (ordering.getFieldName().contains("label") ?".keyword" : ""));

2
dmp-backend/web/src/main/java/eu/eudat/logic/services/forms/VisibilityRuleService.java
View File

@ -9,7 +9,7 @@ import java.util.Map;
* Created by ikalyvas on 3/5/2018.
*/
public interface VisibilityRuleService {
Boolean isElementVisible(String id);
boolean isElementVisible(String id);
void buildVisibilityContext(List<Rule> sources);

2
dmp-backend/web/src/main/java/eu/eudat/logic/services/forms/VisibilityRuleServiceImpl.java
View File

@ -16,7 +16,7 @@ public class VisibilityRuleServiceImpl implements VisibilityRuleService {
private VisibilityContext visibilityContext;
private Map<String, Object> properties;
public Boolean isElementVisible(String id) {
public boolean isElementVisible(String id) {
if (!this.elementVisibility.containsKey(id) || this.elementVisibility.get(id)) return true;
return false;
}