Optimize queries

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDao.java

@@ -13,6 +13,8 @@ public interface DatasetDao extends DatabaseAccessLayer<Dataset, UUID> {
 
     QueryableList<Dataset> getWithCriteria(DatasetCriteria criteria);
 
+    QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids);
+
     QueryableList<Dataset> getAuthenticated(QueryableList<Dataset> query, UserInfo principal, List<Integer> roles);
 
     Dataset isPublicDataset(UUID id);

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDaoImpl.java

@@ -87,6 +87,16 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
         return query;
     }
 
+    public QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids) {
+        QueryableList<Dataset> query = getDatabaseService().getQueryable(Dataset.getHints(), Dataset.class);
+
+        query.where(((builder, root) -> root.get("id").in(ids)));
+        if (!criteria.getAllVersions())
+            query.initSubQuery(String.class).where((builder, root) -> builder.equal(root.get("dmp").get("version"), query.<String>subQueryMax((builder1, externalRoot, nestedRoot) -> builder1.and(builder1.equal(externalRoot.get("dmp").get("groupId"), nestedRoot.get("dmp").get("groupId")), builder1.notEqual(nestedRoot.get("dmp").get("status"), DMP.DMPStatus.DELETED.getValue())), Arrays.asList(new SelectionField(FieldSelectionType.COMPOSITE_FIELD, "dmp:version")), String.class)));
+
+        return query;
+    }
+
     @Override
     public Dataset createOrUpdate(Dataset item) {
         return getDatabaseService().createOrUpdate(item, Dataset.class);
@@ -121,6 +131,7 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
             });
         } else {
             query.where((builder, root) -> builder.equal(root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT).join("user", JoinType.LEFT).get("id"), principal.getId()));
+
         }
         return query;
     }

dmp-backend/data/src/main/java/eu/eudat/data/entities/Dataset.java

@@ -16,13 +16,14 @@ import java.util.stream.Collectors;
 @NamedEntityGraphs({
 		@NamedEntityGraph(
 				name = "datasetListingModel",
-				attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
+				attributeNodes = {/*@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
-						@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),
+						@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),*/
-						@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode("profile"), @NamedAttributeNode("creator")},
+						@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode(value = "profile", subgraph = "profile"), @NamedAttributeNode("creator")},
 				subgraphs = {
 						@NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users"), @NamedAttributeNode("grant"), @NamedAttributeNode("organisations")}),
 						@NamedSubgraph(name = "datasetDataRepositories", attributeNodes = {@NamedAttributeNode("dataRepository")}),
-						@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")})
+						@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")}),
+						@NamedSubgraph(name = "profile", attributeNodes = {@NamedAttributeNode("label")})
 				}),
 
 		@NamedEntityGraph(

dmp-backend/queryable/src/main/java/eu/eudat/queryable/QueryableList.java

@@ -44,6 +44,8 @@ public interface QueryableList<T extends DataEntity> {
 
     QueryableList<T> orderBy(OrderByPredicate<T> predicate);
 
+    QueryableList<T> groupBy(GroupByPredicate<T> predicate);
+
     QueryableList<T> withHint(String hint);
 
     Long count();

dmp-backend/queryable/src/main/java/eu/eudat/queryable/collector/Collector.java

@@ -1,6 +1,9 @@
 package eu.eudat.queryable.collector;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import javax.persistence.Tuple;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -9,9 +12,13 @@ import java.util.stream.Collectors;
 public class Collector {
 
 	public List<Map> buildFromTuple(List<Tuple> results, Map<Object, List<Tuple>> groupedResults, List<String> fields, String key) {
-		return groupedResults.keySet().stream()
+		return results.stream().map(tuple -> {
+			Map<String, Object> parsedResult = new HashMap<>();
+			tuple.getElements().forEach(tupleElement -> parsedResult.put(tupleElement.getAlias(), tuple.get(tupleElement.getAlias())));
+			return parsedResult;
+		}).collect(Collectors.toList());/*groupedResults.keySet().stream()
 				.map(x -> buildOne(groupedResults.get(x), fields, key))
-				.collect(Collectors.toList());
+				.collect(Collectors.toList());*/
 	}
 
 	private Map buildOne(List<Tuple> tuples, List<String> fields, String key) {

dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/hibernatequeryablelist/QueryableHibernateList.java

@@ -1,5 +1,6 @@
 package eu.eudat.queryable.jpa.hibernatequeryablelist;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.queryable.QueryableList;
 import eu.eudat.queryable.collector.Collector;
 import eu.eudat.queryable.exceptions.NotSingleResultException;
@@ -33,6 +34,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 	private List<NestedQuerySinglePredicate<T>> nestedPredicates = new LinkedList<>();
 	private boolean distinct = false;
 	private List<OrderByPredicate<T>> orderings = new LinkedList<>();
+	private List<GroupByPredicate<T>> groupings = new LinkedList<>();
 	private List<String> fields = new LinkedList<>();
 	private Integer length;
 	private Integer offset;
@@ -150,6 +152,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		return this;
 	}
 
+	public QueryableList<T> groupBy(GroupByPredicate<T> predicate) {
+		this.groupings.add(predicate);
+		return this;
+	}
+
 	public Long count() {
 		CriteriaBuilder criteriaBuilder = this.manager.getCriteriaBuilder();
 		CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
@@ -205,6 +212,14 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		return predicates.toArray(new Order[predicates.size()]);
 	}
 
+	private Expression[] generateGroupPredicates(List<GroupByPredicate<T>> groupByPredicates, Root<T> root) {
+		List<Expression> predicates = new LinkedList<>();
+		for (GroupByPredicate<T> groupPredicate : groupByPredicates) {
+			predicates.add(groupPredicate.applyPredicate(this.manager.getCriteriaBuilder(), root));
+		}
+		return predicates.toArray(new Expression[predicates.size()]);
+	}
+
 	public List<T> toList() {
 		CriteriaBuilder builder = this.manager.getCriteriaBuilder();
 		if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
@@ -212,22 +227,20 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		this.root = this.query.from(this.tClass);
 		this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
 		if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
+		if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
 		if (!this.fields.isEmpty()) this.selectFields();
 		if (distinct) this.query.distinct(true);
-		//if (!this.fields.isEmpty()) return this.toListWithFields();
+		//if (!this.fields.isEmpty()) this.query.multiselect(this.parseFields(this.fields));
+		ObjectMapper mapper = new ObjectMapper();
+		if (!this.fields.isEmpty()) return this.toListWithFields().stream().map(m -> mapper.convertValue(m, this.tClass)).collect(Collectors.toList());
 		return this.toListWithOutFields();
 	}
 
 	public List<Map> toListWithFields() {
-		CriteriaBuilder builder = this.manager.getCriteriaBuilder();
+		TypedQuery<Tuple> typedQuery = this.manager.createQuery(this.query);
-		if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
+		if (this.offset != null) typedQuery.setFirstResult(this.offset);
-		else this.query = builder.createQuery(this.tClass);
+		if (this.length != null) typedQuery.setMaxResults(this.length);
-		this.root = this.query.from(this.tClass);
+		List<Tuple> results = typedQuery.getResultList();
-		this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
-		if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
-		if (!this.fields.isEmpty()) this.selectFields();
-		if (distinct) this.query.distinct(true);
-		List<Tuple> results = this.manager.createQuery(query).getResultList();
 		Map<Object, List<Tuple>> groupedResults = results.stream()
 				.collect(Collectors.groupingBy(x -> x.get("id")));
 		return this.collector.buildFromTuple(results, groupedResults, this.fields, "id");
@@ -252,6 +265,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		this.root = this.query.from(this.tClass);
 		this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
 		if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
+		if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
 		if (!this.fields.isEmpty()) this.selectFields();
 		if (distinct) this.query.distinct(true);
 		if (!this.fields.isEmpty()) return this.toListAsyncWithFields();
@@ -356,6 +370,8 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		if (!this.orderings.isEmpty())
 			criteriaQuery.orderBy(this.generateOrderPredicates(this.orderings, criteriaRoot));
 
+		if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
+
 		TypedQuery<T> typedQuery = this.manager.createQuery(criteriaQuery);
 		typedQuery.setHint("javax.persistence.fetchgraph", this.manager.getEntityGraph(this.hint));
 		return typedQuery;
@@ -466,4 +482,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 				.createQuery(update)
 				.executeUpdate();
 	}
+
+	private Path[] parseFields(List<String> selectedFields) {
+		List<Path> paths = new ArrayList<>();
+		selectedFields.forEach(s -> paths.add(root.get(s)));
+
+		return paths.toArray(new Path[0]);
+	}
 }

dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/predicates/GroupByPredicate.java

@@ -0,0 +1,10 @@
+package eu.eudat.queryable.jpa.predicates;
+
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.Expression;
+import javax.persistence.criteria.Root;
+
+public interface GroupByPredicate<T> {
+    Expression<T> applyPredicate(CriteriaBuilder builder, Root<T> root);
+
+}

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java

@@ -1171,8 +1171,13 @@ public class DataManagementPlanManager {
                  .forEach(datasetEntity -> {
                      Map<String, Object> properties = new HashMap<>();
                      if (datasetEntity.getProperties() != null) {
-                         JSONObject jObject = new JSONObject(datasetEntity.getProperties());
+                         try {
-                         properties = jObject.toMap();
+                             properties = new ObjectMapper().readValue(datasetEntity.getProperties(), LinkedHashMap.class);
+                         } catch (IOException e) {
+                             logger.error(e.getLocalizedMessage(), e);
+                         }
+                         /*JSONObject jObject = new JSONObject(datasetEntity.getProperties());
+                         properties = jObject.toMap();*/
                      }
 
                      // Custom style for the Dataset title.

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java

@@ -155,28 +155,33 @@ public class DatasetManager {
         datasetCriteria.setSize(datasetTableRequest.getLength());
         List<eu.eudat.elastic.entities.Dataset> datasets;
         try {
+            long start = System.currentTimeMillis();
             datasets = datasetRepository.exists() ?
                     datasetRepository.queryIds(datasetCriteria) : null;
+            long end = System.currentTimeMillis();
+            logger.info("Elastic took " + (end - start)/ 1000f + "s");
         } catch (Exception ex) {
             logger.warn(ex.getMessage(), ex);
             datasets = null;
         }
 
+        long start = System.currentTimeMillis();
         UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
 //        QueryableList<eu.eudat.data.entities.Dataset> items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
         QueryableList<eu.eudat.data.entities.Dataset> items;
         if (datasets != null) {
 
             if (!datasets.isEmpty()) {
-                items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
+                //items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
-                List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
+                final List<UUID> datasetIds = datasets.stream().map(datasetE -> UUID.fromString(datasetE.getId())).distinct().collect(Collectors.toList());
-                items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
+                items = databaseRepository.getDatasetDao().filterFromElastic(datasetTableRequest.getCriteria(), datasetIds).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
+                //items.where((builder, root) -> root.get("id").in(datasetIds));
             } else {
-                items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
+                items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
                 //items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
             }
         } else {
-            items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
+            items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
         }
         List<Integer> roles = new LinkedList<>();
         QueryableList<eu.eudat.data.entities.Dataset> pagedItems;
@@ -187,9 +192,6 @@ public class DatasetManager {
             }
             if (datasetTableRequest.getCriteria().getRole() != null) {
                 roles.add(datasetTableRequest.getCriteria().getRole());
-            } else {
-                roles.add(UserDMP.UserDMPRoles.USER.getValue());
-                roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
             }
             authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles);
             pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
@@ -209,10 +211,26 @@ public class DatasetManager {
         DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
 
 
-        dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));
+
+       /* long countStart = System.currentTimeMillis();
+        List<UUID> pagedIds = pagedItems.toList().stream().map(Dataset::getId).collect(Collectors.toList());
+        long countEnd = System.currentTimeMillis();
+
+        logger.info("Paged Ids took " + (countEnd - countStart)/ 1000f + "s");*/
 
         dataTable.setTotalCount(authItems.count());
 
+       // items = databaseRepository.getDatasetDao().asQueryable().where((builder, root) -> root.get("id").in(pagedIds)).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
+     //  PaginationManager.applyOrder(items, datasetTableRequest);
+        long dataStart = System.currentTimeMillis();
+        dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));
+
+        long dataEnd = System.currentTimeMillis();
+
+        logger.info("Data retrieval took " + (dataEnd - dataStart)/1000f + "s" );
+
+        long end = System.currentTimeMillis();
+        logger.info("DB took " + (end - start)/ 1000f + "s");
         //CompletableFuture.allOf(itemsFuture, countFuture).join();
         return dataTable;
     }
@@ -1122,10 +1140,10 @@ public class DatasetManager {
 
     @Transactional
     private DatasetListingModel mapModel(Dataset item) {
-        if (item.getProfile() == null)
+        /*if (item.getProfile() == null)
-            return null;
+            return null;*/
         DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item);
-        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
+        /*DatasetProfileCriteria criteria = new DatasetProfileCriteria();
         criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId()));
         List<DatasetProfile> profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
         boolean islast = false;
@@ -1133,7 +1151,7 @@ public class DatasetManager {
             profiles = profiles.stream().sorted(Comparator.comparing(DatasetProfile::getVersion)).collect(Collectors.toList());
             islast = profiles.get(0).getId().equals(item.getProfile().getId());
         }
-        listingModel.setProfileLatestVersion(islast);
+        listingModel.setProfileLatestVersion(islast);*/
         return listingModel;
     }
 

dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DatasetListingModel.java

@@ -196,14 +196,14 @@ public class DatasetListingModel implements DataModel<Dataset, DatasetListingMod
         this.grant = entity.getDmp() != null ? entity.getDmp().getGrant().getLabel() : "";
         this.dmp = entity.getDmp() != null ? entity.getDmp().getLabel() : "";
         this.dmpId = entity.getDmp() != null ? entity.getDmp().getId().toString() : "";
-        this.profile = entity.getProfile() != null ? new DatasetProfileOverviewModel().fromDataModel(entity.getProfile()) : null;
+        //this.profile = entity.getProfile() != null ? new DatasetProfileOverviewModel().fromDataModel(entity.getProfile()) : null;
         this.description = entity.getDescription();
         this.status = entity.getStatus();
-        this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
+        //this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
-        this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
+       // this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
-        this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
+        //this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
-        this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
+       // this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
-        this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
+        //this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
         if (entity.getFinalizedAt() == null && entity.getStatus() == Dataset.Status.FINALISED.getValue()) {
             this.finalizedAt = entity.getDmp().getFinalizedAt();
         } else {