From 91729bab6d5e999a8add4ab6bc98e39e89ca8b7a Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Thu, 30 Sep 2021 18:02:53 +0300 Subject: [PATCH] Optimize queries --- .../eudat/data/dao/entities/DatasetDao.java | 2 + .../data/dao/entities/DatasetDaoImpl.java | 11 +++++ .../java/eu/eudat/data/entities/Dataset.java | 9 ++-- .../eu/eudat/queryable/QueryableList.java | 2 + .../eudat/queryable/collector/Collector.java | 11 ++++- .../QueryableHibernateList.java | 43 +++++++++++++----- .../jpa/predicates/GroupByPredicate.java | 10 +++++ .../managers/DataManagementPlanManager.java | 9 +++- .../eudat/logic/managers/DatasetManager.java | 44 +++++++++++++------ .../listingmodels/DatasetListingModel.java | 12 ++--- 10 files changed, 116 insertions(+), 37 deletions(-) create mode 100644 dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/predicates/GroupByPredicate.java diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDao.java b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDao.java index bcf5d134f..126958e11 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDao.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDao.java @@ -13,6 +13,8 @@ public interface DatasetDao extends DatabaseAccessLayer { QueryableList getWithCriteria(DatasetCriteria criteria); + QueryableList filterFromElastic(DatasetCriteria criteria, List ids); + QueryableList getAuthenticated(QueryableList query, UserInfo principal, List roles); Dataset isPublicDataset(UUID id); diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDaoImpl.java b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDaoImpl.java index 3c5b3d3bc..50462c163 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDaoImpl.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDaoImpl.java @@ -87,6 +87,16 @@ public class DatasetDaoImpl extends DatabaseAccess implements DatasetDa return query; } + public QueryableList filterFromElastic(DatasetCriteria criteria, List ids) { + QueryableList query = getDatabaseService().getQueryable(Dataset.getHints(), Dataset.class); + + query.where(((builder, root) -> root.get("id").in(ids))); + if (!criteria.getAllVersions()) + query.initSubQuery(String.class).where((builder, root) -> builder.equal(root.get("dmp").get("version"), query.subQueryMax((builder1, externalRoot, nestedRoot) -> builder1.and(builder1.equal(externalRoot.get("dmp").get("groupId"), nestedRoot.get("dmp").get("groupId")), builder1.notEqual(nestedRoot.get("dmp").get("status"), DMP.DMPStatus.DELETED.getValue())), Arrays.asList(new SelectionField(FieldSelectionType.COMPOSITE_FIELD, "dmp:version")), String.class))); + + return query; + } + @Override public Dataset createOrUpdate(Dataset item) { return getDatabaseService().createOrUpdate(item, Dataset.class); @@ -121,6 +131,7 @@ public class DatasetDaoImpl extends DatabaseAccess implements DatasetDa }); } else { query.where((builder, root) -> builder.equal(root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT).join("user", JoinType.LEFT).get("id"), principal.getId())); + } return query; } diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/entities/Dataset.java b/dmp-backend/data/src/main/java/eu/eudat/data/entities/Dataset.java index 66c917cb0..41081cefb 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/entities/Dataset.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/entities/Dataset.java @@ -16,13 +16,14 @@ import java.util.stream.Collectors; @NamedEntityGraphs({ @NamedEntityGraph( name = "datasetListingModel", - attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"), - @NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"), - @NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode("profile"), @NamedAttributeNode("creator")}, + attributeNodes = {/*@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"), + @NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),*/ + @NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode(value = "profile", subgraph = "profile"), @NamedAttributeNode("creator")}, subgraphs = { @NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users"), @NamedAttributeNode("grant"), @NamedAttributeNode("organisations")}), @NamedSubgraph(name = "datasetDataRepositories", attributeNodes = {@NamedAttributeNode("dataRepository")}), - @NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")}) + @NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")}), + @NamedSubgraph(name = "profile", attributeNodes = {@NamedAttributeNode("label")}) }), @NamedEntityGraph( diff --git a/dmp-backend/queryable/src/main/java/eu/eudat/queryable/QueryableList.java b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/QueryableList.java index f11c15876..ac457b28a 100644 --- a/dmp-backend/queryable/src/main/java/eu/eudat/queryable/QueryableList.java +++ b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/QueryableList.java @@ -44,6 +44,8 @@ public interface QueryableList { QueryableList orderBy(OrderByPredicate predicate); + QueryableList groupBy(GroupByPredicate predicate); + QueryableList withHint(String hint); Long count(); diff --git a/dmp-backend/queryable/src/main/java/eu/eudat/queryable/collector/Collector.java b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/collector/Collector.java index 5de816bab..a6d710b11 100644 --- a/dmp-backend/queryable/src/main/java/eu/eudat/queryable/collector/Collector.java +++ b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/collector/Collector.java @@ -1,6 +1,9 @@ package eu.eudat.queryable.collector; +import com.fasterxml.jackson.databind.ObjectMapper; + import javax.persistence.Tuple; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -9,9 +12,13 @@ import java.util.stream.Collectors; public class Collector { public List buildFromTuple(List results, Map> groupedResults, List fields, String key) { - return groupedResults.keySet().stream() + return results.stream().map(tuple -> { + Map parsedResult = new HashMap<>(); + tuple.getElements().forEach(tupleElement -> parsedResult.put(tupleElement.getAlias(), tuple.get(tupleElement.getAlias()))); + return parsedResult; + }).collect(Collectors.toList());/*groupedResults.keySet().stream() .map(x -> buildOne(groupedResults.get(x), fields, key)) - .collect(Collectors.toList()); + .collect(Collectors.toList());*/ } private Map buildOne(List tuples, List fields, String key) { diff --git a/dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/hibernatequeryablelist/QueryableHibernateList.java b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/hibernatequeryablelist/QueryableHibernateList.java index 9c461f295..a347e3a2b 100644 --- a/dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/hibernatequeryablelist/QueryableHibernateList.java +++ b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/hibernatequeryablelist/QueryableHibernateList.java @@ -1,5 +1,6 @@ package eu.eudat.queryable.jpa.hibernatequeryablelist; +import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.queryable.QueryableList; import eu.eudat.queryable.collector.Collector; import eu.eudat.queryable.exceptions.NotSingleResultException; @@ -33,6 +34,7 @@ public class QueryableHibernateList implements QueryableLi private List> nestedPredicates = new LinkedList<>(); private boolean distinct = false; private List> orderings = new LinkedList<>(); + private List> groupings = new LinkedList<>(); private List fields = new LinkedList<>(); private Integer length; private Integer offset; @@ -150,6 +152,11 @@ public class QueryableHibernateList implements QueryableLi return this; } + public QueryableList groupBy(GroupByPredicate predicate) { + this.groupings.add(predicate); + return this; + } + public Long count() { CriteriaBuilder criteriaBuilder = this.manager.getCriteriaBuilder(); CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(Long.class); @@ -205,6 +212,14 @@ public class QueryableHibernateList implements QueryableLi return predicates.toArray(new Order[predicates.size()]); } + private Expression[] generateGroupPredicates(List> groupByPredicates, Root root) { + List predicates = new LinkedList<>(); + for (GroupByPredicate groupPredicate : groupByPredicates) { + predicates.add(groupPredicate.applyPredicate(this.manager.getCriteriaBuilder(), root)); + } + return predicates.toArray(new Expression[predicates.size()]); + } + public List toList() { CriteriaBuilder builder = this.manager.getCriteriaBuilder(); if (!this.fields.isEmpty()) this.query = builder.createTupleQuery(); @@ -212,22 +227,20 @@ public class QueryableHibernateList implements QueryableLi this.root = this.query.from(this.tClass); this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot)); if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root)); + if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root)); if (!this.fields.isEmpty()) this.selectFields(); if (distinct) this.query.distinct(true); - //if (!this.fields.isEmpty()) return this.toListWithFields(); + //if (!this.fields.isEmpty()) this.query.multiselect(this.parseFields(this.fields)); + ObjectMapper mapper = new ObjectMapper(); + if (!this.fields.isEmpty()) return this.toListWithFields().stream().map(m -> mapper.convertValue(m, this.tClass)).collect(Collectors.toList()); return this.toListWithOutFields(); } public List toListWithFields() { - CriteriaBuilder builder = this.manager.getCriteriaBuilder(); - if (!this.fields.isEmpty()) this.query = builder.createTupleQuery(); - else this.query = builder.createQuery(this.tClass); - this.root = this.query.from(this.tClass); - this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot)); - if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root)); - if (!this.fields.isEmpty()) this.selectFields(); - if (distinct) this.query.distinct(true); - List results = this.manager.createQuery(query).getResultList(); + TypedQuery typedQuery = this.manager.createQuery(this.query); + if (this.offset != null) typedQuery.setFirstResult(this.offset); + if (this.length != null) typedQuery.setMaxResults(this.length); + List results = typedQuery.getResultList(); Map> groupedResults = results.stream() .collect(Collectors.groupingBy(x -> x.get("id"))); return this.collector.buildFromTuple(results, groupedResults, this.fields, "id"); @@ -252,6 +265,7 @@ public class QueryableHibernateList implements QueryableLi this.root = this.query.from(this.tClass); this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot)); if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root)); + if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root)); if (!this.fields.isEmpty()) this.selectFields(); if (distinct) this.query.distinct(true); if (!this.fields.isEmpty()) return this.toListAsyncWithFields(); @@ -356,6 +370,8 @@ public class QueryableHibernateList implements QueryableLi if (!this.orderings.isEmpty()) criteriaQuery.orderBy(this.generateOrderPredicates(this.orderings, criteriaRoot)); + if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root)); + TypedQuery typedQuery = this.manager.createQuery(criteriaQuery); typedQuery.setHint("javax.persistence.fetchgraph", this.manager.getEntityGraph(this.hint)); return typedQuery; @@ -466,4 +482,11 @@ public class QueryableHibernateList implements QueryableLi .createQuery(update) .executeUpdate(); } + + private Path[] parseFields(List selectedFields) { + List paths = new ArrayList<>(); + selectedFields.forEach(s -> paths.add(root.get(s))); + + return paths.toArray(new Path[0]); + } } diff --git a/dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/predicates/GroupByPredicate.java b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/predicates/GroupByPredicate.java new file mode 100644 index 000000000..849878e4e --- /dev/null +++ b/dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/predicates/GroupByPredicate.java @@ -0,0 +1,10 @@ +package eu.eudat.queryable.jpa.predicates; + +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.Expression; +import javax.persistence.criteria.Root; + +public interface GroupByPredicate { + Expression applyPredicate(CriteriaBuilder builder, Root root); + +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index 5a070671b..3a03a12bc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -1171,8 +1171,13 @@ public class DataManagementPlanManager { .forEach(datasetEntity -> { Map properties = new HashMap<>(); if (datasetEntity.getProperties() != null) { - JSONObject jObject = new JSONObject(datasetEntity.getProperties()); - properties = jObject.toMap(); + try { + properties = new ObjectMapper().readValue(datasetEntity.getProperties(), LinkedHashMap.class); + } catch (IOException e) { + logger.error(e.getLocalizedMessage(), e); + } + /*JSONObject jObject = new JSONObject(datasetEntity.getProperties()); + properties = jObject.toMap();*/ } // Custom style for the Dataset title. diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index d3104b350..b295a54b5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -155,28 +155,33 @@ public class DatasetManager { datasetCriteria.setSize(datasetTableRequest.getLength()); List datasets; try { + long start = System.currentTimeMillis(); datasets = datasetRepository.exists() ? datasetRepository.queryIds(datasetCriteria) : null; + long end = System.currentTimeMillis(); + logger.info("Elastic took " + (end - start)/ 1000f + "s"); } catch (Exception ex) { logger.warn(ex.getMessage(), ex); datasets = null; } + long start = System.currentTimeMillis(); UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); // QueryableList items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); QueryableList items; if (datasets != null) { if (!datasets.isEmpty()) { - items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)); - List finalDatasets = datasets; - items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))); + //items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)); + final List datasetIds = datasets.stream().map(datasetE -> UUID.fromString(datasetE.getId())).distinct().collect(Collectors.toList()); + items = databaseRepository.getDatasetDao().filterFromElastic(datasetTableRequest.getCriteria(), datasetIds).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id")); + //items.where((builder, root) -> root.get("id").in(datasetIds)); } else { - items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); + items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id")); //items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()})); } } else { - items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); + items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id")); } List roles = new LinkedList<>(); QueryableList pagedItems; @@ -187,9 +192,6 @@ public class DatasetManager { } if (datasetTableRequest.getCriteria().getRole() != null) { roles.add(datasetTableRequest.getCriteria().getRole()); - } else { - roles.add(UserDMP.UserDMPRoles.USER.getValue()); - roles.add(UserDMP.UserDMPRoles.OWNER.getValue()); } authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles); pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest); @@ -209,10 +211,26 @@ public class DatasetManager { DataTableData dataTable = new DataTableData<>(); - dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList())); + + /* long countStart = System.currentTimeMillis(); + List pagedIds = pagedItems.toList().stream().map(Dataset::getId).collect(Collectors.toList()); + long countEnd = System.currentTimeMillis(); + + logger.info("Paged Ids took " + (countEnd - countStart)/ 1000f + "s");*/ dataTable.setTotalCount(authItems.count()); + // items = databaseRepository.getDatasetDao().asQueryable().where((builder, root) -> root.get("id").in(pagedIds)).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); + // PaginationManager.applyOrder(items, datasetTableRequest); + long dataStart = System.currentTimeMillis(); + dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList())); + + long dataEnd = System.currentTimeMillis(); + + logger.info("Data retrieval took " + (dataEnd - dataStart)/1000f + "s" ); + + long end = System.currentTimeMillis(); + logger.info("DB took " + (end - start)/ 1000f + "s"); //CompletableFuture.allOf(itemsFuture, countFuture).join(); return dataTable; } @@ -1122,10 +1140,10 @@ public class DatasetManager { @Transactional private DatasetListingModel mapModel(Dataset item) { - if (item.getProfile() == null) - return null; + /*if (item.getProfile() == null) + return null;*/ DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item); - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + /*DatasetProfileCriteria criteria = new DatasetProfileCriteria(); criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId())); List profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); boolean islast = false; @@ -1133,7 +1151,7 @@ public class DatasetManager { profiles = profiles.stream().sorted(Comparator.comparing(DatasetProfile::getVersion)).collect(Collectors.toList()); islast = profiles.get(0).getId().equals(item.getProfile().getId()); } - listingModel.setProfileLatestVersion(islast); + listingModel.setProfileLatestVersion(islast);*/ return listingModel; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DatasetListingModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DatasetListingModel.java index c0bb5a74f..3f4ffcb38 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DatasetListingModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DatasetListingModel.java @@ -196,14 +196,14 @@ public class DatasetListingModel implements DataModel new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList())); - this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList())); - this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList())); + //this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : ""; + // this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : ""; + //this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList())); + // this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList())); + //this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList())); if (entity.getFinalizedAt() == null && entity.getStatus() == Dataset.Status.FINALISED.getValue()) { this.finalizedAt = entity.getDmp().getFinalizedAt(); } else {