MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Optimize queries

This commit is contained in:
George Kalampokis 2021-09-30 18:02:53 +03:00
parent defac6afcc
commit 91729bab6d
10 changed files with 116 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDao.java
View File

@ -13,6 +13,8 @@ public interface DatasetDao extends DatabaseAccessLayer<Dataset, UUID> {
QueryableList<Dataset> getWithCriteria(DatasetCriteria criteria);
QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids);
QueryableList<Dataset> getAuthenticated(QueryableList<Dataset> query, UserInfo principal, List<Integer> roles);
Dataset isPublicDataset(UUID id);

11
dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDaoImpl.java
View File

@ -87,6 +87,16 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
return query;
}
public QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids) {
QueryableList<Dataset> query = getDatabaseService().getQueryable(Dataset.getHints(), Dataset.class);
query.where(((builder, root) -> root.get("id").in(ids)));
if (!criteria.getAllVersions())
query.initSubQuery(String.class).where((builder, root) -> builder.equal(root.get("dmp").get("version"), query.<String>subQueryMax((builder1, externalRoot, nestedRoot) -> builder1.and(builder1.equal(externalRoot.get("dmp").get("groupId"), nestedRoot.get("dmp").get("groupId")), builder1.notEqual(nestedRoot.get("dmp").get("status"), DMP.DMPStatus.DELETED.getValue())), Arrays.asList(new SelectionField(FieldSelectionType.COMPOSITE_FIELD, "dmp:version")), String.class)));
return query;
}
@Override
public Dataset createOrUpdate(Dataset item) {
return getDatabaseService().createOrUpdate(item, Dataset.class);
@ -121,6 +131,7 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
});
} else {
query.where((builder, root) -> builder.equal(root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT).join("user", JoinType.LEFT).get("id"), principal.getId()));
}
return query;
}

9
dmp-backend/data/src/main/java/eu/eudat/data/entities/Dataset.java
View File

@ -16,13 +16,14 @@ import java.util.stream.Collectors;
@NamedEntityGraphs({
@NamedEntityGraph(
name = "datasetListingModel",
attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),
@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode("profile"), @NamedAttributeNode("creator")},
attributeNodes = {/*@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),*/
@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode(value = "profile", subgraph = "profile"), @NamedAttributeNode("creator")},
subgraphs = {
@NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users"), @NamedAttributeNode("grant"), @NamedAttributeNode("organisations")}),
@NamedSubgraph(name = "datasetDataRepositories", attributeNodes = {@NamedAttributeNode("dataRepository")}),
@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")})
@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")}),
@NamedSubgraph(name = "profile", attributeNodes = {@NamedAttributeNode("label")})
}),
@NamedEntityGraph(

2
dmp-backend/queryable/src/main/java/eu/eudat/queryable/QueryableList.java
View File

@ -44,6 +44,8 @@ public interface QueryableList<T extends DataEntity> {
QueryableList<T> orderBy(OrderByPredicate<T> predicate);
QueryableList<T> groupBy(GroupByPredicate<T> predicate);
QueryableList<T> withHint(String hint);
Long count();

11
dmp-backend/queryable/src/main/java/eu/eudat/queryable/collector/Collector.java
View File

@ -1,6 +1,9 @@
package eu.eudat.queryable.collector;
import com.fasterxml.jackson.databind.ObjectMapper;
import javax.persistence.Tuple;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -9,9 +12,13 @@ import java.util.stream.Collectors;
public class Collector {
public List<Map> buildFromTuple(List<Tuple> results, Map<Object, List<Tuple>> groupedResults, List<String> fields, String key) {
return groupedResults.keySet().stream()
return results.stream().map(tuple -> {
Map<String, Object> parsedResult = new HashMap<>();
tuple.getElements().forEach(tupleElement -> parsedResult.put(tupleElement.getAlias(), tuple.get(tupleElement.getAlias())));
return parsedResult;
}).collect(Collectors.toList());/*groupedResults.keySet().stream()
.map(x -> buildOne(groupedResults.get(x), fields, key))
.collect(Collectors.toList());
.collect(Collectors.toList());*/
}
private Map buildOne(List<Tuple> tuples, List<String> fields, String key) {

43
dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/hibernatequeryablelist/QueryableHibernateList.java
View File

@ -1,5 +1,6 @@
package eu.eudat.queryable.jpa.hibernatequeryablelist;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.queryable.QueryableList;
import eu.eudat.queryable.collector.Collector;
import eu.eudat.queryable.exceptions.NotSingleResultException;
@ -33,6 +34,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
private List<NestedQuerySinglePredicate<T>> nestedPredicates = new LinkedList<>();
private boolean distinct = false;
private List<OrderByPredicate<T>> orderings = new LinkedList<>();
private List<GroupByPredicate<T>> groupings = new LinkedList<>();
private List<String> fields = new LinkedList<>();
private Integer length;
private Integer offset;
@ -150,6 +152,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
return this;
}
public QueryableList<T> groupBy(GroupByPredicate<T> predicate) {
this.groupings.add(predicate);
return this;
}
public Long count() {
CriteriaBuilder criteriaBuilder = this.manager.getCriteriaBuilder();
CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
@ -205,6 +212,14 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
return predicates.toArray(new Order[predicates.size()]);
}
private Expression[] generateGroupPredicates(List<GroupByPredicate<T>> groupByPredicates, Root<T> root) {
List<Expression> predicates = new LinkedList<>();
for (GroupByPredicate<T> groupPredicate : groupByPredicates) {
predicates.add(groupPredicate.applyPredicate(this.manager.getCriteriaBuilder(), root));
}
return predicates.toArray(new Expression[predicates.size()]);
}
public List<T> toList() {
CriteriaBuilder builder = this.manager.getCriteriaBuilder();
if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
@ -212,22 +227,20 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
this.root = this.query.from(this.tClass);
this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
if (!this.fields.isEmpty()) this.selectFields();
if (distinct) this.query.distinct(true);
//if (!this.fields.isEmpty()) return this.toListWithFields();
//if (!this.fields.isEmpty()) this.query.multiselect(this.parseFields(this.fields));
ObjectMapper mapper = new ObjectMapper();
if (!this.fields.isEmpty()) return this.toListWithFields().stream().map(m -> mapper.convertValue(m, this.tClass)).collect(Collectors.toList());
return this.toListWithOutFields();
}
public List<Map> toListWithFields() {
CriteriaBuilder builder = this.manager.getCriteriaBuilder();
if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
else this.query = builder.createQuery(this.tClass);
this.root = this.query.from(this.tClass);
this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
if (!this.fields.isEmpty()) this.selectFields();
if (distinct) this.query.distinct(true);
List<Tuple> results = this.manager.createQuery(query).getResultList();
TypedQuery<Tuple> typedQuery = this.manager.createQuery(this.query);
if (this.offset != null) typedQuery.setFirstResult(this.offset);
if (this.length != null) typedQuery.setMaxResults(this.length);
List<Tuple> results = typedQuery.getResultList();
Map<Object, List<Tuple>> groupedResults = results.stream()
.collect(Collectors.groupingBy(x -> x.get("id")));
return this.collector.buildFromTuple(results, groupedResults, this.fields, "id");
@ -252,6 +265,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
this.root = this.query.from(this.tClass);
this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
if (!this.fields.isEmpty()) this.selectFields();
if (distinct) this.query.distinct(true);
if (!this.fields.isEmpty()) return this.toListAsyncWithFields();
@ -356,6 +370,8 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
if (!this.orderings.isEmpty())
criteriaQuery.orderBy(this.generateOrderPredicates(this.orderings, criteriaRoot));
if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
TypedQuery<T> typedQuery = this.manager.createQuery(criteriaQuery);
typedQuery.setHint("javax.persistence.fetchgraph", this.manager.getEntityGraph(this.hint));
return typedQuery;
@ -466,4 +482,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
.createQuery(update)
.executeUpdate();
}
private Path[] parseFields(List<String> selectedFields) {
List<Path> paths = new ArrayList<>();
selectedFields.forEach(s -> paths.add(root.get(s)));
return paths.toArray(new Path[0]);
}
}

10
dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/predicates/GroupByPredicate.java Normal file
View File

@ -0,0 +1,10 @@
package eu.eudat.queryable.jpa.predicates;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.Root;
public interface GroupByPredicate<T> {
Expression<T> applyPredicate(CriteriaBuilder builder, Root<T> root);
}

9
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
View File

@ -1171,8 +1171,13 @@ public class DataManagementPlanManager {
.forEach(datasetEntity -> {
Map<String, Object> properties = new HashMap<>();
if (datasetEntity.getProperties() != null) {
JSONObject jObject = new JSONObject(datasetEntity.getProperties());
properties = jObject.toMap();
try {
properties = new ObjectMapper().readValue(datasetEntity.getProperties(), LinkedHashMap.class);
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
/*JSONObject jObject = new JSONObject(datasetEntity.getProperties());
properties = jObject.toMap();*/
}
// Custom style for the Dataset title.

44
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java
View File

@ -155,28 +155,33 @@ public class DatasetManager {
datasetCriteria.setSize(datasetTableRequest.getLength());
List<eu.eudat.elastic.entities.Dataset> datasets;
try {
long start = System.currentTimeMillis();
datasets = datasetRepository.exists() ?
datasetRepository.queryIds(datasetCriteria) : null;
long end = System.currentTimeMillis();
logger.info("Elastic took " + (end - start)/ 1000f + "s");
} catch (Exception ex) {
logger.warn(ex.getMessage(), ex);
datasets = null;
}
long start = System.currentTimeMillis();
UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
// QueryableList<eu.eudat.data.entities.Dataset> items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
QueryableList<eu.eudat.data.entities.Dataset> items;
if (datasets != null) {
if (!datasets.isEmpty()) {
items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
//items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
final List<UUID> datasetIds = datasets.stream().map(datasetE -> UUID.fromString(datasetE.getId())).distinct().collect(Collectors.toList());
items = databaseRepository.getDatasetDao().filterFromElastic(datasetTableRequest.getCriteria(), datasetIds).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
//items.where((builder, root) -> root.get("id").in(datasetIds));
} else {
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
//items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
}
} else {
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
}
List<Integer> roles = new LinkedList<>();
QueryableList<eu.eudat.data.entities.Dataset> pagedItems;
@ -187,9 +192,6 @@ public class DatasetManager {
}
if (datasetTableRequest.getCriteria().getRole() != null) {
roles.add(datasetTableRequest.getCriteria().getRole());
} else {
roles.add(UserDMP.UserDMPRoles.USER.getValue());
roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
}
authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles);
pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
@ -209,10 +211,26 @@ public class DatasetManager {
DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));
/* long countStart = System.currentTimeMillis();
List<UUID> pagedIds = pagedItems.toList().stream().map(Dataset::getId).collect(Collectors.toList());
long countEnd = System.currentTimeMillis();
logger.info("Paged Ids took " + (countEnd - countStart)/ 1000f + "s");*/
dataTable.setTotalCount(authItems.count());
// items = databaseRepository.getDatasetDao().asQueryable().where((builder, root) -> root.get("id").in(pagedIds)).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
// PaginationManager.applyOrder(items, datasetTableRequest);
long dataStart = System.currentTimeMillis();
dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));
long dataEnd = System.currentTimeMillis();
logger.info("Data retrieval took " + (dataEnd - dataStart)/1000f + "s" );
long end = System.currentTimeMillis();
logger.info("DB took " + (end - start)/ 1000f + "s");
//CompletableFuture.allOf(itemsFuture, countFuture).join();
return dataTable;
}
@ -1122,10 +1140,10 @@ public class DatasetManager {
@Transactional
private DatasetListingModel mapModel(Dataset item) {
if (item.getProfile() == null)
return null;
/*if (item.getProfile() == null)
return null;*/
DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item);
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
/*DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId()));
List<DatasetProfile> profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
boolean islast = false;
@ -1133,7 +1151,7 @@ public class DatasetManager {
profiles = profiles.stream().sorted(Comparator.comparing(DatasetProfile::getVersion)).collect(Collectors.toList());
islast = profiles.get(0).getId().equals(item.getProfile().getId());
}
listingModel.setProfileLatestVersion(islast);
listingModel.setProfileLatestVersion(islast);*/
return listingModel;
}

12
dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DatasetListingModel.java
View File

@ -196,14 +196,14 @@ public class DatasetListingModel implements DataModel<Dataset, DatasetListingMod
this.grant = entity.getDmp() != null ? entity.getDmp().getGrant().getLabel() : "";
this.dmp = entity.getDmp() != null ? entity.getDmp().getLabel() : "";
this.dmpId = entity.getDmp() != null ? entity.getDmp().getId().toString() : "";
this.profile = entity.getProfile() != null ? new DatasetProfileOverviewModel().fromDataModel(entity.getProfile()) : null;
//this.profile = entity.getProfile() != null ? new DatasetProfileOverviewModel().fromDataModel(entity.getProfile()) : null;
this.description = entity.getDescription();
this.status = entity.getStatus();
this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
//this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
// this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
//this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
// this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
//this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
if (entity.getFinalizedAt() == null && entity.getStatus() == Dataset.Status.FINALISED.getValue()) {
this.finalizedAt = entity.getDmp().getFinalizedAt();
} else {