Merge commit '2cb985c23c4ceda99efc78bcc8e77b3eceeb0bcf'

# Conflicts:
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/criteria/DatasetCriteria.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dmp.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DmpRepository.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/criteria/DmpCriteriaMapper.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java
#	dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DataManagementPlanListingModel.java
This commit is contained in:
Diamantis Tziotzios 2021-12-09 12:18:39 +02:00
commit 7a8375b988
346 changed files with 9457 additions and 2680 deletions

6
.gitignore vendored
View File

@ -40,3 +40,9 @@ dmp-frontend/.vscode/
dmp-frontend/package-lock.json
dmp-backend/logging/target/
ELK.Docker/shared/data-elk/
# Eclipse
.project
.settings/
bin/
*.classpath

View File

@ -1,4 +1,4 @@
FROM maven:3-jdk-8-alpine AS MAVEN_BUILD
FROM maven:3-openjdk-11 AS MAVEN_BUILD
COPY pom.xml /build/
COPY data /build/data/
@ -9,7 +9,7 @@ COPY web /build/web/
WORKDIR /build/
RUN mvn package
FROM openjdk:8-jre-alpine
FROM adoptopenjdk/openjdk11:alpine-jre
WORKDIR /app
COPY --from=MAVEN_BUILD /build/web/target/web-1.0-SNAPSHOT.jar /app/app.jar
ENTRYPOINT ["java", "-Djava.security.egd=file:/dev/./urandom", "-Dspring.profiles.active=${PROF}", "-Dspring.config.location=/files/config/", "-jar","/app/app.jar"]
ENTRYPOINT ["java", "-Djava.security.egd=file:/dev/./urandom", "-Dspring.profiles.active=${PROF}", "-Dspring.config.additional-location=/files/config/", "-jar","/app/app.jar"]

View File

@ -2,6 +2,8 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.DatasetProfile;
import java.util.Date;
import java.util.List;
import java.util.UUID;
@ -35,6 +37,8 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
private boolean finalized;
private Integer status;
private Integer role;
private List<UUID> ids;
private Date periodStart;
public boolean getAllVersions() { return allVersions; }
public void setAllVersions(boolean allVersions) { this.allVersions = allVersions; }
@ -78,4 +82,20 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
public void setRole(Integer role) {
this.role = role;
}
public List<UUID> getIds() {
return ids;
}
public void setIds(List<UUID> ids) {
this.ids = ids;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.Funder;
import java.util.Date;
public class FunderCriteria extends Criteria<Funder> {
private String reference;
private String exactReference;
private Date periodStart;
public String getReference() {
return reference;
@ -20,4 +23,12 @@ public class FunderCriteria extends Criteria<Funder> {
public void setExactReference(String exactReference) {
this.exactReference = exactReference;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -5,6 +5,7 @@ import eu.eudat.data.entities.Organisation;
public class OrganisationCriteria extends Criteria<Organisation> {
private String labelLike;
private Boolean isPublic;
private boolean isActive;
public String getLabelLike() {
return labelLike;
@ -19,4 +20,12 @@ public class OrganisationCriteria extends Criteria<Organisation> {
public void setPublic(Boolean aPublic) {
isPublic = aPublic;
}
public boolean isActive() {
return isActive;
}
public void setActive(boolean active) {
isActive = active;
}
}

View File

@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.Project;
import java.util.Date;
public class ProjectCriteria extends Criteria<Project> {
private String reference;
private String exactReference;
private Date periodStart;
public String getReference() {
return reference;
@ -20,4 +23,12 @@ public class ProjectCriteria extends Criteria<Project> {
public void setExactReference(String exactReference) {
this.exactReference = exactReference;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.Researcher;
import java.util.Date;
public class ResearcherCriteria extends Criteria<Researcher> {
private String name;
private String reference;
private Date periodStart;
public String getName() {
return name;
@ -21,4 +24,12 @@ public class ResearcherCriteria extends Criteria<Researcher> {
public void setReference(String reference) {
this.reference = reference;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -7,6 +7,7 @@ import eu.eudat.queryable.queryableentity.DataEntity;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.Set;
@ -28,6 +29,7 @@ public class DatabaseService<T extends DataEntity> {
return this.databaseCtx.getQueryable(tClass);
}
@Transactional
public T createOrUpdate(T item, Class<T> tClass) {
return this.databaseCtx.createOrUpdate(item, tClass);
}

View File

@ -16,6 +16,7 @@ import org.springframework.stereotype.Component;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.transaction.Transactional;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
@ -106,6 +107,7 @@ public class DMPDaoImpl extends DatabaseAccess<DMP> implements DMPDao {
}
@Override
@Transactional
public DMP createOrUpdate(DMP item) {
return this.getDatabaseService().createOrUpdate(item, DMP.class);
}

View File

@ -13,6 +13,8 @@ public interface DatasetDao extends DatabaseAccessLayer<Dataset, UUID> {
QueryableList<Dataset> getWithCriteria(DatasetCriteria criteria);
QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids);
QueryableList<Dataset> getAuthenticated(QueryableList<Dataset> query, UserInfo principal, List<Integer> roles);
Dataset isPublicDataset(UUID id);

View File

@ -87,6 +87,16 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
return query;
}
public QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids) {
QueryableList<Dataset> query = getDatabaseService().getQueryable(Dataset.getHints(), Dataset.class);
query.where(((builder, root) -> root.get("id").in(ids)));
if (!criteria.getAllVersions())
query.initSubQuery(String.class).where((builder, root) -> builder.equal(root.get("dmp").get("version"), query.<String>subQueryMax((builder1, externalRoot, nestedRoot) -> builder1.and(builder1.equal(externalRoot.get("dmp").get("groupId"), nestedRoot.get("dmp").get("groupId")), builder1.notEqual(nestedRoot.get("dmp").get("status"), DMP.DMPStatus.DELETED.getValue())), Arrays.asList(new SelectionField(FieldSelectionType.COMPOSITE_FIELD, "dmp:version")), String.class)));
return query;
}
@Override
public Dataset createOrUpdate(Dataset item) {
return getDatabaseService().createOrUpdate(item, Dataset.class);
@ -121,6 +131,7 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
});
} else {
query.where((builder, root) -> builder.equal(root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT).join("user", JoinType.LEFT).get("id"), principal.getId()));
}
return query;
}

View File

@ -58,11 +58,16 @@ public class DatasetProfileDaoImpl extends DatabaseAccess<DatasetProfile> implem
if (criteria.getStatus() != null) {
query.where(((builder, root) -> builder.equal(root.get("status"), criteria.getStatus())));
}
if (criteria.getIds() != null) {
query.where(((builder, root) -> root.get("id").in(criteria.getIds())));
}
if (criteria.getFinalized()) {
query.where(((builder, root) -> builder.equal(root.get("status"), DatasetProfile.Status.FINALIZED.getValue())));
} else {
query.where(((builder, root) -> builder.notEqual(root.get("status"), DatasetProfile.Status.DELETED.getValue())));
}
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
return query;
}

View File

@ -31,6 +31,8 @@ public class FunderDaoImpl extends DatabaseAccess<Funder> implements FunderDao {
query.where((builder, root) -> builder.like(builder.upper(root.get("reference")), "%" + criteria.getReference().toUpperCase() + "%"));
if (criteria.getExactReference() != null)
query.where((builder, root) -> builder.like(builder.upper(root.get("reference")), criteria.getExactReference().toUpperCase()));
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
query.where((builder, root) -> builder.notEqual(root.get("status"), Funder.Status.DELETED.getValue()));
return query;
}

View File

@ -38,6 +38,11 @@ public class OrganisationDaoImpl extends DatabaseAccess<Organisation> implements
query.where((builder, root) -> builder.equal(root.join("dmps", JoinType.LEFT).get("status"), DMP.DMPStatus.FINALISED.getValue()));
}
}
if (criteria.isActive()) {
query.where((builder, root) -> builder.notEqual(root.join("dmps").get("status"), DMP.DMPStatus.DELETED.getValue())).distinct();
}
return query;
}

View File

@ -30,6 +30,8 @@ public class ProjectDaoImpl extends DatabaseAccess<Project> implements ProjectDa
query.where((builder, root) -> builder.like(root.get("reference"), "%" + criteria.getReference() + "%"));
if (criteria.getExactReference() != null)
query.where((builder, root) -> builder.like(root.get("reference"), criteria.getExactReference()));
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("startdate"), criteria.getPeriodStart()));
query.where((builder, root) -> builder.notEqual(root.get("status"), Project.Status.DELETED.getValue()));
return query;
}

View File

@ -29,6 +29,8 @@ public class ResearcherDaoImpl extends DatabaseAccess<Researcher> implements Res
query.where((builder, root) ->builder.or(builder.like(builder.lower(root.get("label")), "%" + criteria.getName().toLowerCase() + "%")));
if (criteria.getReference() != null && !criteria.getReference().isEmpty())
query.where((builder, root) ->builder.or(builder.like(root.get("reference"), criteria.getReference())));
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
return query;
}

View File

@ -16,13 +16,14 @@ import java.util.stream.Collectors;
@NamedEntityGraphs({
@NamedEntityGraph(
name = "datasetListingModel",
attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),
@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode("profile"), @NamedAttributeNode("creator")},
attributeNodes = {/*@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),*/
@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode(value = "profile", subgraph = "profile"), @NamedAttributeNode("creator")},
subgraphs = {
@NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users"), @NamedAttributeNode("grant"), @NamedAttributeNode("organisations")}),
@NamedSubgraph(name = "datasetDataRepositories", attributeNodes = {@NamedAttributeNode("dataRepository")}),
@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")})
@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")}),
@NamedSubgraph(name = "profile", attributeNodes = {@NamedAttributeNode("label")})
}),
@NamedEntityGraph(

View File

@ -106,11 +106,11 @@ public class Grant implements DataEntity<Grant, UUID> {
@Column(name = "\"Definition\"", columnDefinition = "xml", nullable = true)
private String definition;
@Column(name = "\"StartDate\"", nullable = false)
@Column(name = "\"StartDate\"", nullable = true)
@Convert(converter = DateToUTCConverter.class)
private Date startdate = null;
@Column(name = "\"EndDate\"", nullable = false)
@Column(name = "\"EndDate\"", nullable = true)
@Convert(converter = DateToUTCConverter.class)
private Date enddate = null;

View File

@ -6,22 +6,22 @@ import java.util.List;
public class ColumnOrderings {
private String[] fields;
private List<String> fields;
public String[] getFields() {
public List<String> getFields() {
return fields;
}
public void setFields(String[] fields) {
public void setFields(List<String> fields) {
this.fields = fields;
}
public Ordering[] getFieldOrderings() {
public List<Ordering> getFieldOrderings() {
List<Ordering> orderings = new LinkedList<>();
for (String field : fields) {
orderings.add(this.orderingFromString(field));
}
return orderings.toArray(new Ordering[orderings.size()]);
return orderings;
}
private Ordering orderingFromString(String field) {

View File

@ -18,6 +18,7 @@ public class DatasetCriteria extends Criteria {
private List<UUID> collaborators;
private Boolean allowAllVersions;
private List<String> organiztions;
private Boolean hasTags;
private List<Tag> tags;
private boolean isPublic;
private Short grantStatus;
@ -144,4 +145,12 @@ public class DatasetCriteria extends Criteria {
public void setSortCriteria(List<SortCriteria> sortCriteria) {
this.sortCriteria = sortCriteria;
}
public Boolean getHasTags() {
return hasTags;
}
public void setHasTags(Boolean hasTags) {
this.hasTags = hasTags;
}
}

View File

@ -283,38 +283,46 @@ public class Dataset implements ElasticEntity<Dataset> {
@Override
public Dataset fromElasticEntity(Map<String, Object> fields) {
if (fields != null) {
this.id = (String) fields.get("id");
if (fields.get("tags") != null) {
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
if (fields.size() == 1) {
if (fields.containsKey("id")) {
this.id = (String) fields.get("id");
} else if (fields.containsKey("tags")) {
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
}else if (fields.size() > 1) {
this.id = (String) fields.get("id");
if (fields.get("tags") != null) {
this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
this.label = (String) fields.get("label");
this.description = (String) fields.get("description");
this.template = UUID.fromString((String) fields.get("template"));
this.status = Short.valueOf((String) fields.get("status"));
this.dmp = UUID.fromString((String) fields.get("dmp"));
this.group = UUID.fromString((String) fields.get("group"));
this.grant = UUID.fromString((String) fields.get("grant"));
if (fields.get("created") != null)
this.created = Date.from(Instant.parse((String) fields.get("created")));
if (fields.get("modified") != null)
this.modified = Date.from(Instant.parse((String) fields.get("modified")));
if (fields.get("finalizedAt") != null)
this.finalizedAt = Date.from(Instant.parse((String) fields.get("finalizedAt")));
if (fields.get("collaborators") != null) {
this.collaborators = ((List<HashMap>) fields.get("collaborators")).stream().map(hashMap -> new Collaborator().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
this.lastVersion = Boolean.parseBoolean((String) fields.get("lastVersion"));
this.lastPublicVersion = Boolean.parseBoolean((String) fields.get("lastPublicVersion"));
if (fields.get("organizations") != null) {
this.organizations = ((List<HashMap>) fields.get("organizations")).stream().map(hashMap -> new Organization().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
if (fields.get("public") != null) {
this.isPublic = Boolean.valueOf((String) fields.get("public"));
}
if (fields.get("grantStatus") != null) {
this.grantStatus = Short.valueOf((String) fields.get("grantStatus"));
}
this.formData = (String) fields.get("formData");
}
this.label = (String) fields.get("label");
this.description = (String) fields.get("description");
this.template = UUID.fromString((String) fields.get("template"));
this.status = Short.valueOf((String) fields.get("status"));
this.dmp = UUID.fromString((String) fields.get("dmp"));
this.group = UUID.fromString((String) fields.get("group"));
this.grant = UUID.fromString((String) fields.get("grant"));
if (fields.get("created") != null)
this.created = Date.from(Instant.parse((String) fields.get("created")));
if (fields.get("modified") != null)
this.modified = Date.from(Instant.parse((String) fields.get("modified")));
if (fields.get("finalizedAt") != null)
this.finalizedAt = Date.from(Instant.parse((String) fields.get("finalizedAt")));
if (fields.get("collaborators") != null) {
this.collaborators = ((List<HashMap>) fields.get("collaborators")).stream().map(hashMap -> new Collaborator().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
this.lastVersion = Boolean.parseBoolean((String) fields.get("lastVersion"));
this.lastPublicVersion = Boolean.parseBoolean((String) fields.get("lastPublicVersion"));
if (fields.get("organizations") != null) {
this.organizations = ((List<HashMap>) fields.get("organizations")).stream().map(hashMap -> new Organization().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
if (fields.get("public") != null) {
this.isPublic = Boolean.valueOf((String) fields.get("public"));
}
if (fields.get("grantStatus") != null) {
this.grantStatus = Short.valueOf((String) fields.get("grantStatus"));
}
this.formData = (String) fields.get("formData");
}
return this;
}

View File

@ -5,6 +5,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
@ -286,31 +287,47 @@ public class Dmp implements ElasticEntity<Dmp> {
@Override
public Dmp fromElasticEntity(Map<String, Object> fields) {
if (fields == null || fields.isEmpty()) {
return null;
}
this.id = UUID.fromString((String) fields.get(MapKey.ID.getName()));
this.label = (String) fields.get(MapKey.LABEL.getName());
this.description = (String) fields.get(MapKey.DESCRIPTION.getName());
if (fields.get(MapKey.GROUPID.getName()) != null) {
this.groupId = UUID.fromString((String) fields.get(MapKey.GROUPID.getName()));
}
this.status = Short.valueOf(fields.get(MapKey.STATUS.getName()).toString());
if (fields.get(MapKey.TEMPLATES.getName()) != null) {
this.templates = ((List<HashMap<String, Object>>) fields.get(MapKey.TEMPLATES.getName())).stream().map(hashMap -> new DatasetTempalate().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
if (fields.get(MapKey.COLLABORATORS.getName()) != null) {
this.collaborators = ((List<HashMap<String, Object>>) fields.get(MapKey.COLLABORATORS.getName())).stream().map(map -> new Collaborator().fromElasticEntity(map)).collect(Collectors.toList());
}
if (fields.get(MapKey.ORGANIZATIONS.getName()) != null) {
this.organizations = ((List<HashMap<String, Object>>) fields.get(MapKey.ORGANIZATIONS.getName())).stream().map(map -> new Organization().fromElasticEntity(map)).collect(Collectors.toList());
}
this.lastVersion = (Boolean) fields.get(MapKey.LASTVERSION.getName());
this.lastPublicVersion = (Boolean) fields.get(MapKey.LASTPUBLICVERSION.getName());
this.isPublic = (Boolean) fields.get(MapKey.ISPUBLIC.getName());
if (fields.get(MapKey.DATASETS.getName()) != null) {
this.datasets = ((List<HashMap<String, Object>>) fields.get(MapKey.DATASETS.getName())).stream().map(map -> new Dataset().fromElasticEntity(map)).collect(Collectors.toList());
}
this.grant = UUID.fromString((String) fields.get(MapKey.GRANT.getName()));
if (fields.get(MapKey.GRANTSTATUS.getName()) != null) {
this.grantStatus = Short.valueOf(fields.get(MapKey.GRANTSTATUS.getName()).toString());
if (fields.size() > 1) {
this.label = (String) fields.get(MapKey.LABEL.getName());
this.description = (String) fields.get(MapKey.DESCRIPTION.getName());
if (fields.get(MapKey.GROUPID.getName()) != null) {
this.groupId = UUID.fromString((String) fields.get(MapKey.GROUPID.getName()));
}
this.status = Short.valueOf(fields.get(MapKey.STATUS.getName()).toString());
if (fields.get(MapKey.TEMPLATES.getName()) != null) {
this.templates = ((List<HashMap<String, Object>>) fields.get(MapKey.TEMPLATES.getName())).stream().map(hashMap -> new DatasetTempalate().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
if (fields.get(MapKey.COLLABORATORS.getName()) != null) {
this.collaborators = ((List<HashMap<String, Object>>) fields.get(MapKey.COLLABORATORS.getName())).stream().map(map -> new Collaborator().fromElasticEntity(map)).collect(Collectors.toList());
}
if (fields.get(MapKey.ORGANIZATIONS.getName()) != null) {
this.organizations = ((List<HashMap<String, Object>>) fields.get(MapKey.ORGANIZATIONS.getName())).stream().map(map -> new Organization().fromElasticEntity(map)).collect(Collectors.toList());
}
this.lastVersion = (Boolean) fields.get(MapKey.LASTVERSION.getName());
this.lastPublicVersion = (Boolean) fields.get(MapKey.LASTPUBLICVERSION.getName());
this.isPublic = (Boolean) fields.get(MapKey.ISPUBLIC.getName());
if (fields.get(MapKey.DATASETS.getName()) != null) {
this.datasets = ((List<HashMap<String, Object>>) fields.get(MapKey.DATASETS.getName())).stream().map(map -> new Dataset().fromElasticEntity(map)).collect(Collectors.toList());
}
this.grant = UUID.fromString((String) fields.get(MapKey.GRANT.getName()));
if (fields.get(MapKey.GRANTSTATUS.getName()) != null) {
this.grantStatus = Short.valueOf(fields.get(MapKey.GRANTSTATUS.getName()).toString());
}
this.created = Date.from(Instant.parse(fields.get(MapKey.CREATED.getName()).toString()));
this.modified = Date.from(Instant.parse(fields.get(MapKey.MODIFIED.getName()).toString()));
if (fields.get(MapKey.FINALIZEDAT.getName()) != null) {
this.finalizedAt = Date.from(Instant.parse(fields.get(MapKey.FINALIZEDAT.getName()).toString()));
}
if (fields.get(MapKey.PUBLISHEDAT.getName()) != null) {
this.publishedAt = Date.from(Instant.parse(fields.get(MapKey.PUBLISHEDAT.getName()).toString()));
}
if (fields.get(MapKey.DOI.getName()) != null) {
this.doi = fields.get(MapKey.DOI.getName()).toString();
}
}
return this;
}

View File

@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
@ -54,24 +55,26 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
if (this.getClient() != null) {
XContentBuilder builder = XContentFactory.jsonBuilder();
Dmp dmp = this.dmpRepository.findDocument(entity.getDmp().toString());
boolean found = false;
if (dmp.getDatasets() != null && !dmp.getDatasets().isEmpty()) {
for (int i = 0; i < dmp.getDatasets().size(); i++) {
if (dmp.getDatasets().get(i).getId().equals(entity.getId())) {
dmp.getDatasets().set(i, entity);
found = true;
break;
if (dmp != null) {
boolean found = false;
if (dmp.getDatasets() != null && !dmp.getDatasets().isEmpty()) {
for (int i = 0; i < dmp.getDatasets().size(); i++) {
if (dmp.getDatasets().get(i).getId().equals(entity.getId())) {
dmp.getDatasets().set(i, entity);
found = true;
break;
}
}
}
}
if (!found) {
if (dmp.getDatasets() == null) {
dmp.setDatasets(new ArrayList<>());
if (!found) {
if (dmp.getDatasets() == null) {
dmp.setDatasets(new ArrayList<>());
}
dmp.getDatasets().add(entity);
}
dmp.getDatasets().add(entity);
IndexRequest request = new IndexRequest(this.environment.getProperty("elasticsearch.index")).id(dmp.getId().toString()).source(dmp.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
this.getClient().index(request, RequestOptions.DEFAULT);
}
IndexRequest request = new IndexRequest(this.environment.getProperty("elasticsearch.index")).id(dmp.getId().toString()).source(dmp.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
this.getClient().index(request, RequestOptions.DEFAULT);
return entity;
}
return null;
@ -122,74 +125,15 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
searchSourceBuilder.size(count.intValue());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
List<SortBuilder> sortBuilders = new ArrayList<>();
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastPublicVersion", "true"));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).allowLeadingWildcard(true).fields(Stream.of(new Object[][]{
{"datasets.label", 1.0f},
{"datasets.description", 1.0f},
{"datasets.formData", 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
if (criteria.getDatasetTemplates() != null && criteria.getDatasetTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.template", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", criteria.getStatus().toString()));
}
if (criteria.getDmps() != null && criteria.getDmps().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.dmp", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.group", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.grant", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.grantStatus", criteria.getGrantStatus().toString()));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isPublic()) {
if (criteria.getAllowAllVersions() != null && !criteria.getAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastVersion", "true"));
}
}
if (criteria.getOrganiztions() != null && criteria.getOrganiztions().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.organizations.id", criteria.getOrganiztions()));
}
if (criteria.getTags() != null && criteria.getTags().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
criteria.getSortCriteria().forEach(sortCriteria -> {
switch(sortCriteria.getColumnType()) {
case COLUMN:
sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
sortBuilders.add(SortBuilders.fieldSort("datasets." + sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
break;
case JOIN_COLUMN:
List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
@ -202,8 +146,11 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
}
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).size(criteria.getSize());
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.tags"}, null)));
searchSourceBuilder.query(nestedQueryBuilder)/*.from(criteria.getOffset())*/.fetchSource("datasets.tags", null);
/*if (criteria.getSize() > 0) {
searchSourceBuilder.size(criteria.getSize());
}*/
sortBuilders.forEach(searchSourceBuilder::sort);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
@ -215,6 +162,161 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
return null;
}
public List<Dataset> queryIds(DatasetCriteria criteria) throws IOException {
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
/*CountRequest countRequest = new CountRequest("dmps").routing("datasets").routing("id");
countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT);
Long count = countResponse.getCount();*/
SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
countSourceBuilder.aggregation(nestedAggregationBuilder);
countRequest.source(countSourceBuilder);
SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
Long count = ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
searchSourceBuilder.size(count.intValue());
List<SortBuilder> sortBuilders = new ArrayList<>();
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
criteria.getSortCriteria().forEach(sortCriteria -> {
switch(sortCriteria.getColumnType()) {
case COLUMN:
sortBuilders.add(SortBuilders.fieldSort("datasets." + sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
break;
case JOIN_COLUMN:
List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
});
break;
}
});
}
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.id"}, null)));
searchSourceBuilder.query(nestedQueryBuilder)/*.from(criteria.getOffset()).size(criteria.getSize())*/.fetchSource("datasets.id", null);
sortBuilders.forEach(searchSourceBuilder::sort);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
.map(SearchHits::getHits).flatMap(Arrays::stream)
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
}
return null;
}
@Override
public Long count(DatasetCriteria criteria) throws IOException {
if (this.getClient() != null) {
//CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", boolQuery);
nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
countSourceBuilder.aggregation(nestedAggregationBuilder);
countRequest.source(countSourceBuilder);
SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
return ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
/*NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder());
countRequest.query(nestedQueryBuilder);
CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
return response.getCount();*/
}
return null;
}
private BoolQueryBuilder createBoolQuery(DatasetCriteria criteria) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastPublicVersion", "true"));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).allowLeadingWildcard(true).fields(Stream.of(new Object[][]{
{"datasets.label", 1.0f},
{"datasets.description", 1.0f},
{"datasets.formData", 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getDatasetTemplates() != null && criteria.getDatasetTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.template", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", criteria.getStatus().toString()));
}
if (criteria.getDmps() != null && criteria.getDmps().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.dmp", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.group", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.grant", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.grantStatus", criteria.getGrantStatus().toString()));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isPublic()) {
if (criteria.getAllowAllVersions() != null && !criteria.getAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastVersion", "true"));
}
}
if (criteria.getOrganiztions() != null && criteria.getOrganiztions().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.organizations.id", criteria.getOrganiztions()));
}
if (criteria.getTags() != null && criteria.getTags().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
}
if (criteria.getHasTags() != null) {
boolQuery = criteria.getHasTags() == true ? boolQuery.should(QueryBuilders.existsQuery("datasets.tags.id")) : boolQuery.mustNot(QueryBuilders.existsQuery("datasets.tags.id"));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
return boolQuery;
}
@Override
public boolean exists() throws IOException {
if (this.getClient() != null) {

View File

@ -99,60 +99,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
searchSourceBuilder.size(count.intValue());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
List<SortBuilder> sortBuilders = new ArrayList<>();
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{Dmp.MapKey.LABEL.getName(), 1.0f},
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getRoles() != null && criteria.getRoles().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".role.keyword", criteria.getRoles()));
}
if (!criteria.isAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
}
if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
criteria.getSortCriteria().forEach(sortCriteria -> {
@ -170,8 +118,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
});
}
searchSourceBuilder.query(boolQuery).from(criteria.getOffset());
if (criteria.getSize() != null) {
searchSourceBuilder.query(boolQuery).from(criteria.getOffset()).fetchSource("id", null);
if (criteria.getSize() != null && criteria.getSize() > 0) {
searchSourceBuilder.size(criteria.getSize());
}
sortBuilders.forEach(searchSourceBuilder::sort);
@ -182,59 +130,12 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
return null;
}
@Override
public Long count(DmpCriteria criteria) throws IOException {
if (this.getClient() != null) {
CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{Dmp.MapKey.LABEL.getName(), 1.0f},
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
}
if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
BoolQueryBuilder boolQuery = createBoolQuery(criteria);
countRequest.query(boolQuery);
CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
@ -243,6 +144,59 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
return null;
}
private BoolQueryBuilder createBoolQuery(DmpCriteria criteria) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{Dmp.MapKey.LABEL.getName(), 1.0f},
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().filter(Objects::nonNull).map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
}
if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
return boolQuery;
}
public boolean createIndex() {
try {
if (!this.exists()) {

View File

@ -18,6 +18,8 @@ public interface Repository<ET extends ElasticEntity, C extends Criteria> {
List<ET> query(C criteria) throws ExecutionException, InterruptedException, IOException;
Long count(C criteria) throws ExecutionException, InterruptedException, IOException;
boolean exists() throws IOException;
void clear() throws IOException;

View File

@ -10,7 +10,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.9.RELEASE</version>
<version>2.5.2</version>
</parent>
<modules>
@ -27,25 +27,26 @@
<java.version>1.8</java.version>
<dmp-backend-commons.version>0.0.1-SNAPSHOT</dmp-backend-commons.version>
<org.springframework.version>4.3.8.RELEASE</org.springframework.version>
<org.springframework.version>5.3.8</org.springframework.version>
<!-- <org.springframeweu.eudat.logic.securityrity.version>3.2.10.RELEASE</org.springframeweu.eudat.logic.securityrity.version> -->
<org.springframework.security.version>4.2.3.RELEASE</org.springframework.security.version>
<com.sun.jersey.version>1.19.1</com.sun.jersey.version>
<org.springframework.security.version>5.3.10.RELEASE</org.springframework.security.version>
<!--<com.sun.jersey.version>1.19.1</com.sun.jersey.version>-->
<!--
<org.apache.tomcat.tomcat-jdbc.version>7.0.35</org.apache.tomcat.tomcat-jdbc.version>
<com.fasterxml.jackson>2.8.4</com.fasterxml.jackson>
-->
<!--<com.fasterxml.jackson>2.9.0</com.fasterxml.jackson>-->
<hibernate.version>5.2.11.Final</hibernate.version>
<hibernate.version>5.5.3.Final</hibernate.version>
<commons-codec.version>1.9</commons-codec.version>
<org.junit.version>4.11</org.junit.version>
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.12</slf4j.version>
<jetty.version>9.0.7.v20131107
</jetty.version> <!-- Adapt this to a version found on http://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-maven-plugin/ -->
<!--<jetty.version>11.0.5
</jetty.version>--> <!-- Adapt this to a version found on http://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-maven-plugin/ -->
<logback.version>1.2.3</logback.version>
<javax.inject.version>1</javax.inject.version>
<javax.servlet.servlet-api.version>3.0.1</javax.servlet.servlet-api.version>
<docker.image.prefix>ikalyvas</docker.image.prefix>
<!-- <javax.inject.version>1</javax.inject.version>-->
<!--<javax.servlet.servlet-api.version>3.0.1</javax.servlet.servlet-api.version>-->
</properties>
<dependencies>
@ -58,7 +59,7 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
<version>4.0.0.RELEASE</version>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@ -139,7 +140,7 @@
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
<version>2.9.3</version>
<version>2.12.3</version>
</dependency>
@ -147,13 +148,13 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.9.4</version>
<version>2.12.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackeu.eudat.corecore/jackson-databind -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.4</version>
<version>2.12.3</version>
</dependency>
<!-- g/a spring -->
@ -186,6 +187,12 @@
<artifactId>fop</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.14.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/fr.opensagres.xdocreport/org.apache.poi.xwpf.converter.pdf -->
<dependency>
<groupId>fr.opensagres.xdocreport</groupId>
@ -257,36 +264,53 @@
<version>1.3.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/javax.validation/validation-api -->
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.1.Final</version>
</dependency>
<!-- The client -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Hotspot JVM metrics-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_hotspot</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Exposition HTTPServer-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Pushgateway exposition-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_pushgateway</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Pushgateway exposition-->
<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_spring_boot</artifactId>
<version>0.10.0</version>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<version>1.7.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<!-- Micormeter core dependecy -->
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
</dependency>
</dependencies>

View File

@ -44,6 +44,8 @@ public interface QueryableList<T extends DataEntity> {
QueryableList<T> orderBy(OrderByPredicate<T> predicate);
QueryableList<T> groupBy(GroupByPredicate<T> predicate);
QueryableList<T> withHint(String hint);
Long count();

View File

@ -1,6 +1,9 @@
package eu.eudat.queryable.collector;
import com.fasterxml.jackson.databind.ObjectMapper;
import javax.persistence.Tuple;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -9,9 +12,13 @@ import java.util.stream.Collectors;
public class Collector {
public List<Map> buildFromTuple(List<Tuple> results, Map<Object, List<Tuple>> groupedResults, List<String> fields, String key) {
return groupedResults.keySet().stream()
return results.stream().map(tuple -> {
Map<String, Object> parsedResult = new HashMap<>();
tuple.getElements().forEach(tupleElement -> parsedResult.put(tupleElement.getAlias(), tuple.get(tupleElement.getAlias())));
return parsedResult;
}).collect(Collectors.toList());/*groupedResults.keySet().stream()
.map(x -> buildOne(groupedResults.get(x), fields, key))
.collect(Collectors.toList());
.collect(Collectors.toList());*/
}
private Map buildOne(List<Tuple> tuples, List<String> fields, String key) {

View File

@ -1,5 +1,6 @@
package eu.eudat.queryable.jpa.hibernatequeryablelist;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.queryable.QueryableList;
import eu.eudat.queryable.collector.Collector;
import eu.eudat.queryable.exceptions.NotSingleResultException;
@ -33,6 +34,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
private List<NestedQuerySinglePredicate<T>> nestedPredicates = new LinkedList<>();
private boolean distinct = false;
private List<OrderByPredicate<T>> orderings = new LinkedList<>();
private List<GroupByPredicate<T>> groupings = new LinkedList<>();
private List<String> fields = new LinkedList<>();
private Integer length;
private Integer offset;
@ -150,6 +152,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
return this;
}
public QueryableList<T> groupBy(GroupByPredicate<T> predicate) {
this.groupings.add(predicate);
return this;
}
public Long count() {
CriteriaBuilder criteriaBuilder = this.manager.getCriteriaBuilder();
CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
@ -157,8 +164,16 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
if (distinct) criteriaQuery.select(criteriaBuilder.countDistinct(this.root.get("id")));
else criteriaQuery.select(criteriaBuilder.count(this.root));
criteriaQuery.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
//if (distinct) criteriaQuery.distinct(true);
return this.manager.createQuery(criteriaQuery).getSingleResult();
//GK: Group By special case. When group by is used, since it will result in a list of how many elements have the grouped field common
// then it will instead return the number of the distinct values of the grouped field
if (this.groupings.isEmpty()) {
return this.manager.createQuery(criteriaQuery).getSingleResult();
} else {
return (long) this.manager.createQuery(criteriaQuery).getResultList().size();
}
}
@Async
@ -169,8 +184,15 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
if (distinct) criteriaQuery.select(criteriaBuilder.countDistinct(this.root.get("id")));
else criteriaQuery.select(criteriaBuilder.count(this.root));
criteriaQuery.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
//if (distinct) criteriaQuery.distinct(true);
return CompletableFuture.supplyAsync(() -> this.manager.createQuery(criteriaQuery).getSingleResult());
return CompletableFuture.supplyAsync(() -> {
if (this.groupings.isEmpty()) {
return this.manager.createQuery(criteriaQuery).getSingleResult();
} else {
return (long) this.manager.createQuery(criteriaQuery).getResultList().size();
}
});
}
@ -205,6 +227,14 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
return predicates.toArray(new Order[predicates.size()]);
}
private Expression[] generateGroupPredicates(List<GroupByPredicate<T>> groupByPredicates, Root<T> root) {
List<Expression> predicates = new LinkedList<>();
for (GroupByPredicate<T> groupPredicate : groupByPredicates) {
predicates.add(groupPredicate.applyPredicate(this.manager.getCriteriaBuilder(), root));
}
return predicates.toArray(new Expression[predicates.size()]);
}
public List<T> toList() {
CriteriaBuilder builder = this.manager.getCriteriaBuilder();
if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
@ -212,22 +242,20 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
this.root = this.query.from(this.tClass);
this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
if (!this.fields.isEmpty()) this.selectFields();
if (distinct) this.query.distinct(true);
//if (!this.fields.isEmpty()) return this.toListWithFields();
//if (!this.fields.isEmpty()) this.query.multiselect(this.parseFields(this.fields));
ObjectMapper mapper = new ObjectMapper();
if (!this.fields.isEmpty()) return this.toListWithFields().stream().map(m -> mapper.convertValue(m, this.tClass)).collect(Collectors.toList());
return this.toListWithOutFields();
}
public List<Map> toListWithFields() {
CriteriaBuilder builder = this.manager.getCriteriaBuilder();
if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
else this.query = builder.createQuery(this.tClass);
this.root = this.query.from(this.tClass);
this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
if (!this.fields.isEmpty()) this.selectFields();
if (distinct) this.query.distinct(true);
List<Tuple> results = this.manager.createQuery(query).getResultList();
TypedQuery<Tuple> typedQuery = this.manager.createQuery(this.query);
if (this.offset != null) typedQuery.setFirstResult(this.offset);
if (this.length != null) typedQuery.setMaxResults(this.length);
List<Tuple> results = typedQuery.getResultList();
Map<Object, List<Tuple>> groupedResults = results.stream()
.collect(Collectors.groupingBy(x -> x.get("id")));
return this.collector.buildFromTuple(results, groupedResults, this.fields, "id");
@ -252,6 +280,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
this.root = this.query.from(this.tClass);
this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
if (!this.fields.isEmpty()) this.selectFields();
if (distinct) this.query.distinct(true);
if (!this.fields.isEmpty()) return this.toListAsyncWithFields();
@ -356,6 +385,8 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
if (!this.orderings.isEmpty())
criteriaQuery.orderBy(this.generateOrderPredicates(this.orderings, criteriaRoot));
if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
TypedQuery<T> typedQuery = this.manager.createQuery(criteriaQuery);
typedQuery.setHint("javax.persistence.fetchgraph", this.manager.getEntityGraph(this.hint));
return typedQuery;
@ -466,4 +497,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
.createQuery(update)
.executeUpdate();
}
private Path[] parseFields(List<String> selectedFields) {
List<Path> paths = new ArrayList<>();
selectedFields.forEach(s -> paths.add(root.get(s)));
return paths.toArray(new Path[0]);
}
}

View File

@ -0,0 +1,10 @@
package eu.eudat.queryable.jpa.predicates;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.Root;
public interface GroupByPredicate<T> {
Expression<T> applyPredicate(CriteriaBuilder builder, Root<T> root);
}

View File

@ -50,31 +50,37 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.9.0</version>
<version>2.12.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework/spring-context -->
<dependency>
<!--<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>4.3.10.RELEASE</version>
</dependency>
<version>5.3.8</version>
</dependency>-->
<!-- guava eu.eudat.cache -->
<dependency>
<!-- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<version>30.1.1-jre</version>
</dependency>-->
<!-- https://mvnrepository.com/artifact/org.springframework/spring-context-support -->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.9.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.7.2</version>
<scope>test</scope>
</dependency>
@ -87,7 +93,7 @@
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.7.0</version>
<version>3.0.0</version>
</dependency>
<!-- facebook Login -->
@ -101,7 +107,7 @@
<dependency>
<groupId>org.springframework.social</groupId>
<artifactId>spring-social-linkedin</artifactId>
<version>1.0.0.RELEASE</version>
<version>1.0.2.RELEASE</version>
</dependency>
<!-- tweeter login-->

View File

@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.web.support.SpringBootServletInitializer;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.scheduling.annotation.EnableAsync;
@SpringBootApplication

View File

@ -0,0 +1,22 @@
package eu.eudat.cache;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import org.springframework.cache.interceptor.KeyGenerator;
import java.lang.reflect.Method;
public class ExternalUrlsKeyGenerator implements KeyGenerator {
@Override
public Object generate(Object o, Method method, Object... params) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append(o.getClass().getSimpleName()).append("_");
stringBuffer.append(method.getName()).append("_");
for (Object param: params) {
if (param instanceof ExternalUrlCriteria) {
ExternalUrlCriteria externalUrlCriteria = (ExternalUrlCriteria) param;
stringBuffer.append(externalUrlCriteria);
}
}
return stringBuffer.toString();
}
}

View File

@ -1,12 +1,13 @@
package eu.eudat.cache;
import com.google.common.cache.CacheBuilder;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.guava.GuavaCache;
import org.springframework.cache.caffeine.CaffeineCache;
import org.springframework.cache.interceptor.KeyGenerator;
import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
@ -29,22 +30,27 @@ public class ResponsesCache {
public CacheManager cacheManager() {
logger.info("Loading ResponsesCache...");
SimpleCacheManager simpleCacheManager = new SimpleCacheManager();
List<GuavaCache> caches = new ArrayList<GuavaCache>();
caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("grants", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("funders", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("currencies", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("licenses", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
List<CaffeineCache> caches = new ArrayList<CaffeineCache>();
caches.add(new CaffeineCache("repositories", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("grants", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("projects", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("funders", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("organisations", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("registries", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("services", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("tags", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("researchers", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("externalDatasets", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("currencies", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("licenses", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
simpleCacheManager.setCaches(caches);
logger.info("OK");
return simpleCacheManager;
}
@Bean(name = "externalUrlsKeyGenerator")
private KeyGenerator externalUrlsKeyGenerator() {
return new ExternalUrlsKeyGenerator();
}
}

View File

@ -2,8 +2,8 @@ package eu.eudat.configurations;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.*;
import org.springframework.core.env.Environment;
import org.springframework.dao.annotation.PersistenceExceptionTranslationPostProcessor;

View File

@ -4,7 +4,6 @@ import eu.eudat.controllers.interceptors.RequestInterceptor;
import eu.eudat.logic.handlers.PrincipalArgumentResolver;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.operations.authentication.AuthenticationService;
import io.prometheus.client.spring.boot.EnablePrometheusEndpoint;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
@ -18,7 +17,6 @@ import java.util.List;
@EnableAsync
@Configuration
@EnableScheduling
@EnablePrometheusEndpoint
public class WebMVCConfiguration extends WebMvcConfigurerAdapter {
private ApiContext apiContext;

View File

@ -20,6 +20,7 @@ import eu.eudat.models.data.listingmodels.UserInfoListingModel;
import eu.eudat.models.data.security.Principal;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import eu.eudat.types.ApiMessageCode;
import eu.eudat.types.MetricNames;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
@ -44,13 +45,15 @@ public class Admin extends BaseController {
private DatasetProfileManager datasetProfileManager;
private UserManager userManager;
private ConfigLoader configLoader;
private final MetricsManager metricsManager;
@Autowired
public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader) {
public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader, MetricsManager metricsManager) {
super(apiContext);
this.datasetProfileManager = datasetProfileManager;
this.userManager = userManager;
this.configLoader = configLoader;
this.metricsManager = metricsManager;
}
@Transactional
@ -70,7 +73,7 @@ public class Admin extends BaseController {
userDatasetProfile.setRole(0);
getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile);
MetricsManager.increaseValue("argos_dataset_templates", 1, "total");
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
return ResponseEntity.status(HttpStatus.OK).body(modelDefinition.getId());
}
@ -89,7 +92,7 @@ public class Admin extends BaseController {
eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(datasetprofile);
datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile);
if (datasetProfile.getStatus() == 1 && oldStatus == 0) {
MetricsManager.increaseValue("argos_dataset_templates", 1, "active");
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.ACTIVE);
}
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<UUID>().status(ApiMessageCode.NO_MESSAGE));
}

View File

@ -20,6 +20,7 @@ import eu.eudat.logic.security.claims.ClaimedAuthorities;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
import eu.eudat.logic.utilities.documents.pdf.PDFUtils;
import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel;
import eu.eudat.models.data.datasetwizard.DatasetsToBeFinalized;
import eu.eudat.models.data.dmp.DataManagementPlan;
@ -69,17 +70,15 @@ public class DMPs extends BaseController {
private DynamicGrantConfiguration dynamicGrantConfiguration;
private Environment environment;
private DataManagementPlanManager dataManagementPlanManager;
private DatasetManager datasetManager;
private ConfigLoader configLoader;
@Autowired
public DMPs(ApiContext apiContext, DynamicGrantConfiguration dynamicGrantConfiguration, Environment environment,
DataManagementPlanManager dataManagementPlanManager, DatasetManager datasetManager, ConfigLoader configLoader) {
DataManagementPlanManager dataManagementPlanManager, ConfigLoader configLoader) {
super(apiContext);
this.dynamicGrantConfiguration = dynamicGrantConfiguration;
this.environment = environment;
this.dataManagementPlanManager = dataManagementPlanManager;
this.datasetManager = datasetManager;
this.configLoader = configLoader;
}
@ -175,7 +174,7 @@ public class DMPs extends BaseController {
public @ResponseBody
ResponseEntity getRDAJsonDocument(@PathVariable String id, @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) {
try {
return this.dataManagementPlanManager.getRDAJsonDocument(id, datasetManager, principal);
return this.dataManagementPlanManager.getRDAJsonDocument(id, principal);
} catch (Exception e) {
return ResponseEntity.status(HttpStatus.EXPECTATION_FAILED).body(new ResponseItem<>().message(e.getMessage()).status(ApiMessageCode.ERROR_MESSAGE));
}
@ -186,8 +185,8 @@ public class DMPs extends BaseController {
ResponseEntity<byte[]> getPDFDocument(@PathVariable String id, @RequestHeader("Content-Type") String contentType,
@ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) throws IllegalAccessException, IOException, InstantiationException, InterruptedException {
FileEnvelope file = this.dataManagementPlanManager.getWordDocument(id, principal, configLoader);
String name = file.getFilename().substring(0, file.getFilename().length() - 5);
File pdffile = datasetManager.convertToPDF(file, environment);
String name = file.getFilename().substring(0, file.getFilename().length() - 5).replace(" ", "_").replace(",", "_");
File pdffile = PDFUtils.convertToPDF(file, environment);
InputStream resource = new FileInputStream(pdffile);
logger.info("Mime Type of " + name + " is " +
new MimetypesFileTypeMap().getContentType(file.getFile()));

View File

@ -54,6 +54,7 @@ public class DashBoardController extends BaseController {
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<RecentActivityModel>>().status(ApiMessageCode.NO_MESSAGE).payload(statistics));
}
@Deprecated
@RequestMapping(method = RequestMethod.GET, value = {"/user/recentActivity"}, produces = "application/json")
public ResponseEntity<ResponseItem<RecentActivity>> getRecentActivity(@RequestParam(name = "numOfActivities", required = false, defaultValue = "5") Integer numberOfActivities, Principal principal) {
RecentActivity statistics = dashBoardManager.getRecentActivity(principal, numberOfActivities);

View File

@ -16,6 +16,7 @@ import eu.eudat.logic.security.claims.ClaimedAuthorities;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.forms.VisibilityRuleService;
import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
import eu.eudat.logic.utilities.documents.pdf.PDFUtils;
import eu.eudat.models.data.dataset.DatasetOverviewModel;
import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel;
import eu.eudat.models.data.datasetwizard.DataManagentPlanListingModel;
@ -38,11 +39,11 @@ import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.persistence.NoResultException;
import javax.transaction.Transactional;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -120,7 +121,7 @@ public class Datasets extends BaseController {
// }
}
@javax.transaction.Transactional
@Transactional
@RequestMapping(method = RequestMethod.GET, value = {"{id}"}, produces = "application/json")
public @ResponseBody
ResponseEntity getSingle(@PathVariable String id, @RequestHeader("Content-Type") String contentType, @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) throws IllegalAccessException, IOException, InstantiationException {
@ -134,7 +135,8 @@ public class Datasets extends BaseController {
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentLength(file.getFile().length());
responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
responseHeaders.set("Content-Disposition", "attachment;filename=" + file.getFilename());
String fileName = file.getFilename().replace(" ", "_").replace(",", "_");
responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName);
responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
@ -177,7 +179,7 @@ public class Datasets extends BaseController {
@RequestMapping(method = RequestMethod.POST, value = {"/getAvailableProfiles"}, produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<List<AssociatedProfile>>> getAvailableProfiles(@RequestBody DatasetProfileWizardAutocompleteRequest datasetProfileWizardAutocompleteRequest, @ClaimedAuthorities(claims = {ANONYMOUS}) Principal principal) throws IllegalAccessException, InstantiationException {
List<AssociatedProfile> dataManagementPlans = DatasetWizardManager.getAvailableProfiles(this.getApiContext().getOperationsContext().getDatabaseRepository().getDmpDao(), datasetProfileWizardAutocompleteRequest);
List<AssociatedProfile> dataManagementPlans = DatasetWizardManager.getAvailableProfiles(this.getApiContext().getOperationsContext().getDatabaseRepository().getDmpDao(), this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao(), datasetProfileWizardAutocompleteRequest);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<AssociatedProfile>>().status(ApiMessageCode.NO_MESSAGE).payload(dataManagementPlans));
}
@ -217,11 +219,11 @@ public class Datasets extends BaseController {
public @ResponseBody
ResponseEntity<byte[]> getPDFDocument(@PathVariable String id, @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) throws IllegalAccessException, IOException, InstantiationException, InterruptedException {
FileEnvelope file = datasetManager.getWordDocumentFile(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal);
String fileName = file.getFilename();
String fileName = file.getFilename().replace(" ", "_").replace(",", "_");
if (fileName.endsWith(".docx")){
fileName = fileName.substring(0, fileName.length() - 5);
}
File pdffile = datasetManager.convertToPDF(file, environment);
File pdffile = PDFUtils.convertToPDF(file, environment);
InputStream resource = new FileInputStream(pdffile);
HttpHeaders responseHeaders = new HttpHeaders();
@ -244,7 +246,7 @@ public class Datasets extends BaseController {
* Data Management
* */
@javax.transaction.Transactional
@Transactional
@RequestMapping(method = RequestMethod.POST, consumes = "application/json", produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<DatasetWizardModel>> createOrUpdate(@RequestBody DatasetWizardModel profile, Principal principal) throws Exception {
@ -260,7 +262,7 @@ public class Datasets extends BaseController {
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Dataset>().status(ApiMessageCode.SUCCESS_MESSAGE).message(this.getApiContext().getHelpersService().getMessageSource().getMessage("dataset.public", new Object[]{}, locale)));
}
@javax.transaction.Transactional
@Transactional
@RequestMapping(method = RequestMethod.DELETE, value = {"/delete/{id}"}, produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<Dataset>> delete(@PathVariable(value = "id") UUID id, Principal principal) throws Exception {
@ -268,7 +270,7 @@ public class Datasets extends BaseController {
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Dataset>().status(ApiMessageCode.SUCCESS_MESSAGE).message("Deleted"));
}
@javax.transaction.Transactional
@Transactional
@RequestMapping(method = RequestMethod.GET, value = {"/{id}/unlock"}, produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<Dataset>> unlock(@PathVariable(value = "id") UUID id, Principal principal) throws Exception {
@ -316,7 +318,7 @@ public class Datasets extends BaseController {
* Data Index
* */
@javax.transaction.Transactional
@Transactional
@RequestMapping(method = RequestMethod.POST, value = {"/index"})
public @ResponseBody
ResponseEntity<ResponseItem<Dataset>> generateIndex(Principal principal) throws Exception {
@ -324,7 +326,7 @@ public class Datasets extends BaseController {
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<eu.eudat.data.entities.Dataset>().status(ApiMessageCode.SUCCESS_MESSAGE).message("Generated").payload(null));
}
@javax.transaction.Transactional
@Transactional
@RequestMapping(method = RequestMethod.DELETE, value = {"/index"})
public @ResponseBody
ResponseEntity<ResponseItem<Dataset>> clearIndex(Principal principal) throws Exception {

View File

@ -35,6 +35,7 @@ import eu.eudat.models.data.login.LoginInfo;
import eu.eudat.models.data.principal.PrincipalModel;
import eu.eudat.models.data.security.Principal;
import eu.eudat.types.ApiMessageCode;
import eu.eudat.types.MetricNames;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -64,6 +65,7 @@ public class Login {
private ConfigurableProviderTokenValidator configurableProviderTokenValidator;
private ZenodoTokenValidator zenodoTokenValidator;
private ConfigLoader configLoader;
private final MetricsManager metricsManager;
// private Logger logger;
@ -75,7 +77,8 @@ public class Login {
B2AccessTokenValidator b2AccessTokenValidator, ORCIDTokenValidator orcidTokenValidator,
LinkedInTokenValidator linkedInTokenValidator, OpenAIRETokenValidator openAIRETokenValidator,
ConfigurableProviderTokenValidator configurableProviderTokenValidator, ZenodoTokenValidator zenodoTokenValidator,
ConfigLoader configLoader, UserManager userManager) {
ConfigLoader configLoader, UserManager userManager,
MetricsManager metricsManager) {
this.customAuthenticationProvider = customAuthenticationProvider;
this.nonVerifiedUserAuthenticationService = nonVerifiedUserAuthenticationService;
this.twitterTokenValidator = twitterTokenValidator;
@ -87,6 +90,7 @@ public class Login {
this.zenodoTokenValidator = zenodoTokenValidator;
this.configLoader = configLoader;
this.userManager = userManager;
this.metricsManager = metricsManager;
}
@ -95,7 +99,7 @@ public class Login {
public @ResponseBody
ResponseEntity<ResponseItem<PrincipalModel>> externallogin(@RequestBody LoginInfo credentials) throws GeneralSecurityException, NullEmailException {
logger.info("Trying To Login With " + credentials.getProvider());
MetricsManager.increaseValue("argos_users", 1, "loggedin");
metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<PrincipalModel>().payload(customAuthenticationProvider.authenticate(credentials)).status(ApiMessageCode.SUCCESS_MESSAGE));
}
@ -104,7 +108,7 @@ public class Login {
public @ResponseBody
ResponseEntity<ResponseItem<PrincipalModel>> nativelogin(@RequestBody Credentials credentials) throws NullEmailException {
logger.info(credentials.getUsername() + " Trying To Login");
MetricsManager.increaseValue("argos_users", 1, "loggedin");
metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<PrincipalModel>().payload(userManager.authenticate(this.nonVerifiedUserAuthenticationService, credentials)).status(ApiMessageCode.SUCCESS_MESSAGE));
}
@ -165,7 +169,7 @@ public class Login {
ResponseEntity<ResponseItem<Principal>> logout(Principal principal) {
this.nonVerifiedUserAuthenticationService.Logout(principal.getToken());
logger.info(principal + " Logged Out");
MetricsManager.decreaseValue("argos_users", 1, "loggedin");
metricsManager.decreaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Principal>().status(ApiMessageCode.NO_MESSAGE));
}

View File

@ -1,10 +1,8 @@
package eu.eudat.controllers;
import eu.eudat.elastic.criteria.DatasetCriteria;
import eu.eudat.elastic.criteria.TagCriteria;
import eu.eudat.elastic.entities.Dataset;
import eu.eudat.elastic.entities.Tag;
import eu.eudat.elastic.repository.Repository;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
@ -49,7 +47,9 @@ public class TagController extends BaseController {
/*List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type);
TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);*/
if (this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().exists()) {
List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
DatasetCriteria criteria = new DatasetCriteria();
criteria.setHasTags(true);
List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tag.getName().toLowerCase().startsWith(query.toLowerCase())).collect(Collectors.toList());
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<Tag>>().payload(tags).status(ApiMessageCode.NO_MESSAGE));
} else {

View File

@ -6,6 +6,7 @@ import eu.eudat.models.data.helpers.responses.ResponseItem;
import eu.eudat.models.data.security.Principal;
import eu.eudat.models.data.userguide.UserGuide;
import eu.eudat.types.ApiMessageCode;
import eu.eudat.types.MetricNames;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpHeaders;
@ -30,16 +31,18 @@ import static eu.eudat.types.Authorities.ADMIN;
public class UserGuideController {
private Environment environment;
private final MetricsManager metricsManager;
@Autowired
public UserGuideController(Environment environment) {
public UserGuideController(Environment environment, MetricsManager metricsManager) {
this.environment = environment;
this.metricsManager = metricsManager;
}
@RequestMapping(path = "{lang}", method = RequestMethod.GET )
public ResponseEntity getUserGuide(@PathVariable(name = "lang") String lang) throws IOException {
long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count();
MetricsManager.calculateValue("argos_languages", (int) files, null);
metricsManager.calculateValue(MetricNames.LANGUAGES, (int) files, null);
Stream<Path> walk = Files.walk(Paths.get(this.environment.getProperty("userguide.path")));
List<String> result = walk.filter(Files::isRegularFile)
.map(Path::toString).collect(Collectors.toList());
@ -50,13 +53,12 @@ public class UserGuideController {
}
InputStream is = new FileInputStream(fileName);
String[] filepath = fileName.split("\\.")[0].split("\\\\");
String simplename = filepath[filepath.length - 1];
Path path = Paths.get(fileName);
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentLength(is.available());
responseHeaders.setContentType(MediaType.TEXT_HTML);
responseHeaders.set("Content-Disposition", "attachment;filename=" + simplename);
responseHeaders.set("Content-Disposition", "attachment;filename=" + path.getFileName().toString());
responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
@ -71,7 +73,7 @@ public class UserGuideController {
@RequestMapping(value = "current", method = RequestMethod.POST)
public @ResponseBody
ResponseEntity<ResponseItem<String>> updateGuide(@RequestBody UserGuide guide, @ClaimedAuthorities(claims = {ADMIN}) Principal principal) throws Exception {
String fileName = this.environment.getProperty("userguide.path") + guide.getName() + ".html";
String fileName = this.environment.getProperty("userguide.path") + guide.getName();
OutputStream os = new FileOutputStream(fileName);
os.write(guide.getHtml().getBytes());
os.close();

View File

@ -15,6 +15,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import javax.xml.bind.JAXBException;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
@ -42,15 +43,18 @@ public class UserInvitationController extends BaseController {
@Transactional
@RequestMapping(method = RequestMethod.GET, value = {"/exchange/{invitationID}"}, produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<UUID>> exchange(@PathVariable UUID invitationID, Principal principal) throws JAXBException {
ResponseEntity<ResponseItem<UUID>> exchange(@PathVariable UUID invitationID, Principal principal) throws JAXBException, IOException {
UUID dmpId = invitationsManager.assignUserAcceptedInvitation(invitationID, principal);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<UUID>().status(ApiMessageCode.SUCCESS_MESSAGE).payload(dmpId));
}
@RequestMapping(method = RequestMethod.POST, value = {"/getUsers"}, consumes = "application/json", produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<List<UserInfoInvitationModel>>> getUsers(Principal principal) throws IllegalAccessException, InstantiationException {
List<UserInfoInvitationModel> users = invitationsManager.getUsers(principal);
// ResponseEntity<ResponseItem<List<UserInfoInvitationModel>>> getUsers(Principal principal) throws IllegalAccessException, InstantiationException {
ResponseEntity<ResponseItem<List<UserInfoInvitationModel>>> getUsers(Principal principal, @RequestBody UserInfoRequestItem userInfoRequestItem) throws IllegalAccessException, InstantiationException {
System.out.println(userInfoRequestItem.getCriteria().getLike());
// List<UserInfoInvitationModel> users = invitationsManager.getUsers(principal);
List<UserInfoInvitationModel> users = invitationsManager.getUsersWithCriteria(principal, userInfoRequestItem);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<UserInfoInvitationModel>>().status(ApiMessageCode.SUCCESS_MESSAGE).payload(users));
}
}

View File

@ -8,13 +8,13 @@ import eu.eudat.data.dao.entities.DMPDao;
import eu.eudat.data.dao.entities.DatasetDao;
import eu.eudat.data.dao.entities.GrantDao;
import eu.eudat.data.dao.entities.OrganisationDao;
import eu.eudat.data.entities.DMP;
import eu.eudat.data.entities.Dataset;
import eu.eudat.data.entities.Grant;
import eu.eudat.data.entities.UserInfo;
import eu.eudat.data.entities.*;
import eu.eudat.data.query.PaginationService;
import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
import eu.eudat.elastic.criteria.SortCriteria;
import eu.eudat.elastic.entities.Dmp;
import eu.eudat.logic.builders.model.models.RecentActivityDataBuilder;
import eu.eudat.logic.mapper.elastic.criteria.DmpCriteriaMapper;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.models.HintedModelFactory;
@ -26,6 +26,8 @@ import eu.eudat.models.data.dashboard.recent.model.RecentDmpModel;
import eu.eudat.models.data.dashboard.recent.tablerequest.RecentActivityTableRequest;
import eu.eudat.models.data.dashboard.searchbar.SearchBarItem;
import eu.eudat.models.data.dashboard.statistics.DashBoardStatistics;
import eu.eudat.models.data.listingmodels.DataManagementPlanListingModel;
import eu.eudat.models.data.listingmodels.DatasetListingModel;
import eu.eudat.models.data.security.Principal;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.searchbar.SearchBarItemType;
@ -36,9 +38,11 @@ import org.springframework.stereotype.Component;
import javax.transaction.Transactional;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
@Component
@ -97,8 +101,8 @@ public class DashBoardManager {
}
public DashBoardStatistics getMeStatistics(Principal principal) throws IOException {
List<eu.eudat.elastic.entities.Dataset> datasets = null;
List<eu.eudat.elastic.entities.Dmp> dmps = null;
Long datasets = 0L;
Long dmps = 0L;
DashBoardStatistics statistics = new DashBoardStatistics();
DMPDao dataManagementPlanRepository = databaseRepository.getDmpDao();
DatasetDao datasetRepository = databaseRepository.getDatasetDao();
@ -112,7 +116,8 @@ public class DashBoardManager {
eu.eudat.elastic.criteria.DatasetCriteria datasetElasticCriteria = new eu.eudat.elastic.criteria.DatasetCriteria();
datasetElasticCriteria.setAllowAllVersions(false);
datasetElasticCriteria.setPublic(false);
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().count(datasetElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
datasets = null;
@ -126,33 +131,44 @@ public class DashBoardManager {
eu.eudat.elastic.criteria.DmpCriteria dmpElasticCriteria = new eu.eudat.elastic.criteria.DmpCriteria();
dmpElasticCriteria.setAllowAllVersions(false);
dmpElasticCriteria.setPublic(false);
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().count(dmpElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
dmps = null;
}
}
dataManagementPlanCriteria.setAllVersions(false);
GrantCriteria grantCriteria = new GrantCriteria();
dataManagementPlanCriteria.setOnlyPublic(false);
dataManagementPlanCriteria.setIsPublic(false);
GrantCriteria grantCriteria = new GrantCriteria();
grantCriteria.setActive(true);
OrganisationCriteria organisationCriteria = new OrganisationCriteria();
organisationCriteria.setActive(true);
List<Integer> roles = new LinkedList<>();
List<Dmp> finalDmps = dmps;
CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated((dmps != null && !dmps.isEmpty()) ? dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))) : dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
.whenComplete((dmpsStats, throwable) -> statistics.setTotalDataManagementPlanCount(dmpsStats));
List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
CompletableFuture datasetFuture = datasetRepository.getAuthenticated((datasets != null && !datasets.isEmpty()) ? datasetRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))) : datasetRepository.getWithCriteria(datasetCriteria), user, roles).countAsync()
.whenComplete((datasetsStats, throwable) -> statistics.setTotalDataSetCount(datasetsStats));
if ((dmps == null || dmps == 0L) && (datasets == null || datasets == 0L)) {
CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated(dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
.whenComplete((dmpsStats, throwable) -> statistics.setTotalDataManagementPlanCount(dmpsStats));
CompletableFuture datasetFuture = datasetRepository.getAuthenticated( datasetRepository.getWithCriteria(datasetCriteria), user, roles).distinct().countAsync()
.whenComplete((datasetsStats, throwable) -> statistics.setTotalDataSetCount(datasetsStats));
CompletableFuture.allOf(dmpFuture, datasetFuture).join();
} else {
statistics.setTotalDataManagementPlanCount(dmps);
statistics.setTotalDataSetCount(datasets);
}
CompletableFuture grantFuture = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user).countAsync()
.whenComplete((grantsStats, throwable) -> statistics.setTotalGrantCount(grantsStats));
CompletableFuture orgnanisationFuture = organisationRepository.getAuthenticated(organisationRepository.asQueryable().withHint("organisationRecentActivity"), user).countAsync()
CompletableFuture orgnanisationFuture = organisationRepository.getAuthenticated(organisationRepository.getWithCriteria(organisationCriteria).withHint("organisationRecentActivity"), user).countAsync()
.whenComplete((organisationStats, throwable) -> statistics.setTotalOrganisationCount(organisationStats));
CompletableFuture.allOf(dmpFuture, datasetFuture, grantFuture, orgnanisationFuture).join();
CompletableFuture.allOf( grantFuture, orgnanisationFuture).join();
return statistics;
}
@Deprecated
public RecentActivity getRecentActivity(Principal principal, Integer numberofactivities) {
RecentActivity activity = new RecentActivity();
DMPDao dataManagementPlanRepository = databaseRepository.getDmpDao();
@ -175,7 +191,7 @@ public class DashBoardManager {
.selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
.whenComplete((dmpActivities, throwable) -> activity.setRecentDmpActivities(dmpActivities));
CompletableFuture<List<RecentActivityData>> datasets = datasetRepository.getAuthenticated(datasetRepository.getWithCriteria(datasetCriteria), user, roles)
CompletableFuture<List<RecentActivityData>> datasets = datasetRepository.getAuthenticated(datasetRepository.getWithCriteria(datasetCriteria), user, roles).distinct()
.withHint("datasetRecentActivity")
.orderBy((builder, root) -> builder.desc(root.get("modified")))
.take(numberofactivities)
@ -225,7 +241,17 @@ public class DashBoardManager {
datasetElasticCriteria.setLike(tableRequest.getCriteria().getLike());
datasetElasticCriteria.setAllowAllVersions(false);
datasetElasticCriteria.setPublic(!isAuthenticated);
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
datasetElasticCriteria.setOffset(tableRequest.getDatasetOffset());
datasetElasticCriteria.setSize(tableRequest.getLength());
if (isAuthenticated) {
datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
}
datasetElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
datasetElasticCriteria.getSortCriteria().stream().filter(sortCriteria -> sortCriteria.getFieldName().equals("publishedAt")).forEach(sortCriteria -> {
sortCriteria.setFieldName("dmp:" + sortCriteria.getFieldName());
sortCriteria.setColumnType(SortCriteria.ColumnType.JOIN_COLUMN);
});
datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().queryIds(datasetElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
datasets = null;
@ -238,10 +264,16 @@ public class DashBoardManager {
dmpElasticCriteria.setLike(tableRequest.getCriteria().getLike());
dmpElasticCriteria.setAllowAllVersions(false);
dmpElasticCriteria.setPublic(!isAuthenticated);
dmpElasticCriteria.setOffset(tableRequest.getDmpOffset());
dmpElasticCriteria.setSize(tableRequest.getLength());
if (isAuthenticated) {
dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
}
dmpElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
}catch (Exception e) {
logger.warn(e.getMessage(), e);
datasets = null;
dmps = null;
}
}
@ -250,6 +282,8 @@ public class DashBoardManager {
dmpList = dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))).distinct();
} else {
dmpList = dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria).distinct();
PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
dmpList.skip(tableRequest.getDmpOffset()).take(tableRequest.getLength());
}
if (datasets != null && !datasets.isEmpty()) {
@ -258,53 +292,57 @@ public class DashBoardManager {
} else {
datasetList = datasetRepository.getWithCriteria(datasetCriteria);
}
IntStream.range(0, tableRequest.getOrderings().getFields().size()).filter(i -> tableRequest.getOrderings().getFields().get(i).contains("publishedAt"))
.forEach(i -> tableRequest.getOrderings().getFields().set(i, tableRequest.getOrderings().getFields().get(i).toCharArray()[0] + "dmp:publishedAt|join|"));
// tableRequest.getOrderings().getFields().stream().filter(s -> s.contains("publishedAt")).forEach(s -> s = s.toCharArray()[0] + "dmp:publishedAt|join|" );
/*for (int i = 0; i< tableRequest.getOrderings().getFields().size(); i++) {
if (tableRequest.getOrderings().getFields().get(i).contains("publishedAt")) {
String newField = tableRequest.getOrderings().getFields().get(i).toCharArray()[0] + "dmp:publishedAt|join|";
tableRequest.getOrderings().getFields().set(i, newField);
}
}*/
/*if (tableRequest.getOrderings().getFields().get(0).contains("publishedAt")) {
tableRequest.getOrderings().getFields().set(0, tableRequest.getOrderings().getFields().get(0).charAt(0) + "dmp:" + tableRequest.getOrderings().getFields().get(0).substring(1) + "|join|");
}*/
PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
datasetList.skip(tableRequest.getDatasetOffset()).take(tableRequest.getLength());
if (isAuthenticated) {
List<Integer> roles = new LinkedList<>();
roles.add(UserDMP.UserDMPRoles.USER.getValue());
roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
dmpList = dataManagementPlanRepository.getAuthenticated(dmpList, principal.getId(), roles);
datasetList = datasetRepository.getAuthenticated(datasetList, user, roles);
datasetList = datasetRepository.getAuthenticated(datasetList, user, roles).distinct();
}
PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
for (int i = 0; i< tableRequest.getOrderings().getFields().length; i++) {
if (tableRequest.getOrderings().getFields()[i].contains("publishedAt")) {
String newField = tableRequest.getOrderings().getFields()[i].toCharArray()[0] + "dmp:publishedAt|join|";
tableRequest.getOrderings().getFields()[i] = newField;
}
}
PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
/*CompletableFuture future = CompletableFuture.runAsync(() -> */{
recentActivityModels.addAll(dmpList
.withHint(HintedModelFactory.getHint(RecentDmpModel.class))
// .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
.skip(tableRequest.getDmpOffset())
.take(tableRequest.getLength())
.select(item -> {
item.setDataset(
item.getDataset().stream()
.filter(dataset -> !dataset.getStatus().equals(Dataset.Status.DELETED.getValue()) && !dataset.getStatus().equals(Dataset.Status.CANCELED.getValue())).collect(Collectors.toList()).stream()
.filter(dataset -> dataset.getDmp().getUsers().stream()
.filter(x -> x.getUser().getId().equals(principal.getId()))
.collect(Collectors.toList()).size() > 0)
.collect(Collectors.toSet()));
return new RecentDmpModel().fromEntity(item);
}));
List<DMP> dmps1 = dmpList.withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))
.distinct().toList();
recentActivityModels.addAll(dmps1.stream().map(dmp -> {
DatasetCriteria datasetCriteria1 = new DatasetCriteria();
datasetCriteria1.setDmpIds(Collections.singletonList(dmp.getId()));
datasetCriteria1.setAllVersions(false);
if (isAuthenticated) {
dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1, principal.getId()));
} else {
datasetCriteria1.setIsPublic(true);
dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1));
}
return new RecentDmpModel().fromDataModel(dmp);
}).collect(Collectors.toList()));
recentActivityModels.addAll(datasetList
.withHint(HintedModelFactory.getHint(RecentDatasetModel.class))
// .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
.skip(tableRequest.getDatasetOffset())
.take(tableRequest.getLength())
.select(item -> {
return new RecentDatasetModel().fromEntity(item);
}));
List<RecentActivityModel> recentDatasetModels = datasetList
.withHint(HintedModelFactory.getHint(DatasetListingModel.class))
.select(item -> new RecentDatasetModel().fromEntity(item));
recentActivityModels.addAll(recentDatasetModels);
}/*);*/
//GK: Shuffle the deck otherwise we will summon the DMPodia when sorting with status
int pos = -1;
/*int pos = -1;
for (int i = (recentActivityModels.size() / 2); i < recentActivityModels.size(); i++) {
RecentActivityModel recentActivityModel = recentActivityModels.remove(i);
while (pos < recentActivityModels.size()) {
@ -314,12 +352,29 @@ public class DashBoardManager {
}
}
recentActivityModels.add(pos, recentActivityModel);
}
}*/
//GK: No one likes to play shuffle with the recent activities. So just re-sort them based on how they have been sorted already
recentActivityModels = recentActivityModels.stream().sorted((o1, o2) -> {
try {
String order = tableRequest.getOrderings().getFields().get(0).toCharArray()[0] + "";
String field = tableRequest.getOrderings().getFields().get(0).substring(1);
if (field.contains(":") && field.contains("|")) {
field = field.substring(field.lastIndexOf(":") + 1, field.indexOf("|"));
}
field = field.equals("label") ? "title" : field;
field = field.substring(0, 1).toUpperCase() + field.substring(1);
return (order.equals("+") ? 1 : -1 ) * ((Comparable)o1.getClass().getMethod("get" + field).invoke(o1)).compareTo(o2.getClass().getMethod("get" + field).invoke(o2));
} catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
logger.error(e.getLocalizedMessage(), e);
}
return 0;
}).collect(Collectors.toList());
//CompletableFuture.allOf(future).join();
// CompletableFuture.allOf(dmps, datasets).join();
return recentActivityModels.stream().sorted(this.comparators.get(tableRequest.getCriteria().getOrder())).limit(tableRequest.getLength()).collect(Collectors.toList());
return recentActivityModels;
}
public List<SearchBarItem> searchUserData(String like, Principal principal) {
@ -380,4 +435,38 @@ public class DashBoardManager {
return searchBarItems;
}
private Set<Dataset> retrieveRelevantDatasets(DatasetCriteria datasetCriteria) {
return retrieveRelevantDatasets(datasetCriteria, null);
}
private Set<Dataset> retrieveRelevantDatasets (DatasetCriteria datasetCriteria, UUID principal) {
QueryableList<Dataset> datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria);
if (principal != null) {
UserInfo userInfo = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal);
List<Integer> roles = new ArrayList<>();
roles.add(0);
roles.add(1);
datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getAuthenticated(datasetItems, userInfo, roles);
}
Long maxDatasets = datasetItems.distinct().count();
DatasetTableRequest datasetTableRequest = new DatasetTableRequest();
datasetTableRequest.setOffset(0);
datasetTableRequest.setLength(3);
Set<Dataset> datasetsSet = new LinkedHashSet<>();
try {
datasetItems = PaginationManager.applyPaging(datasetItems, datasetTableRequest);
List<Dataset> datasets = datasetItems.distinct().toList();
datasetsSet.addAll(datasets);
for (int i = 0; i < maxDatasets - datasets.size(); i++) {
Dataset fakedataset = new Dataset();
fakedataset.setId(UUID.randomUUID());
datasetsSet.add(fakedataset);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return datasetsSet;
}
}

View File

@ -37,6 +37,7 @@ import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.logic.services.utilities.UtilitiesService;
import eu.eudat.logic.utilities.builders.XmlBuilder;
import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
import eu.eudat.logic.utilities.documents.pdf.PDFUtils;
import eu.eudat.logic.utilities.documents.types.ParagraphStyle;
import eu.eudat.logic.utilities.documents.word.WordBuilder;
import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder;
@ -59,6 +60,7 @@ import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import eu.eudat.models.data.userinfo.UserListingModel;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.Authorities;
import eu.eudat.types.MetricNames;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
@ -77,6 +79,7 @@ import org.springframework.web.multipart.MultipartFile;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import javax.transaction.Transactional;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
@ -108,9 +111,10 @@ public class DataManagementPlanManager {
private Environment environment;
private RDAManager rdaManager;
private UserManager userManager;
private final MetricsManager metricsManager;
@Autowired
public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager) {
public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager, MetricsManager metricsManager) {
this.apiContext = apiContext;
this.datasetManager = datasetManager;
this.utilitiesService = apiContext.getUtilitiesService();
@ -118,6 +122,7 @@ public class DataManagementPlanManager {
this.environment = environment;
this.rdaManager = rdaManager;
this.userManager = userManager;
this.metricsManager = metricsManager;
}
/*
@ -149,8 +154,10 @@ public class DataManagementPlanManager {
items = null;
}
}
if (items == null) {
items = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dataManagementPlanTableRequest.getCriteria());
}
List<Integer> roles = new LinkedList<>();
if (!dataManagementPlanTableRequest.getCriteria().isOnlyPublic()) {
if (dataManagementPlanTableRequest.getCriteria().getRole() != null)
@ -159,8 +166,11 @@ public class DataManagementPlanManager {
} else {
authItems = items;
}
if (dmps == null) {
totalData = authItems.count();
items = PaginationManager.applyPaging(authItems, dataManagementPlanTableRequest);
} else {
items = authItems;
}
@ -175,6 +185,8 @@ public class DataManagementPlanManager {
dataTable.setData(dmps1.stream().map(dmp -> {
DatasetCriteria datasetCriteria = new DatasetCriteria();
datasetCriteria.setDmpIds(Collections.singletonList(dmp.getId()));
datasetCriteria.setAllVersions(false);
datasetCriteria.setIsPublic(dataManagementPlanTableRequest.getCriteria().getIsPublic());
dmp.setDataset(retrieveRelevantDatasets(datasetCriteria, principalID));
@ -240,14 +252,14 @@ public class DataManagementPlanManager {
roles.add(1);
datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getAuthenticated(datasetItems, userInfo, roles);
}
Long maxDatasets = datasetItems.count();
Long maxDatasets = datasetItems.distinct().count();
DatasetTableRequest datasetTableRequest = new DatasetTableRequest();
datasetTableRequest.setOffset(0);
datasetTableRequest.setLength(3);
Set<Dataset> datasetsSet = new LinkedHashSet<>();
try {
datasetItems = PaginationManager.applyPaging(datasetItems, datasetTableRequest);
List<Dataset> datasets = datasetItems.toList();
List<Dataset> datasets = datasetItems.distinct().toList();
datasetsSet.addAll(datasets);
for (int i = 0; i < maxDatasets - datasets.size(); i++) {
Dataset fakedataset = new Dataset();
@ -441,7 +453,7 @@ public class DataManagementPlanManager {
setNotification = true;
} else {
MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
}
DMP newDmp = dataManagementPlan.toDataModel();
@ -532,6 +544,9 @@ public class DataManagementPlanManager {
if (dataManagementPlan.getAssociatedUsers().size() == 0)
assignUser(newDmp, user);
UUID dmpId = newDmp.getId();
newDmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(newDmp);
if (setNotification) {
@ -565,7 +580,7 @@ public class DataManagementPlanManager {
if (dataManagementPlan.getStatus() == (int) DMP.DMPStatus.FINALISED.getValue() && dmp1.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()))
throw new Exception("DMP is finalized, therefore cannot be edited.");
} else {
MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
}
List<Dataset> datasets = new ArrayList<>();
DMP tempDMP = dataManagementPlan.toDataModel();
@ -586,6 +601,11 @@ public class DataManagementPlanManager {
datasets.add(dataset);
}
UUID dmpId = result.getId();
result.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(result);
return result;
}
@ -623,15 +643,31 @@ public class DataManagementPlanManager {
assignGrandUserIfInternal(newDmp, user);
assignFunderUserIfInternal(newDmp, user);
assignProjectUserIfInternal(newDmp, user);
if (newDmp.getGrant().getStartdate() == null) {
newDmp.getGrant().setStartdate(new Date());
}
if (newDmp.getGrant().getEnddate() == null) {
newDmp.getGrant().setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS)));
}
databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant());
DMP tempDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp);
newDmp.setId(tempDmp.getId());
// Assign creator.
assignUser(newDmp, user);
//assignUser(newDmp, user);
copyDatasets(newDmp, databaseRepository.getDatasetDao());
databaseRepository
.getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), oldDmp.getId()))
.toList().stream().forEach(userDMP -> {
UserDMP temp = new UserDMP();
temp.setUser(userDMP.getUser());
temp.setRole(userDMP.getRole());
temp.setDmp(newDmp);
apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().createOrUpdate(temp);
});
newDmp.setUsers(new HashSet<>(databaseRepository.getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), newDmp.getId())).toList()));
DatasetCriteria criteria1 = new DatasetCriteria();
@ -679,9 +715,11 @@ public class DataManagementPlanManager {
DatasetCriteria criteria1 = new DatasetCriteria();
criteria1.setDmpIds(Collections.singletonList(newDmp.getId()));
newDmp.setDataset(new HashSet<>(databaseRepository.getDatasetDao().getWithCriteria(criteria1).toList()));
UUID dmpId = newDmp.getId();
newDmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(newDmp);
MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
return newDmp.getId();
}
@ -695,26 +733,30 @@ public class DataManagementPlanManager {
DMP oldDmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(uuid);
switch (oldDmp.getStatus()) {
case 0:
MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft");
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
break;
case 1:
if (oldDmp.getDoi() != null) {
MetricsManager.decreaseValue("argos_managed_dmps", 1, "doied");
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DOIED);
}
if (oldDmp.isPublic()) {
MetricsManager.decreaseValue("argos_managed_dmps", 1, "published");
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
}
MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized");
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
break;
}
oldDmp.setStatus(DMP.DMPStatus.DELETED.getValue());
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(oldDmp);
UUID dmpId = oldDmp.getId();
oldDmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(oldDmp);
DataManagementPlanCriteria criteria1 = new DataManagementPlanCriteria();
criteria1.setAllVersions(true);
criteria1.setGroupIds(Collections.singletonList(oldDmp.getGroupId()));
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria1).toList().forEach(dmp -> {
try {
UUID tdmpId = dmp.getId();
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), tdmpId)).toList()));
this.updateIndex(dmp);
} catch (IOException e) {
logger.error(e.getMessage(), e);
@ -726,6 +768,7 @@ public class DataManagementPlanManager {
if (elastic != null) {
tags = elastic.getTags();
}
dataset.setDmp(dmp);
this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -744,7 +787,7 @@ public class DataManagementPlanManager {
else {
researcher.setCreationUser(user);
researcherRepository.createOrUpdate(researcher);
MetricsManager.increaseValue("argos_researchers", 1, null);
metricsManager.increaseValue(MetricNames.RESEARCHER, 1, null);
}
}
}
@ -785,6 +828,12 @@ public class DataManagementPlanManager {
else {
grant.setType(Grant.GrantType.EXTERNAL.getValue());
grant.setCreationUser(null);
if (grant.getStartdate() == null) {
grant.setStartdate(new Date());
}
if (grant.getEnddate() == null) {
grant.setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS)));
}
grantDao.createOrUpdate(grant);
}
}
@ -823,21 +872,24 @@ public class DataManagementPlanManager {
eu.eudat.data.entities.Project projectEntity = projectDao.getWithCritetia(criteria).toList().stream().max(Comparator.comparing(project1 -> project1.getModified().getTime())).orElse(null);
if (projectEntity != null) project.setId(projectEntity.getId());
else {
project.setType(Project.ProjectType.EXTERNAL.getValue());
if (project.getId() == null) project.setId(UUID.randomUUID());
projectDao.createOrUpdate(project);
MetricsManager.increaseValue("argos_projects", 1, null);
createExternalProject(project, projectDao);
}
}
else {
project.setType(Project.ProjectType.EXTERNAL.getValue());
if (project.getId() == null) project.setId(UUID.randomUUID());
projectDao.createOrUpdate(project);
MetricsManager.increaseValue("argos_projects", 1, null);
createExternalProject(project, projectDao);
}
}
}
private void createExternalProject(Project project, ProjectDao projectDao) {
if (project.getStartdate() == null) project.setStartdate(new Date());
if (project.getEnddate() == null) project.setEnddate(new Date());
project.setType(Project.ProjectType.EXTERNAL.getValue());
if (project.getId() == null) project.setId(UUID.randomUUID());
projectDao.createOrUpdate(project);
metricsManager.increaseValue(MetricNames.PROJECT, 1, null);
}
private void copyDatasets(DMP newDmp, DatasetDao datasetDao) {
List<CompletableFuture<Dataset>> futures = new LinkedList<>();
for (Dataset dataset : newDmp.getDataset()) {
@ -851,7 +903,8 @@ public class DataManagementPlanManager {
if (elastic != null) {
tags = elastic.getTags();
}
UUID dmpId = tempDataset.getDmp().getId();
tempDataset.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.datasetManager.updateTags(tempDataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -910,7 +963,7 @@ public class DataManagementPlanManager {
return newDataset;
}).thenApplyAsync(item -> {
futures.add(datasetDao.createOrUpdateAsync(item).whenComplete(((dataset1, throwable) -> {
MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft");
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
eu.eudat.elastic.entities.Dataset datasetElastic = new eu.eudat.elastic.entities.Dataset();
datasetElastic.setId(dataset1.getId().toString());
datasetElastic.setLabel(dataset1.getLabel());
@ -968,8 +1021,10 @@ public class DataManagementPlanManager {
throw new Exception("DMP is not finalized");
dmp.setPublic(true);
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
UUID dmpId = dmp.getId();
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(dmp);
MetricsManager.increaseValue("argos_managed_dmps", 1, "published");
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setGroupIds(Collections.singletonList(dmp.getGroupId()));
criteria.setAllVersions(true);
@ -981,8 +1036,10 @@ public class DataManagementPlanManager {
if (elastic != null) {
tags = elastic.getTags();
}
UUID tmdmpId = dataset.getDmp().getId();
dataset.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), tmdmpId)).toList()));
this.datasetManager.updateTags(dataset, tags);
MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published");
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
@ -995,6 +1052,7 @@ public class DataManagementPlanManager {
}
}
@Transactional
public void makeFinalize(UUID id, Principal principal, DatasetsToBeFinalized datasetsToBeFinalized) throws Exception {
DMP dmp = this.apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(id);
if (!isUserOwnerOfDmp(dmp, principal))
@ -1065,14 +1123,16 @@ public class DataManagementPlanManager {
}
dmp.setStatus(DMP.DMPStatus.FINALISED.getValue());
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
UUID dmpId = dmp.getId();
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(dmp);
UserInfo user = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId());
sendNotification(dmp, user, NotificationType.DMP_FINALISED);
MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft");
MetricsManager.increaseValue("argos_managed_dmps", 1, "finalized");
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
this.updateDatasetsIndex(indexDatasets);
MetricsManager.decreaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "draft");
MetricsManager.increaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "finalized");
metricsManager.decreaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.FINALIZED);
}
public void undoFinalize(UUID id, Principal principal) throws Exception {
@ -1083,9 +1143,11 @@ public class DataManagementPlanManager {
throw new Exception("DMP is already Active");
dmp.setStatus(DMP.DMPStatus.ACTIVE.getValue());
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
UUID dmpId = dmp.getId();
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(dmp);
MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized");
MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
}
public void updateUsers(UUID id, List<UserInfoListingModel> users, Principal principal) throws Exception {
@ -1122,7 +1184,7 @@ public class DataManagementPlanManager {
wordBuilder.addParagraphContent("Data Management Plan Information", document, ParagraphStyle.HEADER1, BigInteger.ZERO);
// DMP title custom style.
wordBuilder.addParagraphContent(dmpEntity.getLabel(), document, ParagraphStyle.HEADER2, BigInteger.ZERO);
wordBuilder.addParagraphContent(dmpEntity.getDescription(), document, ParagraphStyle.TEXT, BigInteger.ZERO);
wordBuilder.addParagraphContent(dmpEntity.getDescription(), document, ParagraphStyle.HTML, BigInteger.ZERO);
wordBuilder.addParagraphContent("Funder", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
if (dmpEntity.getGrant().getFunder() != null)
@ -1161,8 +1223,14 @@ public class DataManagementPlanManager {
.forEach(datasetEntity -> {
Map<String, Object> properties = new HashMap<>();
if (datasetEntity.getProperties() != null) {
JSONObject jObject = new JSONObject(datasetEntity.getProperties());
properties = jObject.toMap();
ObjectMapper objectMapper = new ObjectMapper();
try {
properties = objectMapper.readValue(datasetEntity.getProperties(), LinkedHashMap.class);
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
/*JSONObject jObject = new JSONObject(datasetEntity.getProperties());
properties = jObject.toMap();*/
}
// Custom style for the Dataset title.
@ -1226,7 +1294,7 @@ public class DataManagementPlanManager {
}*/
wordBuilder.addParagraphContent(datasetEntity.getDescription(), document, ParagraphStyle.TEXT, BigInteger.ZERO);
wordBuilder.addParagraphContent(datasetEntity.getDescription(), document, ParagraphStyle.HTML, BigInteger.ZERO);
// Dataset Description custom style.
XWPFParagraph datasetDescriptionParagraph = document.createParagraph();
@ -1277,7 +1345,7 @@ public class DataManagementPlanManager {
eu.eudat.data.entities.DMP dmp = databaseRepository.getDmpDao().find(UUID.fromString(id));
if (!dmp.isPublic() && dmp.getUsers().stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()).collect(Collectors.toList()).size() == 0)
throw new UnauthorisedException();
List<Dataset> datasets = dmp.getDataset().stream().collect(Collectors.toList());
List<Dataset> datasets = dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != Dmp.DMPStatus.DELETED.getValue()).collect(Collectors.toList());
/*String fileName = dmp.getLabel();
fileName = fileName.replaceAll("[^a-zA-Z0-9+ ]", "");*/
String uuid = UUID.randomUUID().toString();
@ -1292,6 +1360,22 @@ public class DataManagementPlanManager {
dmpName.setTextContent(dmp.getLabel());
dmpElement.appendChild(dmpName);
if (dmp.getExtraProperties() != null && !dmp.getExtraProperties().isEmpty()) {
Map<String, Object> extraProperties = new ObjectMapper().readValue(dmp.getExtraProperties(), HashMap.class);
Element language = xmlDoc.createElement("language");
language.setTextContent(extraProperties.get("language") != null ? extraProperties.get("language").toString() : null);
dmpElement.appendChild(language);
Element visibility = xmlDoc.createElement("visibility");
visibility.setTextContent(extraProperties.get("visible") != null ? extraProperties.get("visible").toString() : null);
dmpElement.appendChild(visibility);
Element publicDate = xmlDoc.createElement("publicDate");
publicDate.setTextContent(extraProperties.get("publicDate") != null ? extraProperties.get("publicDate").toString() : null);
dmpElement.appendChild(publicDate);
Element costs = xmlDoc.createElement("costs");
costs.setTextContent(extraProperties.get("costs") != null ? extraProperties.get("costs").toString() : null);
dmpElement.appendChild(costs);
}
DMPProfile dmpProfile = dmp.getProfile();
Element dmpProfileElement = xmlDoc.createElement("dmpProfile");
Element dmpProfileName = xmlDoc.createElement("dmpProfileName");
@ -1407,7 +1491,7 @@ public class DataManagementPlanManager {
return fileEnvelope;
}
public ResponseEntity<byte[]> getRDAJsonDocument(String id, DatasetManager datasetManager, Principal principal) throws Exception {
public ResponseEntity<byte[]> getRDAJsonDocument(String id, Principal principal) throws Exception {
eu.eudat.data.entities.DMP dmp = databaseRepository.getDmpDao().find(UUID.fromString(id));
if (!dmp.isPublic() && dmp.getUsers().stream().noneMatch(userInfo -> userInfo.getUser().getId() == principal.getId()))
throw new UnauthorisedException();
@ -1417,7 +1501,7 @@ public class DataManagementPlanManager {
/*ObjectMapper mapper = new ObjectMapper();
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);*/
String fileName = "DMP_" + dmp.getGrant().getLabel() + "_" + dmp.getVersion();//dmp.getLabel();
fileName = fileName.replaceAll("[^a-zA-Z0-9+ ]", "");
fileName = fileName.replaceAll("[^a-zA-Z0-9+ ]", "").replace(" ", "_").replace(",", "_");
String uuid = UUID.randomUUID().toString();
File file = new File(this.environment.getProperty("temp.temp") + uuid + ".json");
OutputStream output = new FileOutputStream(file);
@ -1456,11 +1540,12 @@ public class DataManagementPlanManager {
default:
file = getXmlDocument(id, principal);
}
String fileName = file.getFilename().replace(" ", "_").replace(",", "_");
InputStream resource = new FileInputStream(file.getFile());
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentLength(file.getFile().length());
responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
responseHeaders.set("Content-Disposition", "attachment;filename=" + file.getFilename());
responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName);
responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
@ -1580,6 +1665,21 @@ public class DataManagementPlanManager {
dm.setAssociatedUsers(associatedUsers); // Sets associatedUsers property.
dm.setDynamicFields(dynamicFields); // Sets dynamicFields property.
dm.setDefinition(dmpProfile);
ObjectMapper mapper = new ObjectMapper();
Map<String, Object> extraPropertiesMap = new HashMap<>();
if (dataManagementPlans.get(0).getLanguage() != null) {
extraPropertiesMap.put("language", dataManagementPlans.get(0).getLanguage());
}
if (dataManagementPlans.get(0).getVisibility() != null) {
extraPropertiesMap.put("visible", dataManagementPlans.get(0).getVisibility());
}
if (dataManagementPlans.get(0).getPublicDate() != null) {
extraPropertiesMap.put("publicDate", dataManagementPlans.get(0).getPublicDate());
}
if (dataManagementPlans.get(0).getCosts() != null) {
extraPropertiesMap.put("costs", mapper.readValue(dataManagementPlans.get(0).getCosts(), ArrayList.class));
}
dm.setExtraProperties(extraPropertiesMap);
//createOrUpdate(apiContext, dm, principal);
DMP dmp = this.createOrUpdate(dm, principal);
@ -1622,7 +1722,7 @@ public class DataManagementPlanManager {
// datasets.add(new DatasetListingModel().fromDataModel(dataset));
}
logger.info(dm.toString());
//logger.info(dm.toString());
}
return dataManagementPlans;
@ -1654,6 +1754,8 @@ public class DataManagementPlanManager {
databaseRepository.getDmpDao().createOrUpdate(dmp);
assignUser(dmp, me);
if (this.apiContext.getOperationsContext().getElasticRepository().getDmpRepository().getClient() != null) {
UUID dmpId = dmp.getId();
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(dmp);
}
dmp.getDataset().forEach(dataset -> {
@ -1676,13 +1778,13 @@ public class DataManagementPlanManager {
try {
List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset);
if (elastic != null) {
tags = elastic.getTags();
DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset);
datasetWizardModel.setTags(tags);
datasetManager.getTagsFromProfile(datasetWizardModel, dataset);
datasetManager.updateTags(dataset, datasetWizardModel.getTags());
}
datasetManager.getTagsFromProfile(datasetWizardModel, dataset);
datasetManager.updateTags(dataset, datasetWizardModel.getTags());
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
@ -1745,6 +1847,9 @@ public class DataManagementPlanManager {
try {
if (dmp.getUsers() != null) {
logger.info(dmp.getUsers().toString());
} else {
UUID dmpId = dmp.getId();
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
}
this.updateIndex(dmp);
} catch (IOException e) {
@ -1762,6 +1867,8 @@ public class DataManagementPlanManager {
if (elastic != null) {
tags = elastic.getTags();
}
UUID dmpId = dataset.getDmp().getId();
dataset.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -1770,7 +1877,7 @@ public class DataManagementPlanManager {
});
}
private void updateIndex(DMP dmp) throws IOException {
public void updateIndex(DMP dmp) throws IOException {
DmpMapper mapper = new DmpMapper(apiContext, datasetManager);
Dmp elastic = mapper.toElastic(dmp);
apiContext.getOperationsContext().getElasticRepository().getDmpRepository().createOrUpdate(elastic);
@ -1782,6 +1889,8 @@ public class DataManagementPlanManager {
List<DMP> dmps = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList();
dmps.forEach(dmp -> {
try {
UUID dmpId = dmp.getId();
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
this.updateIndex(dmp);
} catch (IOException e) {
logger.error(e.getMessage(), e);
@ -2016,10 +2125,10 @@ public class DataManagementPlanManager {
}
dataBuilder.append(" }\n").append("}");
createData = dataBuilder.toString();
ObjectMapper mapper = new ObjectMapper();
/*ObjectMapper mapper = new ObjectMapper();
mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
JsonNode createDataJSON = mapper.readTree(createData);
HttpEntity<JsonNode> request = new HttpEntity<>(createDataJSON, headers);
JsonNode createDataJSON = mapper.readTree(createData);*/
HttpEntity<String> request = new HttpEntity<>(createData, headers);
Map createResponse = null;
LinkedHashMap<String, String> links = null;
String previousDOI = this.getPreviousDOI(dmp.getGroupId(), dmp.getId());
@ -2081,19 +2190,19 @@ public class DataManagementPlanManager {
if (unpublishedUrl == null) {
// Second step, add the file to the entry.
FileEnvelope file = getWordDocument(id.toString(), principal, configLoader);
/*String name = file.getFilename().substring(0, file.getFilename().length() - 5);
File pdfFile = datasetManager.convertToPDF(file, environment);
String fileName = name + ".pdf";*/
FileSystemResource fileSystemResource = new FileSystemResource(file.getFile());
String name = file.getFilename().substring(0, file.getFilename().length() - 5);
File pdfFile = PDFUtils.convertToPDF(file, environment);
String fileName = name + ".pdf";
FileSystemResource fileSystemResource = new FileSystemResource(pdfFile);
HttpEntity<FileSystemResource> addFileMapRequest = new HttpEntity<>(fileSystemResource, null);
String addFileUrl = links.get("bucket") + "/" + file.getFilename() + "?access_token=" + zenodoToken;
String addFileUrl = links.get("bucket") + "/" + fileName + "?access_token=" + zenodoToken;
restTemplate.put(addFileUrl, addFileMapRequest);
Files.deleteIfExists(file.getFile().toPath());
ResponseEntity<byte[]> jsonFile;
try {
jsonFile = getRDAJsonDocument(id.toString(), datasetManager, principal);
jsonFile = getRDAJsonDocument(id.toString(), principal);
} catch (Exception e) {
throw e;
}
@ -2104,7 +2213,9 @@ public class DataManagementPlanManager {
jsonFos.flush();
}
fileSystemResource = new FileSystemResource(tempJsonFile);
addFileMapRequest = new HttpEntity<>(fileSystemResource, null);
HttpHeaders jsonHeaders = new HttpHeaders();
jsonHeaders.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
addFileMapRequest = new HttpEntity<>(fileSystemResource, jsonHeaders);
String jsonFileName = jsonFile.getHeaders().get("Content-Disposition").get(0).substring(jsonFile.getHeaders().get("Content-Disposition").get(0).lastIndexOf('=') + 1);
addFileUrl = links.get("bucket") + "/" + jsonFileName + "?access_token=" + zenodoToken;
restTemplate.put(addFileUrl, addFileMapRequest);
@ -2168,51 +2279,6 @@ public class DataManagementPlanManager {
}
public long countAllDrafts() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllFinalized() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllPublished() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllDoied() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllResearchers() {
ResearcherCriteria criteria = new ResearcherCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
}
public long countAllProjects() {
ProjectCriteria criteria = new ProjectCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
}
public long countAllFunders() {
FunderCriteria criteria = new FunderCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
}
public long countAllGrants() {
GrantCriteria criteria = new GrantCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
}
/*public DataTableData<DataManagementPlanListingModel> getPublicPaged(DataManagmentPlanPublicTableRequest dataManagementPlanPublicTableRequest, String fieldsGroup, Principal principal) throws Exception {
dataManagementPlanPublicTableRequest.setQuery(databaseRepository.getDmpDao().asQueryable().withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class)));
QueryableList<DMP> items = dataManagementPlanPublicTableRequest.applyCriteria();

View File

@ -104,7 +104,8 @@ public class DataManagementProfileManager {
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentLength(envelope.getFile().length());
responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml");
String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
byte[] content = org.apache.poi.util.IOUtils.toByteArray(resource);

View File

@ -48,6 +48,7 @@ import eu.eudat.models.data.security.Principal;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.Authorities;
import eu.eudat.types.MetricNames;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
@ -66,6 +67,7 @@ import org.springframework.util.LinkedMultiValueMap;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -87,7 +89,6 @@ import java.nio.file.Files;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
@ -109,9 +110,10 @@ public class DatasetManager {
private UserManager userManager;
private ConfigLoader configLoader;
private Environment environment;
private final MetricsManager metricsManager;
@Autowired
public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment) {
public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager) {
this.apiContext = apiContext;
this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository();
@ -119,6 +121,7 @@ public class DatasetManager {
this.userManager = userManager;
this.configLoader = configLoader;
this.environment = environment;
this.metricsManager = metricsManager;
}
public DataTableData<DatasetListingModel> getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception {
@ -156,7 +159,7 @@ public class DatasetManager {
List<eu.eudat.elastic.entities.Dataset> datasets;
try {
datasets = datasetRepository.exists() ?
datasetRepository.query(datasetCriteria) : null;
datasetRepository.queryIds(datasetCriteria) : null;
} catch (Exception ex) {
logger.warn(ex.getMessage(), ex);
datasets = null;
@ -168,15 +171,16 @@ public class DatasetManager {
if (datasets != null) {
if (!datasets.isEmpty()) {
items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
//items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
final List<UUID> datasetIds = datasets.stream().map(datasetE -> UUID.fromString(datasetE.getId())).distinct().collect(Collectors.toList());
items = databaseRepository.getDatasetDao().filterFromElastic(datasetTableRequest.getCriteria(), datasetIds).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
//items.where((builder, root) -> root.get("id").in(datasetIds));
} else {
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
//items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
}
} else {
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
}
List<Integer> roles = new LinkedList<>();
QueryableList<eu.eudat.data.entities.Dataset> pagedItems;
@ -185,9 +189,10 @@ public class DatasetManager {
if (principal.getId() == null) {
throw new UnauthorisedException("You are not allowed to access those datasets");
}
if (datasetTableRequest.getCriteria().getRole() != null)
if (datasetTableRequest.getCriteria().getRole() != null) {
roles.add(datasetTableRequest.getCriteria().getRole());
authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles);
}
authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles).distinct();
pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
} else {
if (principal.getId() != null && datasetTableRequest.getCriteria().getRole() != null) {
@ -214,6 +219,7 @@ public class DatasetManager {
}
public DataTableData<DatasetListingModel> getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception {
Long count = 0L;
DatasetCriteria datasetCriteria = new DatasetCriteria();
datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile());
@ -227,7 +233,8 @@ public class DatasetManager {
List<eu.eudat.elastic.entities.Dataset> datasets;
try {
datasets = datasetRepository.exists() ?
datasetRepository.query(datasetCriteria) : new LinkedList<>();
datasetRepository.queryIds(datasetCriteria) : new LinkedList<>();
count = datasetRepository.exists() ? datasetRepository.count(datasetCriteria) : 0L;
} catch (Exception ex) {
logger.warn(ex.getMessage());
datasets = null;
@ -253,22 +260,22 @@ public class DatasetManager {
return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
});
}
String[] strings = new String[1];
strings[0] = "-dmp:publishedAt|join|";
List<String> strings = new ArrayList<>();
strings.add("-dmp:publishedAt|join|");
datasetTableRequest.getOrderings().setFields(strings);
if (count == 0L) {
count = items.count();
}
QueryableList<eu.eudat.data.entities.Dataset> pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
CompletableFuture<List<DatasetListingModel>> itemsFuture = pagedItems.
selectAsync(this::mapModel).whenComplete((resultList, throwable) -> {
dataTable.setData(resultList.stream().filter(Objects::nonNull).collect(Collectors.toList()));
});
List<DatasetListingModel> datasetListis = pagedItems.
select(this::mapModel);
CompletableFuture countFuture = pagedItems.countAsync().whenComplete((count, throwable) -> {
dataTable.setTotalCount(count);
});
dataTable.setData(datasetListis.stream().filter(Objects::nonNull).collect(Collectors.toList()));
CompletableFuture.allOf(itemsFuture, countFuture).join();
dataTable.setTotalCount(count);
//CompletableFuture.allOf(countFuture).join();
return dataTable;
}
@ -549,54 +556,6 @@ public class DatasetManager {
return fileEnvelope;
}
public File convertToPDF(FileEnvelope file, Environment environment) throws IOException, InterruptedException {
LinkedMultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
String uuid = UUID.randomUUID().toString();
map.add("files", new FileSystemResource(file.getFile()));
map.add("filename", uuid + ".pdf");
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
headers.add("Content-disposition", "attachment; filename=" + uuid + ".pdf");
headers.add("Content-type", "application/pdf");
HttpEntity<LinkedMultiValueMap<String, Object>> requestEntity = new HttpEntity<LinkedMultiValueMap<String, Object>>(
map, headers);
byte[] queueResult = new RestTemplate().postForObject(environment.getProperty("pdf.converter.url") + "convert/office"
, requestEntity, byte[].class);
File resultPdf = new File(environment.getProperty("temp.temp") + uuid + ".pdf");
FileOutputStream output = new FileOutputStream(resultPdf);
IOUtils.write(queueResult, output);
output.close();
Files.deleteIfExists(file.getFile().toPath());
return resultPdf;
}
private File extractFromZip(File file, String filename) throws IOException {
byte[] buffer = new byte[1024];
File newFile = new File(filename);
ZipInputStream zis = new ZipInputStream(new FileInputStream(file));
ZipEntry zipEntry = zis.getNextEntry();
while (zipEntry != null) {
String zippedFileName = zipEntry.getName();
if (zippedFileName.equals("pdf")) {
FileOutputStream fos = new FileOutputStream(newFile);
int len;
while ((len = zis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
fos.close();
zipEntry = zis.getNextEntry();
}
}
zis.closeEntry();
zis.close();
return newFile;
}
public eu.eudat.data.entities.Dataset createOrUpdate(DatasetWizardModel datasetWizardModel, Principal principal) throws Exception {
Boolean sendNotification = false;
Dataset tempDataset = null;
@ -613,7 +572,7 @@ public class DatasetManager {
sendNotification = true;
}
} else {
MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft");
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
}
if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null)
throw new Exception("DMP is finalized, therefore Dataset cannot be edited.");
@ -639,6 +598,8 @@ public class DatasetManager {
}
dataset1.setProfile(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(datasetWizardModel.getProfile().getId()));
// datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset1));
UUID dmpId = dataset1.getDmp().getId();
dataset1.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
updateTags(dataset1, datasetWizardModel.getTags());
if (sendNotification) {
if (dataset1.getStatus() != Dataset.Status.FINALISED.getValue()) {
@ -691,6 +652,9 @@ public class DatasetManager {
datasetProfileValidators.add(node.getNodeValue());
}
expression = "//validation/@type[.=1]/ancestor::fieldSet";
nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);
JSONObject obj = new JSONObject(dataset.getProperties());
VisibilityRuleService visibilityRuleService = this.apiContext.getUtilitiesService().getVisibilityRuleService();
@ -702,14 +666,37 @@ public class DatasetManager {
for (String validator : datasetProfileValidators) {
if ((obj.getString(validator) == null || obj.getString(validator).trim().isEmpty()) && visibilityRuleService.isElementVisible(validator)) {
if ((obj.has(validator) && (obj.getString(validator) == null || obj.getString(validator).trim().isEmpty())) && isElementVisible(nodeList, validator, visibilityRuleService)) {
throw new Exception("Field value of " + validator + " must be filled.");
}
}
}
private boolean isElementVisible(NodeList nodeList, String id, VisibilityRuleService visibilityRuleService) {
Element fieldSet = null;
for (int i = 0; i < nodeList.getLength(); i++) {
Node node = nodeList.item(i);
for (int j = 0; j < node.getChildNodes().getLength(); j++) {
Node fcnode = node.getChildNodes().item(j);
if (fcnode.getNodeName().equals("fields")) {
for(int k = 0; k < fcnode.getChildNodes().getLength(); k++) {
Node scnode = fcnode.getChildNodes().item(k);
if (scnode.getNodeName().equals("field") && scnode.getAttributes().getNamedItem("id").getNodeValue().equals(id)) {
fieldSet = (Element) node;
}
}
}
}
}
if (fieldSet != null) {
return visibilityRuleService.isElementVisible(id) && visibilityRuleService.isElementVisible(fieldSet.getAttribute("id"));
} else {
return visibilityRuleService.isElementVisible(id);
}
}
private String propertiesModelToString(PagedDatasetProfile pagedDatasetProfile) {
Map<String, Object> values = new HashMap();
Map<String, Object> values = new LinkedHashMap<>();
pagedDatasetProfile.toMap(values);
JSONObject jobject = new JSONObject(values);
return jobject.toString();
@ -857,7 +844,7 @@ public class DatasetManager {
if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue())
throw new Exception("You cannot make public a Dataset That Has not Been Finalised");
datasetDao.createOrUpdate(dataset);
MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published");
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
}
public ResponseEntity<byte[]> getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException {
@ -868,7 +855,8 @@ public class DatasetManager {
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentLength(envelope.getFile().length());
responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml");
String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
@ -946,7 +934,7 @@ public class DatasetManager {
createServicesIfTheyDontExist(entity);
createExternalDatasetsIfTheyDontExist(entity);
MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft");
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity);
}
@ -1028,6 +1016,8 @@ public class DatasetManager {
datasetEntities.forEach(datasetEntity -> {
try {
eu.eudat.elastic.entities.Dataset dataset = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(datasetEntity.getId().toString());
UUID dmpId = datasetEntity.getDmp().getId();
datasetEntity.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
updateTags(datasetEntity, dataset != null ? dataset.getTags() : null);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -1052,22 +1042,26 @@ public class DatasetManager {
ObjectMapper mapper = new ObjectMapper();
String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition());
JsonNode propertiesJson = mapper.readTree(json);
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
DatasetCriteria criteria = new DatasetCriteria();
criteria.setHasTags(true);
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
Set<JsonNode> tagNodes = new HashSet<>();
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true));
tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "rdaProperty", "dataset.keyword"));
if (!tagNodes.isEmpty()) {
tagNodes.forEach(node -> {
JsonNode value = node.get("value");
String stringValue = value.asText().replaceAll("=", ":");
JSONArray values = new JSONArray(stringValue);
if (values != null) {
values.iterator().forEachRemaining(element -> {
Map<String, Object> data = ((JSONObject) element).toMap();
this.addTag(tags, wizardModel.getTags(), data.get("id").toString(), data.get("name").toString());
});
} else {
this.addTag(tags, wizardModel.getTags(), "", value.asText());
if (!value.toString().equals("\"\"")) {
String stringValue = value.toString().replaceAll("=", ":");
JSONArray values = new JSONArray(stringValue);
if (values != null) {
values.iterator().forEachRemaining(element -> {
Map<String, Object> data = ((JSONObject) element).toMap();
this.addTag(tags, wizardModel.getTags(), data.get("id").toString(), data.get("name").toString());
});
} else {
this.addTag(tags, wizardModel.getTags(), "", value.asText());
}
}
});
}
@ -1088,10 +1082,10 @@ public class DatasetManager {
@Transactional
private DatasetListingModel mapModel(Dataset item) {
if (item.getProfile() == null)
return null;
/*if (item.getProfile() == null)
return null;*/
DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item);
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
/*DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId()));
List<DatasetProfile> profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
boolean islast = false;
@ -1099,31 +1093,7 @@ public class DatasetManager {
profiles = profiles.stream().sorted(Comparator.comparing(DatasetProfile::getVersion)).collect(Collectors.toList());
islast = profiles.get(0).getId().equals(item.getProfile().getId());
}
listingModel.setProfileLatestVersion(islast);
listingModel.setProfileLatestVersion(islast);*/
return listingModel;
}
public long countAllDraft() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllFinalized() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllPublic() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setIsPublic(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllWithDoi() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setHasDoi(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
}

View File

@ -24,9 +24,11 @@ import eu.eudat.models.data.entities.xmlmodels.datasetprofiledefinition.Field;
import eu.eudat.models.data.externaldataset.ExternalAutocompleteFieldModel;
import eu.eudat.models.data.helpers.common.DataTableData;
import eu.eudat.models.data.listingmodels.UserInfoListingModel;
import eu.eudat.models.data.mail.SimpleMail;
import eu.eudat.models.data.security.Principal;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.Authorities;
import eu.eudat.types.MetricNames;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -48,6 +50,7 @@ import javax.xml.xpath.*;
import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@ -59,12 +62,14 @@ public class DatasetProfileManager {
private DatabaseRepository databaseRepository;
private Environment environment;
private List<String> cache;
private final MetricsManager metricsManager;
@Autowired
public DatasetProfileManager(ApiContext apiContext, Environment environment) {
public DatasetProfileManager(ApiContext apiContext, Environment environment, MetricsManager metricsManager) {
this.apiContext = apiContext;
this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
this.environment = environment;
this.metricsManager = metricsManager;
this.cache = new ArrayList<>();
}
@ -92,7 +97,7 @@ public class DatasetProfileManager {
DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(UUID.fromString(id));
apiContext.getOperationsContext().getDatabaseRepository().detachEntity(profile);
profile.setId(null);
MetricsManager.increaseValue("argos_dataset_templates", 1, "total");
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
return profile;
}
@ -102,7 +107,8 @@ public class DatasetProfileManager {
if (principal.getAuthz().contains(Authorities.ADMIN)) {
authItems = items;
} else if (principal.getAuthz().contains(Authorities.DATASET_PROFILE_MANAGER)) {
authItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getAuthenticated(items, principal.getId(), null);
List<Integer> roles = Collections.singletonList(1);
authItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getAuthenticated(items, principal.getId(), roles);
}
QueryableList<DatasetProfile> pagedItems = PaginationManager.applyPaging(authItems, datasetProfileTableRequestItem);
List<DatasetProfileListingModel> datasetProfiles = pagedItems.select(item -> new DatasetProfileListingModel().fromDataModel(item));
@ -170,7 +176,7 @@ public class DatasetProfileManager {
response = restTemplate.exchange(url, HttpMethod.GET, entity, Object.class);
jsonContext = JsonPath.parse(response.getBody());
jsonItems = jsonContext.read(singleData.getOptionsRoot() + "['" + singleData.getAutoCompleteOptions().getLabel() + "','" + singleData.getAutoCompleteOptions().getValue() + "','" + singleData.getAutoCompleteOptions().getSource() + "','" + "uri" + "']");
jsonItems.forEach(item -> result.add(new ExternalAutocompleteFieldModel(item.get(singleData.getAutoCompleteOptions().getValue()), item.get(singleData.getAutoCompleteOptions().getLabel()), item.get(singleData.getAutoCompleteOptions().getSource()) != null ? item.get(singleData.getAutoCompleteOptions().getSource()) : singleData.getAutoCompleteOptions().getSource(), item.get("uri"))));
jsonItems.forEach(item -> result.add(new ExternalAutocompleteFieldModel(parseItem(item.get(singleData.getAutoCompleteOptions().getValue())), parseItem(item.get(singleData.getAutoCompleteOptions().getLabel())), item.get(singleData.getAutoCompleteOptions().getSource()) != null ? parseItem(item.get(singleData.getAutoCompleteOptions().getSource())) : singleData.getAutoCompleteOptions().getSource(), parseItem(item.get("uri")))));
break;
case CACHED:
headers.setAccept(Collections.singletonList(MediaType.valueOf("text/plain; charset=utf-8")));
@ -196,6 +202,20 @@ public class DatasetProfileManager {
//return result;
}
private String parseItem(Object item) {
if (item instanceof String) {
return (String) item;
}
if (item instanceof List) {
List listedItems = (List) item;
return parseItem(listedItems.get(0));
}
if (item instanceof Map) {
return (String) ((Map)item).get("$");
}
return item.toString();
}
public ResponseEntity<byte[]> getDocument(eu.eudat.models.data.user.composite.DatasetProfile datasetProfile, String label) throws IllegalAccessException, IOException, InstantiationException {
FileEnvelope envelope = getXmlDocument(datasetProfile, label);
InputStream resource = new FileInputStream(envelope.getFile());
@ -204,7 +224,8 @@ public class DatasetProfileManager {
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentLength(envelope.getFile().length());
responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml");
String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
@ -232,7 +253,7 @@ public class DatasetProfileManager {
File localFile = convert(multiPartFile);
eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileModel.DatasetProfile profile = xmlBuilder.build(localFile);
Files.deleteIfExists(localFile.toPath());
MetricsManager.increaseValue("argos_dataset_templates", 1, "total");
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
return profile;
} catch (IOException e) {
logger.error(e.getMessage(), e);
@ -281,17 +302,29 @@ public class DatasetProfileManager {
if (entity.getUsers() == null) {
entity.setUsers(new HashSet<>());
}
model.getUsers().stream().filter(userInfoListingModel -> entity.getUsers().stream().filter(userDatasetProfile -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0).forEach(userInfoListingModel -> {
model.getUsers().stream().filter(userInfoListingModel -> entity.getUsers().stream()
.filter(userDatasetProfile -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0)
.forEach(userInfoListingModel -> {
UserDatasetProfile userDatasetProfile1 = new UserDatasetProfile();
userDatasetProfile1.setDatasetProfile(entity);
UserInfo userInfo1 = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(userInfoListingModel.getId());
userDatasetProfile1.setUser(userInfo1);
userDatasetProfile1.setRole(1);
apiContext.getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile1);
sendJoinMail(userDatasetProfile1);
});
entity.getUsers().stream().filter(userDatasetProfile -> model.getUsers().stream()
.filter(userInfoListingModel -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() > 0
&& userDatasetProfile.getRole() == 2).forEach(userDatasetProfile -> {
userDatasetProfile.setRole(1);
apiContext.getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
sendJoinMail(userDatasetProfile);
});
}
if (entity.getUsers() != null && !entity.getUsers().isEmpty()) {
entity.getUsers().stream().filter(userDatasetProfile -> model.getUsers().stream().filter(userInfoListingModel -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0).forEach(userDatasetProfile -> {
entity.getUsers().stream().filter(userDatasetProfile -> model.getUsers().stream()
.filter(userInfoListingModel -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0)
.forEach(userDatasetProfile -> {
userDatasetProfile.setRole(2);
apiContext.getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
});
@ -312,24 +345,21 @@ public class DatasetProfileManager {
}
}
public long countAlldraft() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
private void sendJoinMail(UserDatasetProfile userDatasetProfile) {
SimpleMail mail = new SimpleMail();
mail.setSubject(environment.getProperty("admin.mail.subject").replace( "{templateName}", userDatasetProfile.getDatasetProfile().getLabel()));
String content = apiContext.getUtilitiesService().getMailService().getMailTemplateContent(environment.getProperty("email.dataset.template"));
content = content.replace("{recipient}", userDatasetProfile.getUser().getName());
content = content.replace("{templateName}", userDatasetProfile.getDatasetProfile().getLabel());
content = content.replace("{host}", this.environment.getProperty("dmp.domain"));
content = content.replace("{templateID}", userDatasetProfile.getDatasetProfile().getId().toString());
mail.setContent(content);
mail.setTo(userDatasetProfile.getUser().getEmail());
try {
apiContext.getUtilitiesService().getMailService().sendSimpleMail(mail);
} catch (Exception ex) {
logger.error(ex.getMessage(), ex);
}
public long countAllFinalized() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
@Transactional
public long countAllUsed() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
criteria.setAllVersions(false);
List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
return datasetProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).filter(datasetProfile -> !datasetProfile.getDataset().isEmpty()).count();
}
}

View File

@ -1,15 +1,17 @@
package eu.eudat.logic.managers;
import eu.eudat.data.dao.criteria.DatasetProfileCriteria;
import eu.eudat.data.dao.entities.DMPDao;
import eu.eudat.data.dao.entities.DatasetProfileDao;
import eu.eudat.data.entities.DMP;
import eu.eudat.data.entities.Dataset;
import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.data.entities.UserInfo;
import eu.eudat.data.query.items.item.dataset.DatasetWizardAutocompleteRequest;
import eu.eudat.data.query.items.item.datasetprofile.DatasetProfileWizardAutocompleteRequest;
import eu.eudat.exceptions.datasetwizard.DatasetWizardCannotUnlockException;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.models.data.datasetwizard.DataManagentPlanListingModel;
import eu.eudat.data.query.items.item.datasetprofile.DatasetProfileWizardAutocompleteRequest;
import eu.eudat.data.query.items.item.dataset.DatasetWizardAutocompleteRequest;
import eu.eudat.models.data.dmp.AssociatedProfile;
import eu.eudat.models.data.dmp.DataManagementPlan;
import eu.eudat.models.data.security.Principal;
@ -19,6 +21,7 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
public class DatasetWizardManager {
@ -31,12 +34,18 @@ public class DatasetWizardManager {
return dataManagementPlans;
}
public static List<AssociatedProfile> getAvailableProfiles(DMPDao dmpRepository, DatasetProfileWizardAutocompleteRequest datasetProfileWizardAutocompleteRequest) throws InstantiationException, IllegalAccessException {
public static List<AssociatedProfile> getAvailableProfiles(DMPDao dmpRepository, DatasetProfileDao profileDao, DatasetProfileWizardAutocompleteRequest datasetProfileWizardAutocompleteRequest) throws InstantiationException, IllegalAccessException {
DataManagementPlan dataManagementPlan = new DataManagementPlan().fromDataModel(dmpRepository.find(datasetProfileWizardAutocompleteRequest.getCriteria().getId()));
if (dataManagementPlan.getProfiles() == null || dataManagementPlan.getProfiles().isEmpty()) {
return new LinkedList<>();
}
List<AssociatedProfile> profiles = dataManagementPlan.getProfiles();
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setIds(dataManagementPlan.getProfiles().stream().map(AssociatedProfile::getId).collect(Collectors.toList()));
List<DatasetProfile> datasetProfiles = profileDao.getWithCriteria(criteria).toList();
criteria.setIds(null);
criteria.setGroupIds(datasetProfiles.stream().map(DatasetProfile::getGroupId).collect(Collectors.toList()));
datasetProfiles = profileDao.getWithCriteria(criteria).toList();
List<AssociatedProfile> profiles = datasetProfiles.stream().map(profile -> new AssociatedProfile().fromData(profile)).collect(Collectors.toList());
return profiles;
}

View File

@ -1,159 +0,0 @@
package eu.eudat.logic.managers;
import eu.eudat.data.dao.entities.DatasetDao;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.forms.VisibilityRuleService;
import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
import eu.eudat.logic.utilities.documents.word.WordBuilder;
import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder;
import eu.eudat.models.HintedModelFactory;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.*;
import org.springframework.http.converter.ByteArrayHttpMessageConverter;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.web.client.RestTemplate;
import java.io.*;
import java.net.URL;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
/**
* Created by ikalyvas on 10/16/2018.
*/
@Service
public class DocumentManager {
private static final Logger logger = LoggerFactory.getLogger(DocumentManager.class);
private ApiContext context;
private DatasetManager datasetManager;
private ConfigLoader configLoader;
private Environment environment;
public DocumentManager(ApiContext context, DatasetManager datasetManager, ConfigLoader configLoader, Environment environment) {
this.context = context;
this.datasetManager = datasetManager;
this.configLoader = configLoader;
this.environment = environment;
}
public File getWordDocument(ConfigLoader configLoader, DatasetDao datatasetRepository, String id, VisibilityRuleService visibilityRuleService) throws InstantiationException, IllegalAccessException, IOException {
WordBuilder wordBuilder = new WordBuilder();
DatasetWizardModel dataset = new DatasetWizardModel();
XWPFDocument document = configLoader.getDocument();
eu.eudat.data.entities.Dataset datasetEntity = datatasetRepository.find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
Map<String, Object> properties = new HashMap<>();
if (datasetEntity.getProperties() != null) {
JSONObject jobject = new JSONObject(datasetEntity.getProperties());
properties = jobject.toMap();
}
PagedDatasetProfile pagedDatasetProfile = datasetManager.getPagedProfile(dataset, datasetEntity);
visibilityRuleService.setProperties(properties);
visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
wordBuilder.build(document, pagedDatasetProfile, visibilityRuleService);
File exportFile = new File(dataset.getLabel() + ".docx");
FileOutputStream out = new FileOutputStream(exportFile);
document.write(out);
out.close();
return exportFile;
}
public FileEnvelope getXmlDocument(eu.eudat.data.entities.Dataset datasetEntity, String id, VisibilityRuleService visibilityRuleService) throws InstantiationException, IllegalAccessException, IOException {
ExportXmlBuilder xmlBuilder = new ExportXmlBuilder();
DatasetWizardModel dataset = new DatasetWizardModel();
Map<String, Object> properties = new HashMap<>();
if (datasetEntity.getProperties() != null) {
JSONObject jobject = new JSONObject(datasetEntity.getProperties());
properties = jobject.toMap();
}
PagedDatasetProfile pagedDatasetProfile = datasetManager.getPagedProfile(dataset, datasetEntity);
visibilityRuleService.setProperties(properties);
visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
File file = xmlBuilder.build(pagedDatasetProfile, UUID.fromString(id), visibilityRuleService, environment);
FileEnvelope fileEnvelope = new FileEnvelope();
fileEnvelope.setFile(file);
fileEnvelope.setFilename(datasetEntity.getLabel());
return fileEnvelope;
}
public File convertToPDF(File file, Environment environment, String label) throws IOException, InterruptedException {
LinkedMultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
map.add("file", new FileSystemResource(file));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
headers.add("Content-disposition", "attachment; filename=" + label + ".docx");
headers.add("Content-type", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
HttpEntity<LinkedMultiValueMap<String, Object>> requestEntity = new HttpEntity<LinkedMultiValueMap<String, Object>>(
map, headers);
Map queueResult = new RestTemplate().postForObject(
environment.getProperty("pdf.converter.url") +
"api/v1/", requestEntity, Map.class);
Map mediaResult = new RestTemplate().getForObject(environment.getProperty("pdf.converter.url") +
"/api/v1/" + queueResult.get("id"), Map.class);
logger.info("Status: " + mediaResult.get("status"));
while (!mediaResult.get("status").equals("finished")) {
Thread.sleep(500);
mediaResult = new RestTemplate().getForObject(environment.getProperty("pdf.converter.url") +
"api/v1/" + queueResult.get("id"), Map.class);
logger.info("Polling");
}
RestTemplate restTemplate = new RestTemplate();
restTemplate.getMessageConverters().add(new ByteArrayHttpMessageConverter());
HttpHeaders headers2 = new HttpHeaders();
headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM));
HttpEntity<String> entity = new HttpEntity<String>(headers2);
ResponseEntity<byte[]> response = restTemplate.exchange(environment.getProperty("pdf.converter.url") +
mediaResult.get("result_url"), HttpMethod.GET, entity, byte[].class, "1");
UUID uuid = UUID.randomUUID();
File zip = new File(uuid + ".zip");
if (response.getStatusCode().equals(HttpStatus.OK)) {
FileOutputStream output = new FileOutputStream(zip);
IOUtils.write(response.getBody(), output);
}
return extractFromZip(zip, label + ".pdf");
}
private File extractFromZip(File file, String filename) throws IOException {
byte[] buffer = new byte[1024];
File newFile = new File(filename);
ZipInputStream zis = new ZipInputStream(new FileInputStream(file));
ZipEntry zipEntry = zis.getNextEntry();
while (zipEntry != null) {
String zippedFileName = zipEntry.getName();
if (zippedFileName.equals("pdf")) {
FileOutputStream fos = new FileOutputStream(newFile);
int len;
while ((len = zis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
fos.close();
zipEntry = zis.getNextEntry();
}
}
zis.closeEntry();
zis.close();
return newFile;
}
}

View File

@ -3,6 +3,7 @@ package eu.eudat.logic.managers;
import eu.eudat.data.entities.DMP;
import eu.eudat.data.entities.UserDMP;
import eu.eudat.data.entities.UserInfo;
import eu.eudat.data.query.items.item.userinfo.UserInfoRequestItem;
import eu.eudat.exceptions.security.UnauthorisedException;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
@ -17,6 +18,7 @@ import org.springframework.stereotype.Component;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import java.io.IOException;
import java.io.StringReader;
import java.util.*;
import java.util.stream.Collectors;
@ -26,10 +28,12 @@ import java.util.stream.Stream;
public class InvitationsManager {
private ApiContext apiContext;
private DataManagementPlanManager dataManagementPlanManager;
@Autowired
public InvitationsManager(ApiContext apiContext) {
public InvitationsManager(ApiContext apiContext, DataManagementPlanManager dataManagementPlanManager) {
this.apiContext = apiContext;
this.dataManagementPlanManager = dataManagementPlanManager;
}
public void inviteUsers(Invitation invitation, Principal principal) throws Exception {
@ -79,7 +83,24 @@ public class InvitationsManager {
return userModels;
}
public UUID assignUserAcceptedInvitation(UUID invitationID, Principal principal) throws UnauthorisedException, JAXBException {
public List<UserInfoInvitationModel> getUsersWithCriteria(Principal principal, UserInfoRequestItem userInfoRequestItem) throws IllegalAccessException, InstantiationException {
List<UserInfo> users = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao()
.getAuthenticated(apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable()
.where(((builder, root) ->
builder.notEqual(root.get("status"), DMP.DMPStatus.DELETED.getValue()))), principal.getId(), Stream.of(0, 1).collect(Collectors.toList()))
.toList().stream().map(DMP::getUsers).flatMap(Collection::stream).map(UserDMP::getUser)
.filter(userInfo -> !userInfo.getId().equals(principal.getId())).filter(StreamDistinctBy.distinctByKey(UserInfo::getId))
.filter(userInfo -> (userInfoRequestItem == null || userInfoRequestItem.getCriteria() == null || userInfoRequestItem.getCriteria().getLike() == null
|| userInfo.getName().toLowerCase().contains(userInfoRequestItem.getCriteria().getLike().toLowerCase())
|| (userInfo.getEmail().toLowerCase().contains(userInfoRequestItem.getCriteria().getLike().toLowerCase()))))
.collect(Collectors.toList());
// .where((builder, root) -> builder.like(builder.upper(root.get("name")), "%" + userInfoRequestItem.getCriteria().getLike().toUpperCase() + "%"))
List<UserInfoInvitationModel> userModels = users.stream().map(userInfo -> new UserInfoInvitationModel().fromDataModel(userInfo)).collect(Collectors.toList());
return userModels;
}
public UUID assignUserAcceptedInvitation(UUID invitationID, Principal principal) throws UnauthorisedException, JAXBException, IOException {
eu.eudat.data.entities.Invitation invitation = apiContext.getOperationsContext().getDatabaseRepository().getInvitationDao().find(invitationID);
if (invitation == null)
throw new UnauthorisedException("There is no Data Management Plan assigned to this Link");
@ -98,6 +119,8 @@ public class InvitationsManager {
apiContext.getUtilitiesService().getInvitationService().assignToDmp(apiContext.getOperationsContext().getDatabaseRepository().getDmpDao(), existingUserDMP, datamanagementPlan);
invitation.setAcceptedInvitation(true);
apiContext.getOperationsContext().getDatabaseRepository().getInvitationDao().createOrUpdate(invitation);
datamanagementPlan.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), datamanagementPlan.getId())).toList()));
dataManagementPlanManager.updateIndex(datamanagementPlan);
return datamanagementPlan.getId();
}
} else {
@ -121,6 +144,8 @@ public class InvitationsManager {
apiContext.getUtilitiesService().getInvitationService().assignToDmp(apiContext.getOperationsContext().getDatabaseRepository().getDmpDao(), userDMP, datamanagementPlan);
invitation.setAcceptedInvitation(true);
apiContext.getOperationsContext().getDatabaseRepository().getInvitationDao().createOrUpdate(invitation);
datamanagementPlan.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), datamanagementPlan.getId())).toList()));
dataManagementPlanManager.updateIndex(datamanagementPlan);
return datamanagementPlan.getId();
}
return invitation.getDmp().getId();

View File

@ -42,7 +42,7 @@ public class LicenseManager {
ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
licenseModels.addAll(remoteRepos.stream().map(item -> mapper.convertValue(item, LicenseModel.class)).collect(Collectors.toList()));
licenseModels = licenseModels.stream().filter(licenseModel -> licenseModel.getName().contains(query)).collect(Collectors.toList());
licenseModels = licenseModels.stream().filter(licenseModel -> licenseModel.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
return licenseModels;
}
}

View File

@ -22,7 +22,7 @@ public class LocalFetchManager {
public List<LocalFetchModel> getCurrency(String query) throws Exception {
List<Map<String, String>> data = localFetcher.retrieveCurrency();
List<LocalFetchModel> result = data.stream().map(entry -> new LocalFetchModel(entry.get("name"), entry.get("value"))).collect(Collectors.toList());
result = result.stream().filter(localFetchModel -> localFetchModel.getValue() != null).filter(StreamDistinctBy.distinctByKey(LocalFetchModel::getValue)).filter(localFetchModel -> localFetchModel.getName().contains(query)).collect(Collectors.toList());
result = result.stream().filter(localFetchModel -> localFetchModel.getValue() != null).filter(StreamDistinctBy.distinctByKey(LocalFetchModel::getValue)).filter(localFetchModel -> localFetchModel.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
return result;
}
}

View File

@ -1,5 +1,11 @@
package eu.eudat.logic.managers;
import eu.eudat.data.dao.criteria.*;
import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
import eu.eudat.types.MetricNames;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.Gauge;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -13,32 +19,20 @@ import javax.transaction.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class MetricsManager {
private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class);
private static Map<String, Gauge> gauges = Stream.of( new Object[][]{
{"argos_managed_dmps", Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register()},
private final Map<String, Gauge> gauges;
{"argos_funders", Gauge.build().name("argos_funders").help("Number of registered Funders").register()},
{"argos_grants", Gauge.build().name("argos_grants").help("Number of registered Grants").register()},
{"argos_projects", Gauge.build().name("argos_projects").help("Number of registered Projects").register()},
{"argos_researchers", Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register()},
{"argos_managed_dataset_descriptions", Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register()},
{"argos_dataset_templates", Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register()},
{"argos_users", Gauge.build().name("argos_users").help("Number of users").labelNames("type").register()},
{"argos_languages", Gauge.build().name("argos_languages").help("Number of Languages").register()},
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
public static void increaseValue(String name, int amount, String label) {
public void increaseValue(String name, int amount, String label) {
if(label != null) {
gauges.get(name).labels(label).inc(amount);
@ -47,7 +41,7 @@ public class MetricsManager {
}
}
public static void decreaseValue(String name, int amount, String label) {
public void decreaseValue(String name, int amount, String label) {
if(label != null) {
gauges.get(name).labels(label).dec(amount);
} else {
@ -55,7 +49,7 @@ public class MetricsManager {
}
}
public static Integer getValue(String name, String label) {
public Integer getValue(String name, String label) {
if(label != null) {
return Double.valueOf(gauges.get(name).labels(label).get()).intValue();
} else {
@ -63,7 +57,7 @@ public class MetricsManager {
}
}
public static void calculateValue(String name, int amount, String label) {
public void calculateValue(String name, int amount, String label) {
Integer orig = getValue(name, label);
int diff = orig - amount;
if (diff != 0) {
@ -75,19 +69,51 @@ public class MetricsManager {
}
}
private final DatasetManager datasetManager;
private final DataManagementPlanManager dataManagementPlanManager;
private final DatasetProfileManager datasetProfileManager;
private final ApiContext apiContext;
private final UserManager userManager;
private final Environment environment;
@Autowired
public MetricsManager(DatasetManager datasetManager, DataManagementPlanManager dataManagementPlanManager, DatasetProfileManager datasetProfileManager, UserManager userManager, Environment environment) {
this.datasetManager = datasetManager;
this.dataManagementPlanManager = dataManagementPlanManager;
this.datasetProfileManager = datasetProfileManager;
public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) {
this.apiContext = apiContext;
this.userManager = userManager;
this.environment = environment;
registry.clear();
this.gauges = Stream.of( new Object[][]{
{MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())},
{MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())},
{MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())},
{MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
{MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())},
{MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
}
@PostConstruct
@ -95,32 +121,293 @@ public class MetricsManager {
@Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60)
public void init() throws IOException {
logger.info("Start calculating Metrics");
calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDrafts(), "draft");
calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllFinalized(), "finalized");
calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllPublished(), "published");
calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDoied(), "doied");
calculateValue(MetricNames.DMP, (int) countAllDraftDMPs(), MetricNames.DRAFT);
calculateValue(MetricNames.DMP, (int) countAllFinalizedDMPs(), MetricNames.FINALIZED);
calculateValue(MetricNames.DMP, (int) countAllPublishedDMPs(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DMP, (int) countAllDoiedDMPs(), MetricNames.DOIED);
calculateValue("argos_funders", (int) dataManagementPlanManager.countAllFunders(), null);
calculateValue("argos_grants", (int) dataManagementPlanManager.countAllGrants(), null);
calculateValue("argos_projects", (int) dataManagementPlanManager.countAllProjects(), null);
calculateValue("argos_researchers", (int) dataManagementPlanManager.countAllResearchers(), null);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDraftDMPs(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllFinalizedDMPs(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllPublishedDMPs(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDoiedDMPs(true), MetricNames.DOIED);
calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllDraft(), "draft");
calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllFinalized(), "finalized");
calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllPublic(), "published");
calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllWithDoi(), "doied");
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(), MetricNames.DRAFT);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(), MetricNames.FINALIZED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(), MetricNames.DOIED);
calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAlldraft(), "draft");
calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllFinalized(), "active");
calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllUsed(), "used");
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(true), MetricNames.DOIED);
calculateValue("argos_users", (int) userManager.countActiveUsers().intValue(), "loggedin");
calculateValue("argos_users", (int) userManager.countAllUsers().intValue(), "total");
calculateValue(MetricNames.FUNDERS, (int) countAllFunders(), null);
calculateValue(MetricNames.GRANTS, (int) countAllGrants(), null);
calculateValue(MetricNames.PROJECT, (int) countAllProjects(), null);
calculateValue(MetricNames.RESEARCHER, (int) countAllResearchers(), null);
calculateValue(MetricNames.NEXUS + MetricNames.FUNDERS, (int) countAllFunders(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.GRANTS, (int) countAllGrants(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.PROJECT, (int) countAllProjects(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.RESEARCHER, (int) countAllResearchers(true), null);
calculateValue(MetricNames.DATASET, (int) countAllDraftDatasets(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET, (int) countAllFinalizedDatasets(), MetricNames.FINALIZED);
calculateValue(MetricNames.DATASET, (int) countAllPublicDatasets(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DATASET, (int) countAllDatasetsWithDoi(), MetricNames.DOIED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDraftDatasets(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllFinalizedDatasets(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllPublicDatasets(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDatasetsWithDoi(true), MetricNames.DOIED);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(), MetricNames.ACTIVE);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(), MetricNames.USED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(true), MetricNames.ACTIVE);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(true), MetricNames.USED);
calculateValue(MetricNames.USERS, (int) userManager.countActiveUsers().intValue(), MetricNames.LOGGEDIN);
calculateValue(MetricNames.USERS, (int) userManager.countAllUsers().intValue(), MetricNames.TOTAL);
long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count();
calculateValue("argos_languages", (int) files, null);
calculateValue(MetricNames.LANGUAGES, (int) files, null);
calculateValue(MetricNames.INSTALLATIONS, 1, null);
calculateValue(MetricNames.NEXUS + MetricNames.INSTALLATIONS, 1, null);
logger.info("Metrics calculation Completed");
}
private Date getNexusDate() {
try {
return new SimpleDateFormat("yyyy-MM-dd").parse("2021-01-01");
} catch (ParseException e) {
logger.error(e.getLocalizedMessage(), e);
}
return Date.from(LocalDate.of(2021, 1, 1).atStartOfDay(ZoneId.systemDefault()).toInstant());
}
private long countAllDraftDMPs(){
return countAllDraftDMPs(false);
}
private long countAllDraftDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllFinalizedDMPs() {
return countAllFinalizedDMPs(false);
}
private long countAllFinalizedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllPublishedDMPs() {
return countAllPublishedDMPs(false);
}
private long countAllPublishedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllDoiedDMPs() {
return countAllDoiedDMPs(false);
}
private long countAllDoiedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllDraftDMPsWithGrantId() {
return countAllDraftDMPsWithGrantId(false);
}
private long countAllDraftDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllFinalizedDMPsWithGrantId() {
return countAllFinalizedDMPsWithGrantId(false);
}
private long countAllFinalizedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllPublishedDMPsWithGrantId() {
return countAllPublishedDMPsWithGrantId(false);
}
private long countAllPublishedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllDoiedDMPsWithGrantId() {
return countAllDoiedDMPsWithGrantId(false);
}
private long countAllDoiedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllResearchers() {
return countAllResearchers(false);
}
private long countAllResearchers(boolean countNexus) {
ResearcherCriteria criteria = new ResearcherCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
}
private long countAllProjects() {
return countAllProjects(false);
}
private long countAllProjects(boolean countNexus) {
ProjectCriteria criteria = new ProjectCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
}
private long countAllFunders() {
return countAllFunders(false);
}
private long countAllFunders(boolean countNexus) {
FunderCriteria criteria = new FunderCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
}
private long countAllGrants() {
return countAllGrants(false);
}
private long countAllGrants(boolean countNexus) {
GrantCriteria criteria = new GrantCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
}
public long countAllDraftDatasets() {
return countAllDraftDatasets(false);
}
public long countAllDraftDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedDatasets() {
return countAllFinalizedDatasets(false);
}
public long countAllFinalizedDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllPublicDatasets() {
return countAllPublicDatasets(false);
}
public long countAllPublicDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setIsPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDatasetsWithDoi() {
return countAllDatasetsWithDoi(false);
}
public long countAllDatasetsWithDoi(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDraftTemplates() {
return countAllDraftTemplates(false);
}
public long countAllDraftTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedTemplates() {
return countAllFinalizedTemplates(false);
}
public long countAllFinalizedTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
@Transactional
public long countAllUsedTemplates() {
return countAllUsedTemplates(false);
}
@Transactional
public long countAllUsedTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
criteria.setAllVersions(false);
if (countNexus) criteria.setPeriodStart(getNexusDate());
List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
List<DatasetProfile> filteredProfiles = new ArrayList<>();
for (DatasetProfile datasetProfile : datasetProfiles) {
DatasetCriteria datasetCriteria = new DatasetCriteria();
datasetCriteria.setDatasetTemplates(Collections.singletonList(datasetProfile.getId()));
Long datasetCount = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).count();
if (datasetCount > 0) {
filteredProfiles.add(datasetProfile);
}
}
return filteredProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).count();
}
}

View File

@ -1,6 +1,5 @@
package eu.eudat.logic.managers;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.entities.DMP;
@ -12,7 +11,6 @@ import org.springframework.stereotype.Component;
import javax.transaction.Transactional;
import java.io.IOException;
import java.io.Serializable;
import java.text.SimpleDateFormat;
@Component
@ -43,9 +41,8 @@ public class RDAManager {
public DMP convertToEntity(String json, String[] profiles) throws IOException {
ObjectMapper mapper = new ObjectMapper();
mapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"));
Dmp rda = mapper.readValue(json, RDAModel.class).getDmp();
Dmp rda = mapper.readValue(json, RDAModel.class).getDmp();
return dmpRDAMapper.toEntity(rda, profiles);
}
}

View File

@ -167,7 +167,7 @@ public class UserManager {
try {
this.updateDOIToken(ZenodoAccessType.REFRESH_TOKEN, principal.getZenodoRefresh(), this.environment.getProperty("zenodo.login.redirect_uri"), principal);
return true;
}catch (IOException e) {
}catch (Exception e) {
this.deleteDOIToken(principal);
throw new ExpiredTokenException("Zenodo Token is expired.");
}

View File

@ -31,6 +31,7 @@ public class DatasetMapper {
if (tags != null && !tags.isEmpty()) {
DatasetCriteria criteria = new DatasetCriteria();
criteria.setTags(tags);
criteria.setHasTags(true);
List<Tag> tags1 = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
.filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tags.stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
if (tags1.isEmpty()) {

View File

@ -67,6 +67,7 @@ public class DmpMapper {
if (dataset1 != null) {
tags = dataset1.getTags();
}
dataset.setDmp(dmp);
return datasetMapper.toElastic(dataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);

View File

@ -45,7 +45,7 @@ public class DmpCriteriaMapper {
public static List<SortCriteria> toElasticSorting(ColumnOrderings columnOrderings) {
List<SortCriteria> sortCriteria = new ArrayList<>();
if (columnOrderings.getFieldOrderings() != null && columnOrderings.getFieldOrderings().length > 0) {
if (columnOrderings.getFieldOrderings() != null && !columnOrderings.getFieldOrderings().isEmpty()) {
for (Ordering ordering: columnOrderings.getFieldOrderings()) {
SortCriteria sortCriteria1 = new SortCriteria();
sortCriteria1.setFieldName(ordering.getFieldName() + (ordering.getFieldName().contains("label") ?".keyword" : ""));

View File

@ -58,4 +58,16 @@ public class ExternalUrlCriteria {
public ExternalUrlCriteria() {
}
@Override
public String toString() {
return "{" +
"like='" + like + '\'' +
", page='" + page + '\'' +
", pageSize='" + pageSize + '\'' +
", funderId='" + funderId + '\'' +
", path='" + path + '\'' +
", host='" + host + '\'' +
'}';
}
}

View File

@ -8,7 +8,6 @@ import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
@ -26,16 +25,13 @@ import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.*;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
@Service("configLoader")
@Profile("devel")
public class DevelConfigLoader implements ConfigLoader {
private static final Logger logger = LoggerFactory.getLogger(DevelConfigLoader.class);
//@Profile("devel")
public class DefaultConfigLoader implements ConfigLoader {
private static final Logger logger = LoggerFactory.getLogger(DefaultConfigLoader.class);
private ExternalUrls externalUrls;
private List<String> rdaProperties;
@ -53,10 +49,10 @@ public class DevelConfigLoader implements ConfigLoader {
try {
JAXBContext jaxbContext = JAXBContext.newInstance(ExternalUrls.class);
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
is = getClass().getClassLoader().getResource(fileUrl).openStream();
is = getStreamFromPath(fileUrl);
externalUrls = (ExternalUrls) jaxbUnmarshaller.unmarshal(is);
} catch (Exception ex) {
logger.error("Cannot find resource in classpath", ex);
logger.error("Cannot find resource", ex);
} finally {
try {
if (is != null) is.close();
@ -68,10 +64,11 @@ public class DevelConfigLoader implements ConfigLoader {
private void setRdaProperties() {
String filePath = environment.getProperty("configuration.rda");
logger.info("Loaded also config file: " + filePath);
BufferedReader reader;
List<String> rdaList = new LinkedList<>();
try {
reader = new BufferedReader(new FileReader(getClass().getClassLoader().getResource(filePath).getFile()));
reader = new BufferedReader(new InputStreamReader(getStreamFromPath(filePath)));
String line = reader.readLine();
while (line != null) {
rdaList.add(line);
@ -87,9 +84,10 @@ public class DevelConfigLoader implements ConfigLoader {
private void setDocument() {
String filePath = environment.getProperty("configuration.h2020template");
logger.info("Loaded also config file: " + filePath);
InputStream is = null;
try {
is = getClass().getClassLoader().getResource(filePath).openStream();
is = getStreamFromPath(filePath);
this.document = new XWPFDocument(is);
} catch (IOException | NullPointerException e) {
logger.error(e.getMessage(), e);
@ -104,9 +102,10 @@ public class DevelConfigLoader implements ConfigLoader {
private void setConfigurableProviders() {
String filePath = environment.getProperty("configuration.configurable_login_providers");
logger.info("Loaded also config file: " + filePath);
InputStream is = null;
try {
is = getClass().getClassLoader().getResource(filePath).openStream();
is = getStreamFromPath(filePath);
ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
this.configurableProviders = mapper.readValue(is, ConfigurableProviders.class);
} catch (IOException | NullPointerException e) {
@ -145,12 +144,34 @@ public class DevelConfigLoader implements ConfigLoader {
public ExternalUrls getExternalUrls() {
this.setExternalUrls();
if (externalUrls == null) {
externalUrls = new ExternalUrls();
this.setExternalUrls();
} else {
while (externalUrls.getMaxresults() == null) {
try {
Thread.sleep(100L);
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
}
}
}
return externalUrls;
}
public List<String> getRdaProperties() {
this.setRdaProperties();
if (rdaProperties == null) {
rdaProperties = new ArrayList<>();
this.setRdaProperties();
} else {
while (rdaProperties.size() == 0) {
try {
Thread.sleep(100L);
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
}
}
}
return rdaProperties;
}
@ -160,12 +181,34 @@ public class DevelConfigLoader implements ConfigLoader {
}
public ConfigurableProviders getConfigurableProviders() {
this.setConfigurableProviders();
if (configurableProviders == null) {
configurableProviders = new ConfigurableProviders();
this.setConfigurableProviders();
} else {
while (configurableProviders.getProviders().size() == 0) {
try {
Thread.sleep(100L);
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
}
}
}
return configurableProviders;
}
public Map<String, String> getKeyToSourceMap() {
this.setKeyToSourceMap();
if (keyToSourceMap == null) {
keyToSourceMap = new HashMap<>();
this.setKeyToSourceMap();
} else {
while (keyToSourceMap.size() == 0) {
try {
Thread.sleep(100L);
} catch (InterruptedException e) {
logger.error(e.getMessage(), e);
}
}
}
return keyToSourceMap;
}
@ -173,7 +216,7 @@ public class DevelConfigLoader implements ConfigLoader {
InputStream is = null;
Document doc;
try {
is = getClass().getClassLoader().getResource(filePath).openStream();
is = getStreamFromPath(filePath);
DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
doc = documentBuilder.parse(is);
@ -211,4 +254,13 @@ public class DevelConfigLoader implements ConfigLoader {
}
return values;
}
private InputStream getStreamFromPath(String filePath) {
try {
return new FileInputStream(filePath);
} catch (FileNotFoundException e) {
logger.info("loading from classpath");
return getClass().getClassLoader().getResourceAsStream(filePath);
}
}
}

View File

@ -33,6 +33,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/*
@Service("configLoader")
@Profile({ "production", "staging" })
public class ProductionConfigLoader implements ConfigLoader {
@ -236,4 +237,4 @@ public class ProductionConfigLoader implements ConfigLoader {
}
return values;
}
}
}*/

View File

@ -8,7 +8,6 @@ import org.springframework.stereotype.Component;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller;
import java.beans.PropertyDescriptor;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.util.*;
@ -22,6 +21,7 @@ public class LocalFetcher {
JAXBContext context = JAXBContext.newInstance(Config.class);
Unmarshaller unmarshaller = context.createUnmarshaller();
Config config = (Config) unmarshaller.unmarshal(is);
is.close();
ConfigSingle currencyConfig = config.getConfigs().stream().filter(configSingle -> configSingle.getType().equals("currency")).findFirst().get();
return retrieveData(currencyConfig);
@ -39,10 +39,11 @@ public class LocalFetcher {
Unmarshaller unmarshaller = context.createUnmarshaller();
Object object = unmarshaller.unmarshal(is);
is.close();
Method reader = null;
if (configSingle.getParseField() != null && !configSingle.getParseField().isEmpty()) {
reader = new PropertyDescriptor(configSingle.getParseField(), aClass).getReadMethod();
String camelCaseGetter =configSingle.getParseField() != null && !configSingle.getParseField().isEmpty() ? "get" + configSingle.getParseField().substring(0, 1).toUpperCase() + configSingle.getParseField().substring(1) : "";
reader = aClass.getMethod(camelCaseGetter);
}
ObjectMapper objectMapper = new ObjectMapper();
List<Map<String, String>> values = new ArrayList<>();

View File

@ -12,29 +12,20 @@ import eu.eudat.logic.proxy.config.UrlConfiguration;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import net.minidev.json.JSONArray;
import eu.eudat.logic.proxy.fetching.entities.Results;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.*;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller;
import java.beans.PropertyDescriptor;
import java.io.*;
import java.io.File;
import java.io.StringReader;
import java.lang.reflect.Method;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
@ -50,7 +41,7 @@ public class RemoteFetcher {
this.configLoader = configLoader;
}
@Cacheable("repositories")
@Cacheable(value = "repositories", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@ -59,28 +50,28 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("grants")
@Cacheable(value = "grants", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getGrants().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode();
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("projects")
@Cacheable(value = "projects", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("funders")
@Cacheable(value = "funders", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode();
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("organisations")
@Cacheable(value = "organisations", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@ -89,7 +80,7 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("registries")
@Cacheable(value = "registries", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@ -98,7 +89,7 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("services")
@Cacheable(value = "services", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@ -107,7 +98,7 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("researchers")
@Cacheable(value = "researchers", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@ -125,7 +116,7 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}*/
@Cacheable("externalDatasets")
@Cacheable(value = "externalDatasets", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@ -134,7 +125,7 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("licenses")
@Cacheable(value = "licenses", keyGenerator = "externalUrlsKeyGenerator")
public List<Map<String, String>> getlicenses(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getLicenses().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@ -153,22 +144,36 @@ public class RemoteFetcher {
}
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) {
List<Map<String, String>> results = new LinkedList<>();
if (urlConfigs == null || urlConfigs.isEmpty()) return results;
if (urlConfigs == null || urlConfigs.isEmpty()) {
return results;
}
// throw new NoURLFound("No Repository urls found in configuration");
urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
for (UrlConfiguration urlConfig : urlConfigs) {
urlConfigs.forEach(urlConfiguration -> {
ifFunderQueryExist(urlConfiguration, externalUrlCriteria);
if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
try {
results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalUrlCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType()));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
} else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) {
results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalUrlCriteria.getLike()));
}
});
/* for (UrlConfiguration urlConfig : urlConfigs) {
ifFunderQueryExist(urlConfig, externalUrlCriteria);
if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
} else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
}
}
}*/
return results;
}
@ -183,7 +188,7 @@ public class RemoteFetcher {
}
}
private String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria, String firstPage) {
protected String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria, String firstPage) {
String completedPath = path;
if (externalUrlCriteria.getLike() != null) {
if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror")) && externalUrlCriteria.getLike().equals("")) {
@ -233,7 +238,7 @@ public class RemoteFetcher {
return completedPath;
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws HugeResultSet {
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws Exception {
Set<Integer> pages = new HashSet<>();
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
@ -253,6 +258,7 @@ public class RemoteFetcher {
Optional<Results> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType))
.filter(Objects::nonNull)
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
return result1;
@ -264,7 +270,7 @@ public class RemoteFetcher {
}
private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) {
protected Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) {
try {
RestTemplate restTemplate = new RestTemplate();
@ -292,103 +298,23 @@ public class RemoteFetcher {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) {
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
new HashMap<>(1, 1));
results = RemoteFetcherUtils.getFromJsonWithSource(jsonContext, jsonDataPath);
} else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
new HashMap<>(1, 1));
List<Map<String, String>> fixedResults = results.getResults().stream().map(item -> {
for (int i = 0; i < 2; i++) {
String id;
if (i == 0) {
id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", "");
} else {
id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", "");
}
if (!(item.get(id) instanceof String)) {
Object obj = item.get(id);
if (obj instanceof JSONArray) {
JSONArray jarr = (JSONArray) obj;
if (jarr.get(0) instanceof String) {
item.put(id, jarr.get(0).toString());
} else {
for (int j = 0; j < jarr.size(); j++) {
mapToMap(id, (Map<String, String>)jarr.get(j), item, i == 1);
}
}
} else {
if (obj instanceof Map) {
mapToMap(id, (Map<String, String>) obj, item, i == 1);
} else if (obj != null){
item.put(id, obj.toString());
}
}
}
}
return item;
}).collect(Collectors.toList());
results = new Results(fixedResults, new HashMap<>(1, 1));
results = RemoteFetcherUtils.getFromJsonWithParsedData(jsonContext, jsonDataPath);
} else if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getPath()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"),
new HashMap<>(1, 1));
List<Map<String, String>> multiResults = results.results.stream().map(result -> {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
externalUrlCriteria.setPath(result.get("path"));
externalUrlCriteria.setHost(result.get("host"));
String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType);
}).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList());
results = new Results(multiResults, new HashMap<>(1, 1));
results = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType);
} else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) {
List<Map<String, Object>> tempRes = jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]");
List<Map<String, String>> finalRes = new ArrayList<>();
tempRes.forEach(map -> {
Map<String, String> resMap = new HashMap<>();
map.forEach((key, value) -> {
if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
resMap.put("tags", ((JSONArray) value).toJSONString());
} else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
resMap.put(key, ((JSONArray) value).toJSONString());
} else {
resMap.put(key, (String) value);
}
});
finalRes.add(resMap);
});
results = new Results(finalRes,
new HashMap<>(1, 1));
results = RemoteFetcherUtils.getFromJsonWithType(jsonContext, jsonDataPath);
} else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"),
new HashMap<>(1, 1));
results.getResults().stream().forEach(entry -> {
String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
entry.put("name", name);
entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
});
results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
} else {
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
new HashMap<>(1, 1));
}
results.results = results.results.stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
.collect(Collectors.toList());
results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
.collect(Collectors.toList()));
}
else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) {
Class<?> aClass = Class.forName(jsonDataPath.getParseClass());
@ -398,7 +324,8 @@ public class RemoteFetcher {
Object data = unmarshaller.unmarshal(stringReader);
Method reader = null;
if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) {
reader = new PropertyDescriptor(jsonDataPath.getParseField(), aClass).getReadMethod();
String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : "";
reader = aClass.getMethod(camelCaseGetter);
}
ObjectMapper objectMapper = new ObjectMapper();
List<Map<String, String>> values = new ArrayList<>();
@ -448,8 +375,6 @@ public class RemoteFetcher {
} catch (Exception exception) {
logger.error(exception.getMessage(), exception);
} //maybe print smth...
finally {
}
return null;
}
@ -459,7 +384,7 @@ public class RemoteFetcher {
try {
String filePath = Paths.get(path).toUri().toURL().toString();
ObjectMapper mapper = new ObjectMapper();
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, Object>>>(){});
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
return searchListMap(internalResults, query);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -494,48 +419,7 @@ public class RemoteFetcher {
}
static class Results {
List<Map<String, String>> results;
Map<String, Integer> pagination;
Results() {
this.results = new ArrayList<>();
this.pagination = new HashMap<>();
}
Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
this.results = results;
this.pagination = pagination;
}
List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
Map<String, Integer> getPagination() {
return pagination;
}
public void setPagination(Map<String, Integer> pagination) {
this.pagination = pagination;
}
}
private void mapToMap(String key, Map<String, String> source, Map<String, String> destination, boolean isTitle) {
if (source != null) {
String content = source.get("content");
if (isTitle) {
String classId = source.get("classid");
if (classId.equals("main title")) {
destination.put(key, content);
}
} else {
destination.put(key, content);
}
}
}
}

View File

@ -0,0 +1,128 @@
package eu.eudat.logic.proxy.fetching;
import com.jayway.jsonpath.DocumentContext;
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.fetching.entities.Results;
import net.minidev.json.JSONArray;
import java.util.*;
import java.util.stream.Collectors;
public class RemoteFetcherUtils {
public static Results getFromJsonWithSource(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
return new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
new HashMap<>(1, 1));
}
public static Results getFromJsonWithParsedData(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
new HashMap<>(1, 1));
List<Map<String, String>> fixedResults = results.getResults().stream().map(item -> {
for (int i = 0; i < 2; i++) {
String id;
if (i == 0) {
id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", "");
} else {
id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", "");
}
if (!(item.get(id) instanceof String)) {
Object obj = item.get(id);
if (obj instanceof JSONArray) {
JSONArray jarr = (JSONArray) obj;
if (jarr.get(0) instanceof String) {
item.put(id, jarr.get(0).toString());
} else {
for (int j = 0; j < jarr.size(); j++) {
mapToMap(id, (Map<String, String>)jarr.get(j), item, i == 1);
}
}
} else {
if (obj instanceof Map) {
mapToMap(id, (Map<String, String>) obj, item, i == 1);
} else if (obj != null){
item.put(id, obj.toString());
}
}
}
}
return item;
}).collect(Collectors.toList());
return new Results(fixedResults, new HashMap<>(1, 1));
}
public static Results getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcher remoteFetcher, String requestBody, String requestType) {
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getPath()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"),
new HashMap<>(1, 1));
List<Map<String, String>> multiResults = results.getResults().stream().map(result -> {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
externalUrlCriteria.setPath(result.get("path"));
externalUrlCriteria.setHost(result.get("host"));
String replacedPath = remoteFetcher.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
return remoteFetcher.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType);
}).filter(Objects::nonNull).map(results1 -> results1.getResults().get(0)).collect(Collectors.toList());
return new Results(multiResults, new HashMap<>(1, 1));
}
public static Results getFromJsonWithType(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
List<Map<String, Object>> tempRes = jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]");
List<Map<String, String>> finalRes = new ArrayList<>();
tempRes.forEach(map -> {
Map<String, String> resMap = new HashMap<>();
map.forEach((key, value) -> {
if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
resMap.put("tags", ((JSONArray) value).toJSONString());
} else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
resMap.put(key, ((JSONArray) value).toJSONString());
} else {
resMap.put(key, (String) value);
}
});
finalRes.add(resMap);
});
return new Results(finalRes,
new HashMap<>(1, 1));
}
public static Results getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"),
new HashMap<>(1, 1));
results.getResults().stream().forEach(entry -> {
String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
entry.put("name", name);
entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
});
return results;
}
private static void mapToMap(String key, Map<String, String> source, Map<String, String> destination, boolean isTitle) {
if (source != null) {
String content = source.get("content");
/*if (isTitle) {
String classId = source.get("classid");
if (classId.equals("main title")) {
destination.put(key, content);
}
} else {*/
destination.put(key, content);
// }
}
}
}

View File

@ -0,0 +1,37 @@
package eu.eudat.logic.proxy.fetching.entities;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class Results {
List<Map<String, String>> results;
Map<String, Integer> pagination;
public Results() {
this.results = new ArrayList<>();
this.pagination = new HashMap<>();
}
public Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
this.results = results;
this.pagination = pagination;
}
public List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
public Map<String, Integer> getPagination() {
return pagination;
}
public void setPagination(Map<String, Integer> pagination) {
this.pagination = pagination;
}
}

View File

@ -9,7 +9,7 @@ import java.util.Map;
* Created by ikalyvas on 3/5/2018.
*/
public interface VisibilityRuleService {
Boolean isElementVisible(String id);
boolean isElementVisible(String id);
void buildVisibilityContext(List<Rule> sources);

View File

@ -3,9 +3,7 @@ package eu.eudat.logic.services.forms;
import eu.eudat.models.data.user.components.commons.Rule;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
* Created by ikalyvas on 3/5/2018.
@ -16,13 +14,22 @@ public class VisibilityRuleServiceImpl implements VisibilityRuleService {
private VisibilityContext visibilityContext;
private Map<String, Object> properties;
public Boolean isElementVisible(String id) {
public boolean isElementVisible(String id) {
if (!this.elementVisibility.containsKey(id) || this.elementVisibility.get(id)) return true;
return false;
}
public void setProperties(Map<String, Object> properties) {
this.properties = properties;
this.properties.entrySet().stream()
.filter(stringObjectEntry -> stringObjectEntry.getValue() instanceof String && ((String) stringObjectEntry.getValue()).startsWith("[") && ((String) stringObjectEntry.getValue()).endsWith("]")).forEach(stringObjectEntry -> {
stringObjectEntry.setValue(parseArray((String) stringObjectEntry.getValue()));
});
}
private List<String> parseArray(String original) {
String parsed = original.replace("[", "").replace("\"", "").replace("]", "");
return Arrays.asList(parsed.split(","));
}
public void buildVisibilityContext(List<Rule> sources) {
@ -34,12 +41,19 @@ public class VisibilityRuleServiceImpl implements VisibilityRuleService {
private void evaluateVisibility(VisibilityRule rule) {
List<VisibilityRuleSource> sources = rule.getVisibilityRuleSources();
for(int i = 0; i < sources.size(); i++){
if (!properties.containsKey(sources.get(i).getVisibilityRuleSourceId()) || !properties.get(sources.get(i).getVisibilityRuleSourceId()).equals(sources.get(i).getVisibilityRuleSourceValue())) {
if (properties.containsKey(sources.get(i).getVisibilityRuleSourceId()) && (isContained(properties.get(sources.get(i).getVisibilityRuleSourceId()), sources.get(i).getVisibilityRuleSourceValue()) || properties.get(sources.get(i).getVisibilityRuleSourceId()).equals(sources.get(i).getVisibilityRuleSourceValue()))) {
this.elementVisibility.put(rule.getVisibilityRuleTargetId(), true);
}else{
this.elementVisibility.put(rule.getVisibilityRuleTargetId(), false);
return;
}else{
this.elementVisibility.put(rule.getVisibilityRuleTargetId(), true);
}
}
}
private Boolean isContained(Object values, String source) {
if (values instanceof List) {
return ((Collection<?>) values).contains(source);
}
return false;
}
}

View File

@ -100,7 +100,7 @@ public abstract class AbstractAuthenticationService implements AuthenticationSer
UserToken userToken = this.apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserTokenBuilder.class)
.issuedAt(new Date()).user(credential.getUserInfo())
.token(UUID.randomUUID()).expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(1)))
.token(UUID.randomUUID()).expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(10)))
.build();
userToken = apiContext.getOperationsContext().getDatabaseRepository().getUserTokenDao().createOrUpdate(userToken);
@ -200,7 +200,7 @@ public abstract class AbstractAuthenticationService implements AuthenticationSer
UserToken userToken = this.apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserTokenBuilder.class)
.token(UUID.randomUUID()).user(userInfo)
.expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(1))).issuedAt(new Date())
.expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(10))).issuedAt(new Date())
.build();
apiContext.getOperationsContext().getDatabaseRepository().getUserTokenDao().createOrUpdate(userToken);

View File

@ -61,7 +61,7 @@ public class ConfirmationEmailServiceImpl implements ConfirmationEmailService {
}
private String createContent(UUID confirmationToken, MailService mailService) {
String content = mailService.getMailTemplateContent("classpath:templates/email/emailConfirmation.html");
String content = mailService.getMailTemplateContent(this.environment.getProperty("email.confirmation"));
content = content.replace("{confirmationToken}", confirmationToken.toString());
content = content.replace("{expiration_time}", secondsToTime(Integer.parseInt(this.environment.getProperty("conf_email.expiration_time_seconds"))));
content = content.replace("{host}", this.environment.getProperty("dmp.domain"));
@ -85,7 +85,7 @@ public class ConfirmationEmailServiceImpl implements ConfirmationEmailService {
}
private String createMergeContent(UUID confirmationToken, MailService mailService, String userName) {
String content = mailService.getMailTemplateContent("classpath:templates/email/emailMergeConfirmation.html");
String content = mailService.getMailTemplateContent(this.environment.getProperty("email.merge"));
content = content.replace("{userName}", userName);
content = content.replace("{confirmationToken}", confirmationToken.toString());
content = content.replace("{expiration_time}", secondsToTime(Integer.parseInt(this.environment.getProperty("conf_email.expiration_time_seconds"))));

View File

@ -80,7 +80,7 @@ public class InvitationServiceImpl implements InvitationService {
return CompletableFuture.runAsync(() -> {
SimpleMail mail = new SimpleMail();
mail.setSubject(createSubject(dmp, mailService.getMailTemplateSubject()));
mail.setContent(createContent(invitation.getId(), dmp, recipient, mailService.getMailTemplateContent("classpath:templates/email/email.html"), role));
mail.setContent(createContent(invitation.getId(), dmp, recipient, mailService.getMailTemplateContent(this.environment.getProperty("email.invite")), role));
mail.setTo(invitation.getInvitationEmail());
try {
mailService.sendSimpleMail(mail);

View File

@ -86,6 +86,7 @@ public class MailServiceImpl implements MailService {
InputStream inputStream = resource.getInputStream();
StringWriter writer = new StringWriter();
IOUtils.copy(inputStream, writer, "UTF-8");
inputStream.close();
return writer.toString();
} catch (IOException e) {
logger.error(e.getMessage(), e);

View File

@ -84,6 +84,7 @@ public class ModelBuilder {
if (type.equals("checkBox")) return (FieldData<U>) new CheckBoxData().fromData(data);
if (type.equals("freetext")) return (FieldData<U>) new FreeTextData().fromData(data);
if (type.equals("textarea")) return (FieldData<U>) new TextAreaData().fromData(data);
if (type.equals("richTextarea")) return (FieldData<U>) new RichTextAreaData().fromData(data);
if (type.equals("datePicker")) return (FieldData<U>) new DatePickerData().fromData(data);
if (type.equals("externalDatasets")) return (FieldData<U>) new ExternalDatasetsData().fromData(data);
if (type.equals("dataRepositories")) return (FieldData<U>) new DataRepositoriesData().fromData(data);
@ -123,6 +124,7 @@ public class ModelBuilder {
if (type.equals("checkBox")) return (FieldData<U>) new CheckBoxData().fromData(data);
if (type.equals("freetext")) return (FieldData<U>) new FreeTextData().fromData(data);
if (type.equals("textarea")) return (FieldData<U>) new TextAreaData().fromData(data);
if (type.equals("richTextarea")) return (FieldData<U>) new RichTextAreaData().fromData(data);
if (type.equals("datePicker")) return (FieldData<U>) new DatePickerData().fromData(data);
if (type.equals("externalDatasets")) return (FieldData<U>) new ExternalDatasetsData().fromData(data);
if (type.equals("dataRepositories")) return (FieldData<U>) new DataRepositoriesData().fromData(data);

View File

@ -0,0 +1,45 @@
package eu.eudat.logic.utilities.documents.pdf;
import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
import org.apache.commons.io.IOUtils;
import org.springframework.core.env.Environment;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.web.client.RestTemplate;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.UUID;
public class PDFUtils {
public static File convertToPDF(FileEnvelope file, Environment environment) throws IOException {
LinkedMultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
String uuid = UUID.randomUUID().toString();
map.add("files", new FileSystemResource(file.getFile()));
map.add("filename", uuid + ".pdf");
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
headers.add("Content-disposition", "attachment; filename=" + uuid + ".pdf");
headers.add("Content-type", "application/pdf");
HttpEntity<LinkedMultiValueMap<String, Object>> requestEntity = new HttpEntity<LinkedMultiValueMap<String, Object>>(
map, headers);
byte[] queueResult = new RestTemplate().postForObject(environment.getProperty("pdf.converter.url") + "forms/libreoffice/convert"
, requestEntity, byte[].class);
File resultPdf = new File(environment.getProperty("temp.temp") + uuid + ".pdf");
FileOutputStream output = new FileOutputStream(resultPdf);
IOUtils.write(queueResult, output);
output.close();
Files.deleteIfExists(file.getFile().toPath());
return resultPdf;
}
}

View File

@ -4,7 +4,7 @@ package eu.eudat.logic.utilities.documents.types;
* Created by ikalyvas on 2/26/2018.
*/
public enum ParagraphStyle {
TEXT(0), HEADER1(1), HEADER2(2), HEADER3(3), HEADER4(4), TITLE(5), FOOTER(6), COMMENT(7), HEADER5(8), HEADER6(9);
TEXT(0), HEADER1(1), HEADER2(2), HEADER3(3), HEADER4(4), TITLE(5), FOOTER(6), COMMENT(7), HEADER5(8), HEADER6(9), HTML(10);
private Integer value;
@ -38,6 +38,8 @@ public enum ParagraphStyle {
return HEADER5;
case 9:
return HEADER6;
case 10:
return HTML;
default:
throw new RuntimeException("Unsupported ParagraphStyle Code");
}

View File

@ -0,0 +1,249 @@
package eu.eudat.logic.utilities.documents.word;
import org.apache.poi.xwpf.usermodel.*;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.*;
import java.math.BigInteger;
import java.util.*;
public class HtmlToWorldBuilder implements NodeVisitor {
private final Map<String, Boolean> properties = new LinkedHashMap<>();
private XWPFParagraph paragraph;
private XWPFRun run;
private Boolean dumpRun;
private final float indentation;
private Boolean isIdentationUsed;
private XWPFNumbering numbering;
private Queue<BigInteger> abstractNumId;
private BigInteger numberingLevel;
public static HtmlToWorldBuilder convert(XWPFDocument document, Document htmlDocument, float indentation) {
XWPFParagraph paragraph = document.createParagraph();
HtmlToWorldBuilder htmlToWorldBuilder = new HtmlToWorldBuilder(paragraph, indentation);
NodeTraversor.traverse(htmlToWorldBuilder, htmlDocument);
return htmlToWorldBuilder;
}
public HtmlToWorldBuilder(XWPFParagraph paragraph, float indentation) {
this.paragraph = paragraph;
this.run = this.paragraph.createRun();
this.dumpRun = false;
this.indentation = indentation;
this.isIdentationUsed = false;
this.run.setFontSize(11);
this.abstractNumId = new ArrayDeque<>();
this.numberingLevel = BigInteger.valueOf(-1);
this.setDefaultIndentation();
}
@Override
public void head(Node node, int i) {
String name = node.nodeName();
if (name.equals("#text")) {
String text = ((TextNode)node).text();
this.run.setText(text);
this.dumpRun = true;
} else {
properties.put(name, true);
}
if (dumpRun) {
this.run = this.paragraph.createRun();
this.run.setFontSize(11);
this.dumpRun = false;
}
parseProperties(node);
properties.clear();
}
private void parseProperties(Node node) {
properties.entrySet().forEach(stringBooleanEntry -> {
switch (stringBooleanEntry.getKey()) {
case "i" :
case "em":
this.run.setItalic(stringBooleanEntry.getValue());
break;
case "b":
case "strong":
this.run.setBold(stringBooleanEntry.getValue());
break;
case "u":
case "ins":
this.run.setUnderline(stringBooleanEntry.getValue() ? UnderlinePatterns.SINGLE : UnderlinePatterns.NONE);
break;
case "small":
this.run.setFontSize(stringBooleanEntry.getValue() ? 8 : 11);
break;
case "del":
case "strike":
case "strikethrough":
case "s":
this.run.setStrikeThrough(stringBooleanEntry.getValue());
break;
case "mark":
this.run.setTextHighlightColor(stringBooleanEntry.getValue() ? STHighlightColor.YELLOW.toString() : STHighlightColor.NONE.toString());
break;
case "sub":
this.run.setSubscript(stringBooleanEntry.getValue() ? VerticalAlign.SUBSCRIPT : VerticalAlign.BASELINE);
break;
case "sup":
this.run.setSubscript(stringBooleanEntry.getValue() ? VerticalAlign.SUPERSCRIPT : VerticalAlign.BASELINE);
break;
case "div":
case "p":
this.paragraph = this.paragraph.getDocument().createParagraph();
this.run = this.paragraph.createRun();
this.isIdentationUsed = false;
this.setDefaultIndentation();
if (stringBooleanEntry.getValue()) {
if (node.hasAttr("align")) {
String alignment = node.attr("align");
this.paragraph.setAlignment(ParagraphAlignment.valueOf(alignment.toUpperCase(Locale.ROOT)));
}
}
break;
case "blockquote":
this.paragraph = this.paragraph.getDocument().createParagraph();
this.run = this.paragraph.createRun();
if (stringBooleanEntry.getValue()) {
this.paragraph.setIndentationLeft(720);
} else {
this.isIdentationUsed = false;
this.setDefaultIndentation();
}
break;
case "ul":
if (stringBooleanEntry.getValue()) {
createNumbering(STNumberFormat.BULLET);
} else {
this.paragraph = this.paragraph.getDocument().createParagraph();
this.run = this.paragraph.createRun();
this.isIdentationUsed = false;
this.setDefaultIndentation();
this.numberingLevel = this.numberingLevel.subtract(BigInteger.ONE);
((ArrayDeque)this.abstractNumId).removeLast();
}
break;
case "ol":
if (stringBooleanEntry.getValue()) {
createNumbering(STNumberFormat.DECIMAL);
} else {
this.paragraph = this.paragraph.getDocument().createParagraph();
this.run = this.paragraph.createRun();
this.isIdentationUsed = false;
this.setDefaultIndentation();
this.numberingLevel = this.numberingLevel.subtract(BigInteger.ONE);
((ArrayDeque)this.abstractNumId).removeLast();
}
break;
case "li":
if (stringBooleanEntry.getValue()) {
this.paragraph = this.paragraph.getDocument().createParagraph();
this.paragraph.setIndentationLeft(Math.round(indentation * 720) * (numberingLevel.intValue() + 1));
this.run = this.paragraph.createRun();
this.paragraph.setNumID(((ArrayDeque<BigInteger>)abstractNumId).getLast());
}
break;
case "font":
if (stringBooleanEntry.getValue()) {
if (node.hasAttr("color")) {
this.run.setColor(node.attr("color").substring(1));
}
} else {
this.run.setColor("000000");
}
break;
case "a":
if (stringBooleanEntry.getValue()) {
if (node.hasAttr("href")) {
this.run = createHyperLinkRun(node.attr("href"));
this.run.setColor("0000FF");
this.run.setUnderline(UnderlinePatterns.SINGLE);
}
} else {
this.run = paragraph.createRun();
}
break;
case "br":
if (stringBooleanEntry.getValue()) {
this.run.addBreak();
}
break;
}
});
}
@Override
public void tail(Node node, int i) {
String name = node.nodeName();
properties.put(name, false);
parseProperties(node);
properties.clear();
}
//GK: This function creates one numbering.xml for the word document and adds a specific format.
//It imitates the numbering.xml that is usually generated by word editors like LibreOffice
private void createNumbering(STNumberFormat.Enum format) {
CTAbstractNum ctAbstractNum = CTAbstractNum.Factory.newInstance();
if (this.numbering == null) this.numbering = this.paragraph.getDocument().createNumbering();
BigInteger tempNumId = BigInteger.ONE;
boolean found = false;
while (!found) {
Object o = numbering.getAbstractNum(tempNumId);
found = (o == null);
if (!found) tempNumId = tempNumId.add(BigInteger.ONE);
}
ctAbstractNum.setAbstractNumId(tempNumId);
CTLvl ctLvl = ctAbstractNum.addNewLvl();
this.numberingLevel = numberingLevel.add(BigInteger.ONE);
ctLvl.setIlvl(numberingLevel);
ctLvl.addNewNumFmt().setVal(format);
ctLvl.addNewStart().setVal(BigInteger.ONE);
if (format == STNumberFormat.BULLET) {
ctLvl.addNewLvlJc().setVal(STJc.LEFT);
ctLvl.addNewLvlText().setVal("\u2022");
ctLvl.addNewRPr(); //Set the Symbol font
CTFonts f = ctLvl.getRPr().addNewRFonts();
f.setAscii("Symbol");
f.setHAnsi("Symbol");
f.setCs("Symbol");
f.setHint(STHint.DEFAULT);
} else {
ctLvl.addNewLvlText().setVal("%1.");
}
XWPFAbstractNum xwpfAbstractNum = new XWPFAbstractNum(ctAbstractNum);
this.abstractNumId.add(this.numbering.addAbstractNum(xwpfAbstractNum));
this.numbering.addNum(((ArrayDeque<BigInteger>)abstractNumId).getLast());
}
private XWPFHyperlinkRun createHyperLinkRun(String uri) {
String rId = this.paragraph.getDocument().getPackagePart().addExternalRelationship(uri, XWPFRelation.HYPERLINK.getRelation()).getId();
CTHyperlink cthyperLink=paragraph.getCTP().addNewHyperlink();
cthyperLink.setId(rId);
cthyperLink.addNewR();
return new XWPFHyperlinkRun(
cthyperLink,
cthyperLink.getRArray(0),
paragraph
);
}
private void setDefaultIndentation() {
if (!isIdentationUsed) {
this.paragraph.setIndentationLeft(Math.round(indentation * 720.0F));
this.isIdentationUsed = true;
}
}
public XWPFParagraph getParagraph() {
return paragraph;
}
}

View File

@ -1,6 +1,5 @@
package eu.eudat.logic.utilities.documents.word;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.logic.services.forms.VisibilityRuleService;
@ -15,6 +14,8 @@ import eu.eudat.models.data.user.components.datasetprofile.Section;
import eu.eudat.models.data.user.composite.DatasetProfilePage;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import org.apache.poi.xwpf.usermodel.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTAbstractNum;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDecimalNumber;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLvl;
@ -24,7 +25,14 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.math.BigInteger;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.*;
import java.util.stream.Collectors;
public class WordBuilder {
private static final Logger logger = LoggerFactory.getLogger(WordBuilder.class);
@ -32,10 +40,12 @@ public class WordBuilder {
private Map<ParagraphStyle, ApplierWithValue<XWPFDocument, String, XWPFParagraph>> options = new HashMap<>();
private CTAbstractNum cTAbstractNum;
private BigInteger numId;
private Integer indent;
public WordBuilder() {
this.cTAbstractNum = CTAbstractNum.Factory.newInstance();
this.cTAbstractNum.setAbstractNumId(BigInteger.valueOf(1));
this.indent = 0;
this.buildOptions();
}
@ -48,6 +58,11 @@ public class WordBuilder {
run.setFontSize(11);
return paragraph;
});
this.options.put(ParagraphStyle.HTML, (mainDocumentPart, item) -> {
Document htmlDoc = Jsoup.parse(item.replaceAll("\n", "<br>"));
HtmlToWorldBuilder htmlToWorldBuilder = HtmlToWorldBuilder.convert(mainDocumentPart, htmlDoc, indent > 0 ? (indent/2.0F) * 0.8F : 0.8F);
return htmlToWorldBuilder.getParagraph();
});
this.options.put(ParagraphStyle.TITLE, (mainDocumentPart, item) -> {
XWPFParagraph paragraph = mainDocumentPart.createParagraph();
paragraph.setStyle("Title");
@ -124,8 +139,16 @@ public class WordBuilder {
public XWPFDocument build(XWPFDocument document, PagedDatasetProfile pagedDatasetProfile, VisibilityRuleService visibilityRuleService) throws IOException {
createPages(pagedDatasetProfile.getPages(), document, true, visibilityRuleService);
XWPFAbstractNum abstractNum = new XWPFAbstractNum(cTAbstractNum);
XWPFNumbering numbering = document.createNumbering();
BigInteger tempNumId = BigInteger.ONE;
boolean found = false;
while (!found) {
Object o = numbering.getAbstractNum(tempNumId);
found = (o == null);
if (!found) tempNumId = tempNumId.add(BigInteger.ONE);
}
cTAbstractNum.setAbstractNumId(tempNumId);
XWPFAbstractNum abstractNum = new XWPFAbstractNum(cTAbstractNum);
BigInteger abstractNumID = numbering.addAbstractNum(abstractNum);
this.numId = numbering.addNum(abstractNumID);
createPages(pagedDatasetProfile.getPages(), document, false, visibilityRuleService);
@ -179,7 +202,8 @@ public class WordBuilder {
}
hasValue = createFields(compositeField.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService);
if (compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()) {
for (FieldSet multiplicityFieldset : compositeField.getMultiplicityItems()) {
List<FieldSet> list = compositeField.getMultiplicityItems().stream().sorted(Comparator.comparingInt(FieldSet::getOrdinal)).collect(Collectors.toList());
for (FieldSet multiplicityFieldset : list) {
hasValue = createFields(multiplicityFieldset.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService);
}
}
@ -199,15 +223,19 @@ public class WordBuilder {
private Boolean createFields(List<Field> fields, XWPFDocument mainDocumentPart, Integer indent, Boolean createListing, VisibilityRuleService visibilityRuleService) {
if (createListing) this.addListing(mainDocumentPart, indent, false, false);
boolean hasValue = false;
for (Field field: fields) {
List<Field> tempFields = fields.stream().sorted(Comparator.comparingInt(Field::getOrdinal)).collect(Collectors.toList());
for (Field field: tempFields) {
if (visibilityRuleService.isElementVisible(field.getId())) {
if (!createListing) {
try {
XWPFParagraph paragraph = addParagraphContent(this.formatter(field), mainDocumentPart, ParagraphStyle.TEXT, numId);
if (paragraph != null) {
CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl();
number.setVal(BigInteger.valueOf(indent));
hasValue = true;
if (field.getValue() != null && !field.getValue().toString().isEmpty()) {
this.indent = indent;
XWPFParagraph paragraph = addParagraphContent(this.formatter(field), mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId);
if (paragraph != null) {
CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl();
number.setVal(BigInteger.valueOf(indent));
hasValue = true;
}
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
@ -221,10 +249,12 @@ public class WordBuilder {
public XWPFParagraph addParagraphContent(String text, XWPFDocument mainDocumentPart, ParagraphStyle style, BigInteger numId) {
if (text != null && !text.isEmpty()) {
XWPFParagraph paragraph = this.options.get(style).apply(mainDocumentPart, text);
if (numId != null) {
paragraph.setNumID(numId);
if (paragraph != null) {
if (numId != null) {
paragraph.setNumID(numId);
}
return paragraph;
}
return paragraph;
}
return null;
}
@ -238,13 +268,16 @@ public class WordBuilder {
}
if (question) {
cTLvl.addNewNumFmt().setVal(STNumberFormat.DECIMAL);
cTLvl.addNewLvlText().setVal("");
cTLvl.setIlvl(BigInteger.valueOf(indent));
} else if (!question && hasIndication) {
cTLvl.addNewNumFmt().setVal(STNumberFormat.DECIMAL);
cTLvl.addNewLvlText().setVal("");
cTLvl.setIlvl(BigInteger.valueOf(indent));
}
if (!question && !hasIndication) {
cTLvl.addNewNumFmt().setVal(STNumberFormat.NONE);
cTLvl.addNewLvlText().setVal("");
cTLvl.setIlvl(BigInteger.valueOf(indent));
}
}
@ -275,44 +308,16 @@ public class WordBuilder {
try {
mapList = Arrays.asList(mapper.readValue(field.getValue().toString(), HashMap[].class));
}catch (Exception e) {
logger.warn(e.getMessage(), e);
logger.info("Moving to fallback parsing");
// logger.warn(e.getMessage(), e);
// logger.info("Moving to fallback parsing");
Map <String, Object> map = new HashMap<>();
map.put("label", field.getValue().toString());
mapList.add(map);
}
/*try {
if (field.getValue().toString().startsWith("[")) {
JSONArray jsonarray = new JSONArray(field.getValue().toString());
for (int i = 0; i < jsonarray.length(); i++) {
JSONObject jsonObject = jsonarray.getJSONObject(i);
String id = jsonObject.get("id").toString();
String label = jsonObject.getString("label");
if (id != null && label != null) {
map.put(id, label);
}
}
} else if (field.getValue().toString().startsWith("{")) {
JSONObject jsonObject = new JSONObject(field.getValue().toString());
String id = jsonObject.get("id").toString();
String label = jsonObject.getString("label");
if (id != null && label != null) {
map.put(id, label);
}
}
} catch (Exception e) {
Map<String, String> exMap = mapper.readValue(field.getValue().toString(), new TypeReference<Map<String, String>>() {
});
return exMap.get("label");
}*/
}
StringBuilder sb = new StringBuilder();
int index = 0;
for (Map<String, Object> map: mapList) {
/*if (!map.containsKey("label") && !map.containsKey("description")) {
logger.error("Value is missing the \"label\" and the \"description\" attributes");
map.put("label", "unknown Name");
}*/
for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() != null && (entry.getKey().equals("label") || entry.getKey().equals("description") || entry.getKey().equals("name"))) {
sb.append(entry.getValue().toString());
@ -325,15 +330,13 @@ public class WordBuilder {
return sb.toString();
} else if (comboboxType.equals("wordlist")) {
WordListData wordListData = (WordListData) field.getData();
if (wordListData.getOptions().isEmpty() && field.getValue() != null) {
logger.warn("World List has no values but the field has");
logger.info("Return value as is");
return field.getValue().toString();
} else if (field.getValue() != null){
if (field.getValue() != null){
ComboBoxData<WordListData>.Option selectedOption = null;
for (ComboBoxData<WordListData>.Option option: wordListData.getOptions()) {
if (option.getValue().equals(field.getValue())) {
selectedOption = option;
if (!wordListData.getOptions().isEmpty()) {
for (ComboBoxData<WordListData>.Option option : wordListData.getOptions()) {
if (option.getValue().equals(field.getValue())) {
selectedOption = option;
}
}
}
return selectedOption != null ? selectedOption.getLabel() : field.getValue().toString();
@ -343,33 +346,39 @@ public class WordBuilder {
}
case "booleanDecision":
if (field.getValue() != null && field.getValue().equals("true")) return "Yes";
else return "No";
if (field.getValue() != null && field.getValue().equals("false")) return "No";
return null;
case "radiobox":
return field.getValue() != null ? field.getValue().toString() : null;
case "checkBox":
CheckBoxData data = (CheckBoxData) field.getData();
if (field.getValue() == null || field.getValue().equals("false")) return null;
return data.getLabel();
case "freetext":
case "datepicker":
case "datePicker":{
Instant instant;
try {
instant = Instant.parse((String) field.getValue());
} catch (DateTimeParseException ex) {
instant = (Instant) DateTimeFormatter.ofPattern("yyyy-MM-dd").parse((String)field.getValue());
}
return field.getValue() != null ? DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()).format(instant) : "";
}
case "freetext":
case "textarea":
case "richTextarea":
return field.getValue() != null ? field.getValue().toString(): "";
case "datasetIdentifier":
case "validation":
if (field.getValue() != null) {
if (field.getValue() != null && !field.getValue().toString().isEmpty()) {
Map<String, String> identifierData;
ObjectMapper mapper = new ObjectMapper();
try {
ObjectMapper mapper = new ObjectMapper();
identifierData = mapper.readValue(field.getValue().toString(), HashMap.class);
} catch (JsonParseException ex) {
identifierData = new HashMap<>();
String parsedData = field.getValue().toString().substring(1, field.getValue().toString().length() - 1);
StringTokenizer commaTokens = new StringTokenizer(parsedData, ", ");
while (commaTokens.hasMoreTokens()) {
String token = commaTokens.nextToken();
StringTokenizer equalTokens = new StringTokenizer(token, "=");
identifierData.put(equalTokens.nextToken(), equalTokens.nextToken());
}
} catch (Exception ex) {
// logger.warn(ex.getLocalizedMessage(), ex);
// logger.info("Reverting to custom parsing");
identifierData = customParse(field.getValue().toString());
}
return "id: " + identifierData.get("identifier") + ", Validation Type: " + identifierData.get("type");
}
@ -381,4 +390,17 @@ public class WordBuilder {
private boolean hasVisibleFields(FieldSet compositeFields, VisibilityRuleService visibilityRuleService) {
return compositeFields.getFields().stream().anyMatch(field -> visibilityRuleService.isElementVisible(field.getId()));
}
private Map<String, String> customParse(String value) {
Map<String, String> result = new LinkedHashMap<>();
String parsedValue = value.replaceAll("[^a-zA-Z0-9\\s:=,]", "");
StringTokenizer commaTokens = new StringTokenizer(parsedValue, ", ");
String delimeter = parsedValue.contains("=") ? "=" : ":";
while (commaTokens.hasMoreTokens()) {
String token = commaTokens.nextToken();
StringTokenizer delimiterTokens = new StringTokenizer(token, delimeter);
result.put(delimiterTokens.nextToken(), delimiterTokens.nextToken());
}
return result;
}
}

View File

@ -0,0 +1,63 @@
package eu.eudat.logic.utilities.documents.word;
import org.apache.poi.ooxml.POIXMLDocumentPart;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.opc.PackagePart;
import org.apache.poi.openxml4j.opc.PackagePartName;
import org.apache.poi.openxml4j.opc.PackagingURIHelper;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.UUID;
public class XWPFHtmlDocument extends POIXMLDocumentPart {
private String html;
private String id;
public XWPFHtmlDocument(PackagePart pkg, String id) {
super(pkg);
this.html = "<!DOCTYPE html><html><head><meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\"><style></style><title>HTML import</title></head><body><p></p></body>";
this.id = id;
}
public String getHtml() {
return html;
}
public void setHtml(String html) {
this.html = this.html.replace("<p></p>", html);
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
@Override
protected void commit() throws IOException {
PackagePart packagePart = getPackagePart();
OutputStream outputStream = packagePart.getOutputStream();
Writer writer = new OutputStreamWriter(outputStream, "UTF-8");
writer.write(html);
writer.close();
outputStream.close();
}
public static XWPFHtmlDocument addHtmlDocument(XWPFDocument document) throws InvalidFormatException {
OPCPackage oPCPackage = document.getPackage();
String id = UUID.randomUUID().toString();
PackagePartName partName = PackagingURIHelper.createPartName("/word/" + id + ".html");
PackagePart part = oPCPackage.createPart(partName, "text/html");
XWPFHtmlDocument xWPFHtmlDocument = new XWPFHtmlDocument(part, id);
document.addRelation(xWPFHtmlDocument.getId(), new XWPFHtmlRelation(), xWPFHtmlDocument);
return xWPFHtmlDocument;
}
}

View File

@ -0,0 +1,11 @@
package eu.eudat.logic.utilities.documents.word;
import org.apache.poi.ooxml.POIXMLRelation;
public class XWPFHtmlRelation extends POIXMLRelation {
public XWPFHtmlRelation() {
super("text/html",
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/aFChunk",
"/word/htmlDoc#.html");
}
}

View File

@ -1,7 +1,10 @@
package eu.eudat.logic.utilities.documents.xml;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.MismatchedInputException;
import eu.eudat.logic.services.forms.VisibilityRuleService;
import eu.eudat.logic.utilities.builders.XmlBuilder;
import eu.eudat.models.data.components.commons.datafield.ExternalDatasetsData;
import eu.eudat.models.data.user.components.datasetprofile.Field;
import eu.eudat.models.data.user.components.datasetprofile.FieldSet;
import eu.eudat.models.data.user.components.datasetprofile.Section;
@ -17,7 +20,11 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public class ExportXmlBuilder {
@ -92,18 +99,42 @@ public class ExportXmlBuilder {
if (visibilityRuleService.isElementVisible(field.getId())) {
Element elementField = element.createElement("field");
elementField.setAttribute("id", field.getId());
if (field.getViewStyle().getRenderStyle().equals("externalDatasets")) {
elementField.setAttribute("type", ((ExternalDatasetsData)field.getData()).getType());
}
if (field.getValue() != null) {
Element valueField = element.createElement("value");
ObjectMapper mapper = new ObjectMapper();
try {
JSONArray jsonArray = new JSONArray(field.getValue().toString());
List<Map<String, Object>> jsonArray = mapper.readValue(field.getValue().toString(), List.class);
// JSONArray jsonArray = new JSONArray(field.getValue().toString());
StringBuilder sb = new StringBuilder();
for (int i = 0; i < jsonArray.length(); i++) {
boolean firstTime = true;
for (Map<String, Object> jsonElement: jsonArray) {
if (!firstTime) {
sb.append(", ");
}
sb.append(jsonElement.get("label") != null ? jsonElement.get("label") : jsonElement.get("name"));
firstTime = false;
}
/*for (int i = 0; i < jsonArray.length(); i++) {
sb.append(jsonArray.getJSONObject(i).get("label").toString());
if (i != jsonArray.length() - 1) sb.append(", ");
}
}*/
valueField.setTextContent(sb.toString());
} catch (JSONException ex) {
valueField.setTextContent(field.getValue().toString());
} catch (IOException ex) {
try {
Map<String, Object> jsonElement = mapper.readValue(field.getValue().toString(), Map.class);
valueField.setTextContent((jsonElement.get("label") != null ? jsonElement.get("label").toString() : jsonElement.get("name") != null ? jsonElement.get("name").toString() : ""));
} catch (IOException e) {
try {
valueField.setTextContent(DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()).format(Instant.parse(field.getValue().toString())));
} catch (Exception exc) {
valueField.setTextContent(field.getValue().toString());
}
}
}
elementField.appendChild(valueField);
}

View File

@ -260,6 +260,7 @@ public class ExportXmlBuilderDatasetProfile {
case CHECK_BOX:
case FREE_TEXT:
case TEXT_AREA:
case RICH_TEXT_AREA:
case DATE_PICKER:
case DATASET_IDENTIFIER:
case CURRENCY:
@ -290,6 +291,7 @@ public class ExportXmlBuilderDatasetProfile {
ExternalDatasetsData externalDatasetsData = (ExternalDatasetsData) field.getData();
dataOut.setAttribute("label", externalDatasetsData.getLabel());
dataOut.setAttribute("multiAutocomplete", externalDatasetsData.getMultiAutoComplete().toString());
dataOut.setAttribute("type", externalDatasetsData.getType());
break;
case DATA_REPOSITORIES:
DataRepositoriesData dataRepositoriesData = (DataRepositoriesData) field.getData();

View File

@ -52,7 +52,9 @@ public class DatasetProfile {
List<eu.eudat.models.data.admin.components.datasetprofile.Section> sectionDatasetEntity = new LinkedList<>();
for (Page xmlPage: page) {
pagesDatasetEntity.add(xmlPage.toAdminCompositeModelPage());
sectionDatasetEntity.add(xmlPage.toAdminCompositeModelSection());
for (int i = 0; i < xmlPage.getSections().size(); i++) {
sectionDatasetEntity.add(xmlPage.toAdminCompositeModelSection(i));
}
}
newDatasetEntityProfile.setPages(pagesDatasetEntity);
newDatasetEntityProfile.setSections(sectionDatasetEntity);

View File

@ -3,13 +3,14 @@ package eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileM
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
@XmlRootElement(name = "page")
public class Page {
private String id;
private int ordinal;
private String title;
private Sections sections;
private List<Sections> sections;
@XmlAttribute(name = "id")
public String getId() {
@ -39,11 +40,11 @@ public class Page {
}
@XmlElement(name = "sections")
public Sections getSections() {
public List<Sections> getSections() {
return sections;
}
public void setSections(Sections sections) {
public void setSections(List<Sections> sections) {
this.sections = sections;
}
@ -55,7 +56,7 @@ public class Page {
return pageEntity;
}
public eu.eudat.models.data.admin.components.datasetprofile.Section toAdminCompositeModelSection(){
public eu.eudat.models.data.admin.components.datasetprofile.Section toAdminCompositeModelSection(int i){
/* eu.eudat.models.data.admin.components.datasetprofile.Section sectionEntity =new eu.eudat.models.data.admin.components.datasetprofile.Section();
// List<eu.eudat.models.data.admin.components.datasetprofile.Section> sectionsListEntity = new LinkedList<>();
// for (Section xmlsection:this.sections.section) {
@ -68,6 +69,6 @@ public class Page {
sectionEntity.setId(this.id);
sectionEntity.setOrdinal(this.ordinal);
sectionEntity.setTitle(this.title);*/
return sections.toAdminCompositeModelSection();
return sections.get(i).toAdminCompositeModelSection();
}
}

View File

@ -0,0 +1,41 @@
package eu.eudat.logic.utilities.json;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
public class MultiDateDeserializer extends StdDeserializer<Date> {
private static final List<String> DATE_FORMATS = Arrays.asList("yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss.S");
public MultiDateDeserializer() {
super(Date.class);
}
protected MultiDateDeserializer(Class<?> vc) {
super(vc);
}
@Override
public Date deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
String text = p.getText();
for (String dateFormat: DATE_FORMATS) {
try {
return new SimpleDateFormat(dateFormat).parse(text);
} catch (ParseException ignored) {
}
}
throw new JsonParseException(p, "No supported Date format");
}
}

View File

@ -28,6 +28,7 @@ public class ViewStyle {
CHECK_BOX("checkBox"),
FREE_TEXT("freetext"),
TEXT_AREA("textarea"),
RICH_TEXT_AREA("richTextarea"),
DATE_PICKER("datePicker"),
EXTERNAL_DATASETS("externalDatasets"),
DATA_REPOSITORIES("dataRepositories"),

View File

@ -95,14 +95,14 @@ public class AutoCompleteData extends ComboBoxData<AutoCompleteData> {
this.autoCompleteSingleDataList.add(new AutoCompleteSingleData());
this.mapFromXml(item, this.autoCompleteSingleDataList.get(0));
}
this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutocomplete"));
this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutoComplete"));
return this;
}
private void mapFromXml(Element item, AutoCompleteSingleData singleData) {
singleData.url = item.getAttribute("url");
singleData.optionsRoot = item.getAttribute("optionsRoot");
this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutocomplete"));
this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutoComplete"));
if (item.getAttribute("autoCompleteType") == null || item.getAttribute("autoCompleteType").equals("") ) {
singleData.autocompleteType = AutocompleteType.UNCACHED.getValue();
} else {
@ -173,7 +173,7 @@ public class AutoCompleteData extends ComboBoxData<AutoCompleteData> {
dataMap.put("label", item != null ? item.getAttribute("label") : "");
//dataMap.put("url", item != null ? item.getAttribute("url") : "");
dataMap.put("type", item != null ? item.getAttribute("type") : "autocomplete");
dataMap.put("multiAutoComplete", item != null ? Boolean.valueOf(item.getAttribute("multiAutocomplete")) : false);
dataMap.put("multiAutoComplete", item != null ? Boolean.valueOf(item.getAttribute("multiAutoComplete")) : false);
List<Map<String, Object>> autoCompletes = new ArrayList<>();
NodeList autoCompleteSingles = item.getChildNodes();
for (int i = 0; i < autoCompleteSingles.getLength(); i++) {

View File

@ -8,6 +8,7 @@ import java.util.Map;
public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
private Boolean multiAutoComplete;
private String type;
public Boolean getMultiAutoComplete() {
return multiAutoComplete;
@ -17,11 +18,20 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
this.multiAutoComplete = multiAutoComplete;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
@Override
public ExternalDatasetsData fromData(Object data) {
if (data != null) {
this.setLabel((String) ((Map<String, Object>) data).get("label"));
this.setMultiAutoComplete(((Map<String, Object>) data).get("multiAutoComplete") != null && !((Map<String, Object>) data).get("multiAutoComplete").toString().isEmpty()? Boolean.parseBoolean( ((Map<String, Object>) data).get("multiAutoComplete").toString()) : false);
this.setType(((Map<String, Object>) data).get("type") != null && !((Map<String, Object>) data).get("type").toString().isEmpty()? ((Map<String, Object>) data).get("type").toString() : "other");
}
return this;
}
@ -38,6 +48,9 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
if (this.getMultiAutoComplete() != null) {
root.setAttribute("multiAutoComplete", this.getMultiAutoComplete().toString());
}
if (this.getType() != null) {
root.setAttribute("type", this.getType());
}
return root;
}
@ -45,6 +58,7 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
public ExternalDatasetsData fromXml(Element item) {
this.setLabel(item != null ? item.getAttribute("label") : "");
this.setMultiAutoComplete(Boolean.parseBoolean(item.getAttribute("multiAutoComplete")));
this.setType(item.getAttribute("type") != null ? item.getAttribute("type"): "other");
return this;
}
@ -53,6 +67,7 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
HashMap dataMap = new HashMap();
dataMap.put("label", item != null && item.getAttributes().getLength() > 0? item.getAttribute("label") : "");
dataMap.put("multiAutoComplete", item != null && item.getAttributes().getLength() > 0? Boolean.parseBoolean(item.getAttribute("multiAutocomplete")) : false);
dataMap.put("type", item != null && item.getAttributes().getLength() > 0? item.getAttribute("type") : "other");
return dataMap;
}
}

View File

@ -0,0 +1,43 @@
package eu.eudat.models.data.components.commons.datafield;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import java.util.HashMap;
import java.util.Map;
public class RichTextAreaData extends FieldData<RichTextAreaData> {
@Override
public RichTextAreaData fromData(Object data) {
if (data != null) {
this.setLabel(((Map<String, String>) data).get("label"));
}
return this;
}
@Override
public Object toData() {
return null;
}
@Override
public Element toXml(Document doc) {
Element root = doc.createElement("data");
root.setAttribute("label", this.getLabel());
return root;
}
@Override
public RichTextAreaData fromXml(Element item) {
this.setLabel(item.getAttribute("label"));
return this;
}
@Override
public Map<String, Object> toMap(Element item) {
HashMap dataMap = new HashMap();
dataMap.put("label", item != null ? item.getAttribute("label") : "");
return dataMap;
}
}

View File

@ -7,7 +7,10 @@ import eu.eudat.models.data.funder.FunderDMPEditorModel;
import eu.eudat.models.data.grant.GrantDMPEditorModel;
import eu.eudat.models.data.project.ProjectDMPEditorModel;
import eu.eudat.models.data.userinfo.UserInfo;
import eu.eudat.models.data.userinfo.UserListingModel;
import net.minidev.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.stream.Collectors;
@ -16,6 +19,8 @@ import java.util.stream.Collectors;
* Created by ikalyvas on 2/5/2018.
*/
public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataManagementPlanNewVersionModel> {
private static final Logger logger = LoggerFactory.getLogger(DataManagementPlanNewVersionModel.class);
private UUID id;
private String label;
private UUID groupId;
@ -26,7 +31,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
private eu.eudat.models.data.grant.GrantDMPEditorModel grant;
private List<Organisation> organisations;
private List<Researcher> researchers;
private List<UserDMP> associatedUsers;
private List<UserListingModel> associatedUsers;
private eu.eudat.models.data.userinfo.UserInfo creator;
private Date created;
private List<Dataset> datasets;
@ -104,10 +109,10 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
this.researchers = researchers;
}
public List<UserDMP> getAssociatedUsers() {
public List<UserListingModel> getAssociatedUsers() {
return associatedUsers;
}
public void setAssociatedUsers(List<UserDMP> associatedUsers) {
public void setAssociatedUsers(List<UserListingModel> associatedUsers) {
this.associatedUsers = associatedUsers;
}
@ -163,7 +168,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
public DMP toDataModel() throws Exception {
DMP entity = new DMP();
entity.setId(this.id);
entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers)));
// entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers)));
entity.setDescription(this.description);
entity.setStatus((short) this.status);
entity.setGroupId(this.groupId == null ? UUID.randomUUID() : this.groupId);
@ -227,7 +232,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
funder.setModified(new Date());
entity.getGrant().setFunder(funder);
} else if (this.funder.getExistFunder() != null && this.funder.getLabel() == null){
} else if (this.funder.getExistFunder() != null && this.funder.getLabel() == null) {
entity.getGrant().setFunder(this.funder.getExistFunder().toDataModel());
entity.getGrant().getFunder().setType(Funder.FunderType.EXTERNAL.getValue());
}

View File

@ -19,6 +19,10 @@ public class DmpImportModel {
private List<UserInfoImportModels> associatedUsersImportModels;
private List<DynamicFieldWithValueImportModels> dynamicFieldsImportModels;
private List<DatasetImportModels> datasetImportModels;
private String language;
private Boolean visibility;
private String publicDate;
private String costs;
@XmlElement(name = "description")
public String getDescriptionImport() {
@ -113,4 +117,37 @@ public class DmpImportModel {
@XmlElement(name = "dataset")
public List<DatasetImportModels> getDatasetImportModels() { return datasetImportModels; }
public void setDatasetImportModels(List<DatasetImportModels> datasetImportModels) { this.datasetImportModels = datasetImportModels; }
@XmlElement(name = "language")
public String getLanguage() {
return language;
}
public void setLanguage(String language) {
this.language = language;
}
@XmlElement(name = "visibility")
public Boolean getVisibility() {
return visibility;
}
public void setVisibility(Boolean visibility) {
this.visibility = visibility;
}
@XmlElement(name = "publicDate")
public String getPublicDate() {
return publicDate;
}
public void setPublicDate(String publicDate) {
this.publicDate = publicDate;
}
@XmlElement(name = "costs")
public String getCosts() {
return costs;
}
public void setCosts(String costs) {
this.costs = costs;
}
}

View File

@ -25,7 +25,7 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
private String label;
private String grant;
// private String profile;
// private Date creationTime;
private Date creationTime;
private Date modifiedTime;
//private String organisations;
private int version;
@ -70,12 +70,12 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
this.profile = profile;
}*/
/*public Date getCreationTime() {
public Date getCreationTime() {
return creationTime;
}
public void setCreationTime(Date creationTime) {
this.creationTime = creationTime;
}*/
}
public Date getModifiedTime() {
return modifiedTime;
@ -187,7 +187,7 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
this.id = entity.getId().toString();
this.label = entity.getLabel();
this.groupId = entity.getGroupId();
// this.creationTime = entity.getCreated();
this.creationTime = entity.getCreated();
// this.associatedProfiles = entity.getAssociatedDmps().stream().map(item -> new AssociatedProfile().fromData(item)).collect(Collectors.toList());
return this;
}
@ -196,7 +196,7 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
this.id = entity.getId().toString();
this.label = entity.getLabel();
this.groupId = entity.getGroupId();
// this.creationTime = entity.getCreated();
this.creationTime = entity.getCreated();
return this;
}
@ -205,8 +205,8 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
this.status = entity.getStatus();
this.version = entity.getVersion();
this.grant = entity.getGrant().getLabel();
/*if (entity.getProfile() != null) this.profile = entity.getProfile().getLabel();
this.creationTime = entity.getCreated();*/
/*if (entity.getProfile() != null) this.profile = entity.getProfile().getLabel(); */
this.creationTime = entity.getCreated();
this.modifiedTime = entity.getModified();
// this.organisations = LabelBuilder.getLabel(entity.getOrganisations().stream().map(item -> new Organisation().fromDataModel(item)).collect(Collectors.toList()));
this.datasets = entity.getDataset().stream().map(x-> new DatasetUrlListing().fromDataModel(x)).collect(Collectors.toList());

View File

@ -199,11 +199,11 @@ public class DatasetListingModel implements DataModel<Dataset, DatasetListingMod
this.profile = entity.getProfile() != null ? new DatasetProfileOverviewModel().fromDataModel(entity.getProfile()) : null;
this.description = entity.getDescription();
this.status = entity.getStatus();
this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
//this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
// this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
//this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
// this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
//this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
if (entity.getFinalizedAt() == null && entity.getStatus() == Dataset.Status.FINALISED.getValue()) {
this.finalizedAt = entity.getDmp().getFinalizedAt();
} else {

View File

@ -17,6 +17,7 @@ import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@ -201,13 +202,10 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
@Override
public void fromJsonObject(Map<String, Object> properties) {
try {
JSONArray jsonArray = new JSONArray(properties.get(this.id).toString());
List<String> stringList = new LinkedList<>();
for (int i = 0; i < jsonArray.length(); i++) {
stringList.add(jsonArray.getJSONObject(i).toString());
}
ObjectMapper mapper = new ObjectMapper();
List<String> stringList = mapper.readValue(properties.get(this.id).toString(), LinkedList.class);
this.value = stringList;
} catch (JSONException | NullPointerException e) {
} catch (JSONException | NullPointerException | IOException e) {
this.value = (String) properties.get(this.id);
}
this.multiplicityItems = new LinkedList<>();
@ -231,7 +229,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
@Override
public void toMap(Map<String, Object> fieldValues) {
if (this.value != null) {
if (this.viewStyle != null && this.viewStyle.getRenderStyle().equals("datasetIdentifier")) {
if ((this.viewStyle != null && this.viewStyle.getRenderStyle().equals("datasetIdentifier") || this.value instanceof Collection)) {
ObjectMapper mapper = new ObjectMapper();
String valueString = null;
try {
@ -240,7 +238,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
} catch (JsonProcessingException e) {
logger.error(e.getMessage(), e);
}
} else if (this.value instanceof Collection) {
} /*else if (this.value instanceof Collection) {
Collection valueCollection = (Collection) this.value;
StringBuilder valueBuilder = new StringBuilder();
valueBuilder.append("[");
@ -252,7 +250,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
}
valueBuilder.append("]");
fieldValues.put(this.id, valueBuilder.toString());
} else {
}*/ else {
fieldValues.put(this.id, this.value.toString());
}
} else {

View File

@ -175,12 +175,12 @@ public class FieldSet implements Comparable, PropertiesModelBuilder, ViewStyleDe
private FieldSet CloneForMultiplicity2(List<String> key, Map<String, Object> properties,String[] ids, int index){
FieldSet newFieldSet = new FieldSet();
newFieldSet.id = ids[0]+"_"+ids[1]+"_"+ids[2];
newFieldSet.id = ids[0]+"_"+ids[1]+"_"+ids[2] + (ids.length > 4 ? "_" + ids[3] : "");
newFieldSet.description = this.description;
newFieldSet.extendedDescription = this.extendedDescription;
newFieldSet.additionalInformation=this.additionalInformation;
newFieldSet.title = this.title;
newFieldSet.ordinal = this.ordinal;
newFieldSet.ordinal = ids.length > 4 ? Integer.valueOf(ids[3]) : this.ordinal;
newFieldSet.fields = new LinkedList();
for (Field field: this.fields) {

View File

@ -9,6 +9,8 @@ import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.*;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import eu.eudat.logic.utilities.json.MultiDateDeserializer;
/**
@ -71,6 +73,7 @@ public class Dmp implements Serializable
*
*/
@JsonProperty("created")
@JsonDeserialize(using = MultiDateDeserializer.class)
@JsonPropertyDescription("")
private Date created;
/**
@ -147,6 +150,7 @@ public class Dmp implements Serializable
*
*/
@JsonProperty("modified")
@JsonDeserialize(using = MultiDateDeserializer.class)
@JsonPropertyDescription("Must be set each time DMP is modified. Indicates DMP version.")
private Date modified;
/**

View File

@ -20,6 +20,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.transaction.Transactional;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -152,7 +156,12 @@ public class DatasetRDAMapper {
templateIdsToValues.entrySet().forEach(entry -> {
boolean isFound = foundNodes.stream().anyMatch(node -> node.get("id").asText().equals(entry.getKey()));
if (!isFound && entry.getValue() != null && !entry.getValue().toString().isEmpty()) {
rda.setAdditionalProperty(entry.getKey(), entry.getValue());
try {
Instant time = Instant.parse(entry.getValue().toString());
rda.setAdditionalProperty(entry.getKey(), DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()).format(time));
} catch (DateTimeParseException e) {
rda.setAdditionalProperty(entry.getKey(), entry.getValue());
}
}
});

View File

@ -39,9 +39,9 @@ public class DmpRDAMapper {
throw new IllegalArgumentException("DMP is missing language and contact properties");
} else {
extraProperties = new org.json.JSONObject(dmp.getExtraProperties()).toMap();
if (extraProperties.get("language") == null) {
/*if (extraProperties.get("language") == null) {
throw new IllegalArgumentException("DMP must have it's language property defined");
}
}*/
if (extraProperties.get("contact") == null) {
throw new IllegalArgumentException("DMP must have it's contact property defined");
}
@ -72,7 +72,7 @@ public class DmpRDAMapper {
} else {
rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN);
}
rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(extraProperties.get("language").toString()));
rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(extraProperties.get("language") != null ? extraProperties.get("language").toString() : "en"));
if (extraProperties.get("costs") != null) {
rda.setCost(new ArrayList<>());
((List) extraProperties.get("costs")).forEach(costl -> {
@ -95,7 +95,7 @@ public class DmpRDAMapper {
rda.getContributor().addAll(dmp.getResearchers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList()));
}
// rda.getContributor().addAll(dmp.getUsers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList()));
rda.setDataset(dmp.getDataset().stream().map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor())).collect(Collectors.toList()));
rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor())).collect(Collectors.toList()));
rda.setProject(Collections.singletonList(ProjectRDAMapper.toRDA(dmp.getProject(), dmp.getGrant())));
rda.setAdditionalProperty("templates", dmp.getAssociatedDmps().stream().map(datasetProfile -> datasetProfile.getId().toString()).toArray());
return rda;
@ -127,9 +127,11 @@ public class DmpRDAMapper {
entity.setDescription(rda.getDescription());
DatasetProfile defaultProfile = ((DatasetProfile)entity.getAssociatedDmps().toArray()[0]);
entity.setDataset(rda.getDataset().stream().map(rda1 -> datasetRDAMapper.toEntity(rda1, defaultProfile)).collect(Collectors.toSet()));
Map<String, Object> result = ProjectRDAMapper.toEntity(rda.getProject().get(0), apiContext);
entity.setProject((Project) result.get("project"));
result.entrySet().stream().filter(entry -> entry.getKey().startsWith("grant")).forEach(entry -> entity.setGrant((Grant) entry.getValue()));
if (rda.getProject().size() > 0) {
Map<String, Object> result = ProjectRDAMapper.toEntity(rda.getProject().get(0), apiContext);
entity.setProject((Project) result.get("project"));
result.entrySet().stream().filter(entry -> entry.getKey().startsWith("grant")).forEach(entry -> entity.setGrant((Grant) entry.getValue()));
}
Map<String, Object> extraProperties = new HashMap<>();
extraProperties.put("language", LanguageRDAMapper.mapRDAIsoToLanguageIso(rda.getLanguage()));

View File

@ -17,9 +17,11 @@ public class KeywordRDAMapper {
ObjectMapper mapper = new ObjectMapper();
try {
value = JavaToJson.objectStringToJson(value);
List<Tag> tags = Arrays.asList(mapper.readValue(value, Tag[].class));
List<String> keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList());
return keywordNames;
if (!value.isEmpty()) {
List<Tag> tags = Arrays.asList(mapper.readValue(value, Tag[].class));
List<String> keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList());
return keywordNames;
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
}

View File

@ -1,28 +1,29 @@
package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.models.rda.Language;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
public class LanguageRDAMapper {
private final static Map<String, Object> langMap = new HashMap<>();
private static final Logger logger = LoggerFactory.getLogger(LanguageRDAMapper.class);
static {
String json = null;
try {
json = new BufferedReader(new InputStreamReader(LanguageRDAMapper.class.getClassLoader().getResource("internal/rda-lang-map.json").openStream(), StandardCharsets.UTF_8))
.lines().collect(Collectors.joining("\n"));
langMap.putAll(new org.json.JSONObject(json).toMap());
ObjectMapper mapper = new ObjectMapper();
InputStreamReader isr = new InputStreamReader(LanguageRDAMapper.class.getClassLoader().getResource("internal/rda-lang-map.json").openStream(), StandardCharsets.UTF_8);
langMap.putAll(mapper.readValue(isr, LinkedHashMap.class));
isr.close();
} catch (IOException e) {
logger.error(e.getMessage(), e);
}

Some files were not shown because too many files have changed in this diff Show More