Merge commit '2cb985c23c4ceda99efc78bcc8e77b3eceeb0bcf'

# Conflicts:
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/criteria/DatasetCriteria.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dmp.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java
#	dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DmpRepository.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/criteria/DmpCriteriaMapper.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java
#	dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DataManagementPlanListingModel.java

.gitignore

@@ -40,3 +40,9 @@ dmp-frontend/.vscode/
 dmp-frontend/package-lock.json
 dmp-backend/logging/target/
 ELK.Docker/shared/data-elk/
+
+# Eclipse
+.project
+.settings/
+bin/
+*.classpath

dmp-backend/Dockerfile.CI

@@ -1,4 +1,4 @@
-FROM maven:3-jdk-8-alpine AS MAVEN_BUILD
+FROM maven:3-openjdk-11 AS MAVEN_BUILD
 
 COPY pom.xml /build/
 COPY data /build/data/
@@ -9,7 +9,7 @@ COPY web /build/web/
 WORKDIR /build/
 RUN mvn package
 
-FROM openjdk:8-jre-alpine
+FROM adoptopenjdk/openjdk11:alpine-jre
 WORKDIR /app
 COPY --from=MAVEN_BUILD /build/web/target/web-1.0-SNAPSHOT.jar /app/app.jar
-ENTRYPOINT ["java", "-Djava.security.egd=file:/dev/./urandom", "-Dspring.profiles.active=${PROF}", "-Dspring.config.location=/files/config/", "-jar","/app/app.jar"]
+ENTRYPOINT ["java", "-Djava.security.egd=file:/dev/./urandom", "-Dspring.profiles.active=${PROF}", "-Dspring.config.additional-location=/files/config/", "-jar","/app/app.jar"]

dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/DatasetProfileCriteria.java

@@ -2,6 +2,8 @@ package eu.eudat.data.dao.criteria;
 
 
 import eu.eudat.data.entities.DatasetProfile;
+
+import java.util.Date;
 import java.util.List;
 import java.util.UUID;
 
@@ -35,6 +37,8 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
     private boolean finalized;
     private Integer status;
     private Integer role;
+    private List<UUID> ids;
+    private Date periodStart;
 
     public boolean getAllVersions() { return allVersions; }
     public void setAllVersions(boolean allVersions) { this.allVersions = allVersions; }
@@ -78,4 +82,20 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
     public void setRole(Integer role) {
         this.role = role;
     }
+
+    public List<UUID> getIds() {
+        return ids;
+    }
+
+    public void setIds(List<UUID> ids) {
+        this.ids = ids;
+    }
+
+    public Date getPeriodStart() {
+        return periodStart;
+    }
+
+    public void setPeriodStart(Date periodStart) {
+        this.periodStart = periodStart;
+    }
 }

dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/FunderCriteria.java

@@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
 
 import eu.eudat.data.entities.Funder;
 
+import java.util.Date;
+
 public class FunderCriteria extends Criteria<Funder> {
 	private String reference;
 	private String exactReference;
+	private Date periodStart;
 
 	public String getReference() {
 		return reference;
@@ -20,4 +23,12 @@ public class FunderCriteria extends Criteria<Funder> {
 	public void setExactReference(String exactReference) {
 		this.exactReference = exactReference;
 	}
+
+	public Date getPeriodStart() {
+		return periodStart;
+	}
+
+	public void setPeriodStart(Date periodStart) {
+		this.periodStart = periodStart;
+	}
 }

dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/OrganisationCriteria.java

@@ -5,6 +5,7 @@ import eu.eudat.data.entities.Organisation;
 public class OrganisationCriteria extends Criteria<Organisation> {
     private String labelLike;
     private Boolean isPublic;
+    private boolean isActive;
 
     public String getLabelLike() {
         return labelLike;
@@ -19,4 +20,12 @@ public class OrganisationCriteria extends Criteria<Organisation> {
     public void setPublic(Boolean aPublic) {
         isPublic = aPublic;
     }
+
+    public boolean isActive() {
+        return isActive;
+    }
+
+    public void setActive(boolean active) {
+        isActive = active;
+    }
 }

dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/ProjectCriteria.java

@@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
 
 import eu.eudat.data.entities.Project;
 
+import java.util.Date;
+
 public class ProjectCriteria extends Criteria<Project> {
 	private String reference;
 	private String exactReference;
+	private Date periodStart;
 
 	public String getReference() {
 		return reference;
@@ -20,4 +23,12 @@ public class ProjectCriteria extends Criteria<Project> {
 	public void setExactReference(String exactReference) {
 		this.exactReference = exactReference;
 	}
+
+	public Date getPeriodStart() {
+		return periodStart;
+	}
+
+	public void setPeriodStart(Date periodStart) {
+		this.periodStart = periodStart;
+	}
 }

dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/ResearcherCriteria.java

@@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
 
 import eu.eudat.data.entities.Researcher;
 
+import java.util.Date;
+
 public class ResearcherCriteria extends Criteria<Researcher> {
     private String name;
     private String reference;
+    private Date periodStart;
 
     public String getName() {
         return name;
@@ -21,4 +24,12 @@ public class ResearcherCriteria extends Criteria<Researcher> {
     public void setReference(String reference) {
         this.reference = reference;
     }
+
+    public Date getPeriodStart() {
+        return periodStart;
+    }
+
+    public void setPeriodStart(Date periodStart) {
+        this.periodStart = periodStart;
+    }
 }

dmp-backend/data/src/main/java/eu/eudat/data/dao/databaselayer/service/DatabaseService.java

@@ -7,6 +7,7 @@ import eu.eudat.queryable.queryableentity.DataEntity;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.transaction.Transactional;
 import java.util.Set;
 
 
@@ -28,6 +29,7 @@ public class DatabaseService<T extends DataEntity> {
         return this.databaseCtx.getQueryable(tClass);
     }
 
+    @Transactional
     public T createOrUpdate(T item, Class<T> tClass) {
         return this.databaseCtx.createOrUpdate(item, tClass);
     }

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DMPDaoImpl.java

@@ -16,6 +16,7 @@ import org.springframework.stereotype.Component;
 
 import javax.persistence.criteria.Join;
 import javax.persistence.criteria.JoinType;
+import javax.transaction.Transactional;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
@@ -106,6 +107,7 @@ public class DMPDaoImpl extends DatabaseAccess<DMP> implements DMPDao {
 	}
 
 	@Override
+	@Transactional
 	public DMP createOrUpdate(DMP item) {
 		return this.getDatabaseService().createOrUpdate(item, DMP.class);
 	}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDao.java

@@ -13,6 +13,8 @@ public interface DatasetDao extends DatabaseAccessLayer<Dataset, UUID> {
 
     QueryableList<Dataset> getWithCriteria(DatasetCriteria criteria);
 
+    QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids);
+
     QueryableList<Dataset> getAuthenticated(QueryableList<Dataset> query, UserInfo principal, List<Integer> roles);
 
     Dataset isPublicDataset(UUID id);

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetDaoImpl.java

@@ -87,6 +87,16 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
         return query;
     }
 
+    public QueryableList<Dataset> filterFromElastic(DatasetCriteria criteria, List<UUID> ids) {
+        QueryableList<Dataset> query = getDatabaseService().getQueryable(Dataset.getHints(), Dataset.class);
+
+        query.where(((builder, root) -> root.get("id").in(ids)));
+        if (!criteria.getAllVersions())
+            query.initSubQuery(String.class).where((builder, root) -> builder.equal(root.get("dmp").get("version"), query.<String>subQueryMax((builder1, externalRoot, nestedRoot) -> builder1.and(builder1.equal(externalRoot.get("dmp").get("groupId"), nestedRoot.get("dmp").get("groupId")), builder1.notEqual(nestedRoot.get("dmp").get("status"), DMP.DMPStatus.DELETED.getValue())), Arrays.asList(new SelectionField(FieldSelectionType.COMPOSITE_FIELD, "dmp:version")), String.class)));
+
+        return query;
+    }
+
     @Override
     public Dataset createOrUpdate(Dataset item) {
         return getDatabaseService().createOrUpdate(item, Dataset.class);
@@ -121,6 +131,7 @@ public class DatasetDaoImpl extends DatabaseAccess<Dataset> implements DatasetDa
             });
         } else {
             query.where((builder, root) -> builder.equal(root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT).join("user", JoinType.LEFT).get("id"), principal.getId()));
+
         }
         return query;
     }

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetProfileDaoImpl.java

@@ -58,11 +58,16 @@ public class DatasetProfileDaoImpl extends DatabaseAccess<DatasetProfile> implem
         if (criteria.getStatus() != null) {
             query.where(((builder, root) -> builder.equal(root.get("status"), criteria.getStatus())));
         }
+        if (criteria.getIds() != null) {
+            query.where(((builder, root) -> root.get("id").in(criteria.getIds())));
+        }
         if (criteria.getFinalized()) {
             query.where(((builder, root) -> builder.equal(root.get("status"), DatasetProfile.Status.FINALIZED.getValue())));
         } else {
             query.where(((builder, root) -> builder.notEqual(root.get("status"), DatasetProfile.Status.DELETED.getValue())));
         }
+        if (criteria.getPeriodStart() != null)
+            query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
         return query;
     }
 

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/FunderDaoImpl.java

@@ -31,6 +31,8 @@ public class FunderDaoImpl extends DatabaseAccess<Funder> implements FunderDao {
 			query.where((builder, root) -> builder.like(builder.upper(root.get("reference")), "%" + criteria.getReference().toUpperCase() + "%"));
 		if (criteria.getExactReference() != null)
 			query.where((builder, root) -> builder.like(builder.upper(root.get("reference")), criteria.getExactReference().toUpperCase()));
+		if (criteria.getPeriodStart() != null)
+			query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
 		query.where((builder, root) -> builder.notEqual(root.get("status"), Funder.Status.DELETED.getValue()));
 		return query;
 	}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/OrganisationDaoImpl.java

@@ -38,6 +38,11 @@ public class OrganisationDaoImpl extends DatabaseAccess<Organisation> implements
                 query.where((builder, root) -> builder.equal(root.join("dmps", JoinType.LEFT).get("status"), DMP.DMPStatus.FINALISED.getValue()));
             }
         }
+
+        if (criteria.isActive()) {
+            query.where((builder, root) -> builder.notEqual(root.join("dmps").get("status"), DMP.DMPStatus.DELETED.getValue())).distinct();
+        }
+
         return query;
     }
 

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/ProjectDaoImpl.java

@@ -30,6 +30,8 @@ public class ProjectDaoImpl extends DatabaseAccess<Project> implements ProjectDa
 			query.where((builder, root) -> builder.like(root.get("reference"), "%" + criteria.getReference() + "%"));
 		if (criteria.getExactReference() != null)
 			query.where((builder, root) -> builder.like(root.get("reference"), criteria.getExactReference()));
+		if (criteria.getPeriodStart() != null)
+			query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("startdate"), criteria.getPeriodStart()));
 		query.where((builder, root) -> builder.notEqual(root.get("status"), Project.Status.DELETED.getValue()));
 		return query;
 	}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/ResearcherDaoImpl.java

@@ -29,6 +29,8 @@ public class ResearcherDaoImpl extends DatabaseAccess<Researcher> implements Res
             query.where((builder, root) ->builder.or(builder.like(builder.lower(root.get("label")), "%" + criteria.getName().toLowerCase() + "%")));
         if (criteria.getReference() != null && !criteria.getReference().isEmpty())
             query.where((builder, root) ->builder.or(builder.like(root.get("reference"), criteria.getReference())));
+        if (criteria.getPeriodStart() != null)
+            query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
         return query;
     }
 

dmp-backend/data/src/main/java/eu/eudat/data/entities/Dataset.java

@@ -16,13 +16,14 @@ import java.util.stream.Collectors;
 @NamedEntityGraphs({
 		@NamedEntityGraph(
 				name = "datasetListingModel",
-				attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
-						@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),
-						@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode("profile"), @NamedAttributeNode("creator")},
+				attributeNodes = {/*@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"),
+						@NamedAttributeNode(value = "datasetExternalDatasets", subgraph = "datasetExternalDatasets"), @NamedAttributeNode("registries"),*/
+						@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode(value = "profile", subgraph = "profile"), @NamedAttributeNode("creator")},
 				subgraphs = {
 						@NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users"), @NamedAttributeNode("grant"), @NamedAttributeNode("organisations")}),
 						@NamedSubgraph(name = "datasetDataRepositories", attributeNodes = {@NamedAttributeNode("dataRepository")}),
-						@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")})
+						@NamedSubgraph(name = "datasetExternalDatasets", attributeNodes = {@NamedAttributeNode("externalDataset")}),
+						@NamedSubgraph(name = "profile", attributeNodes = {@NamedAttributeNode("label")})
 				}),
 
 		@NamedEntityGraph(

dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java

@@ -106,11 +106,11 @@ public class Grant implements DataEntity<Grant, UUID> {
 	@Column(name = "\"Definition\"", columnDefinition = "xml", nullable = true)
 	private String definition;
 
-	@Column(name = "\"StartDate\"", nullable = false)
+	@Column(name = "\"StartDate\"", nullable = true)
 	@Convert(converter = DateToUTCConverter.class)
 	private Date startdate = null;
 
-	@Column(name = "\"EndDate\"", nullable = false)
+	@Column(name = "\"EndDate\"", nullable = true)
 	@Convert(converter = DateToUTCConverter.class)
 	private Date enddate = null;
 

dmp-backend/data/src/main/java/eu/eudat/data/query/definition/helpers/ColumnOrderings.java

@@ -6,22 +6,22 @@ import java.util.List;
 
 public class ColumnOrderings {
 
-    private String[] fields;
+    private List<String> fields;
 
-    public String[] getFields() {
+    public List<String> getFields() {
         return fields;
     }
 
-    public void setFields(String[] fields) {
+    public void setFields(List<String> fields) {
         this.fields = fields;
     }
 
-    public Ordering[] getFieldOrderings()  {
+    public List<Ordering> getFieldOrderings()  {
         List<Ordering> orderings = new LinkedList<>();
         for (String field : fields) {
             orderings.add(this.orderingFromString(field));
         }
-        return orderings.toArray(new Ordering[orderings.size()]);
+        return orderings;
     }
 
     private Ordering orderingFromString(String field) {

dmp-backend/elastic/src/main/java/eu/eudat/elastic/criteria/DatasetCriteria.java

@@ -18,6 +18,7 @@ public class DatasetCriteria extends Criteria {
     private List<UUID> collaborators;
     private Boolean allowAllVersions;
     private List<String> organiztions;
+    private Boolean hasTags;
     private List<Tag> tags;
     private boolean isPublic;
     private Short grantStatus;
@@ -144,4 +145,12 @@ public class DatasetCriteria extends Criteria {
     public void setSortCriteria(List<SortCriteria> sortCriteria) {
         this.sortCriteria = sortCriteria;
     }
+
+    public Boolean getHasTags() {
+        return hasTags;
+    }
+
+    public void setHasTags(Boolean hasTags) {
+        this.hasTags = hasTags;
+    }
 }

dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java

@@ -283,6 +283,13 @@ public class Dataset implements ElasticEntity<Dataset> {
     @Override
     public Dataset fromElasticEntity(Map<String, Object> fields) {
         if (fields != null) {
+            if (fields.size() == 1) {
+                if (fields.containsKey("id")) {
+                    this.id = (String) fields.get("id");
+                } else if (fields.containsKey("tags")) {
+                    this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
+                }
+            }else if (fields.size() > 1) {
                 this.id = (String) fields.get("id");
                 if (fields.get("tags") != null) {
                     this.tags = ((List<HashMap>) fields.get("tags")).stream().map(hashMap -> new Tag().fromElasticEntity(hashMap)).collect(Collectors.toList());
@@ -316,6 +323,7 @@ public class Dataset implements ElasticEntity<Dataset> {
                 }
                 this.formData = (String) fields.get("formData");
             }
+        }
         return this;
     }
 

dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dmp.java

@@ -5,6 +5,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -286,7 +287,11 @@ public class Dmp implements ElasticEntity<Dmp> {
 
 	@Override
 	public Dmp fromElasticEntity(Map<String, Object> fields) {
+		if (fields == null || fields.isEmpty()) {
+			return null;
+		}
 		this.id = UUID.fromString((String) fields.get(MapKey.ID.getName()));
+		if (fields.size() > 1) {
 			this.label = (String) fields.get(MapKey.LABEL.getName());
 			this.description = (String) fields.get(MapKey.DESCRIPTION.getName());
 			if (fields.get(MapKey.GROUPID.getName()) != null) {
@@ -312,6 +317,18 @@ public class Dmp implements ElasticEntity<Dmp> {
 			if (fields.get(MapKey.GRANTSTATUS.getName()) != null) {
 				this.grantStatus = Short.valueOf(fields.get(MapKey.GRANTSTATUS.getName()).toString());
 			}
+			this.created = Date.from(Instant.parse(fields.get(MapKey.CREATED.getName()).toString()));
+			this.modified = Date.from(Instant.parse(fields.get(MapKey.MODIFIED.getName()).toString()));
+			if (fields.get(MapKey.FINALIZEDAT.getName()) != null) {
+				this.finalizedAt = Date.from(Instant.parse(fields.get(MapKey.FINALIZEDAT.getName()).toString()));
+			}
+			if (fields.get(MapKey.PUBLISHEDAT.getName()) != null) {
+				this.publishedAt = Date.from(Instant.parse(fields.get(MapKey.PUBLISHEDAT.getName()).toString()));
+			}
+			if (fields.get(MapKey.DOI.getName()) != null) {
+				this.doi = fields.get(MapKey.DOI.getName()).toString();
+			}
+		}
 		return this;
 	}
 

dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java

@@ -24,6 +24,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
 import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -54,6 +55,7 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
         if (this.getClient() != null) {
             XContentBuilder builder = XContentFactory.jsonBuilder();
             Dmp dmp = this.dmpRepository.findDocument(entity.getDmp().toString());
+            if (dmp != null) {
                 boolean found = false;
                 if (dmp.getDatasets() != null && !dmp.getDatasets().isEmpty()) {
                     for (int i = 0; i < dmp.getDatasets().size(); i++) {
@@ -72,6 +74,7 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
                 }
                 IndexRequest request = new IndexRequest(this.environment.getProperty("elasticsearch.index")).id(dmp.getId().toString()).source(dmp.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
                 this.getClient().index(request, RequestOptions.DEFAULT);
+            }
             return entity;
         }
         return null;
@@ -122,8 +125,128 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
 
             searchSourceBuilder.size(count.intValue());
 
-            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
             List<SortBuilder> sortBuilders = new ArrayList<>();
+            BoolQueryBuilder boolQuery = createBoolQuery(criteria);
+
+
+            if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
+                criteria.getSortCriteria().forEach(sortCriteria -> {
+                    switch(sortCriteria.getColumnType()) {
+                        case COLUMN:
+                            sortBuilders.add(SortBuilders.fieldSort("datasets." + sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
+                            break;
+                        case JOIN_COLUMN:
+                            List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
+                            fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
+                                sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
+                            });
+                            break;
+                    }
+                });
+
+            }
+
+            NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.tags"}, null)));
+            searchSourceBuilder.query(nestedQueryBuilder)/*.from(criteria.getOffset())*/.fetchSource("datasets.tags", null);
+            /*if (criteria.getSize() > 0) {
+                searchSourceBuilder.size(criteria.getSize());
+            }*/
+            sortBuilders.forEach(searchSourceBuilder::sort);
+            searchRequest.source(searchSourceBuilder);
+            SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
+            return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
+                                           .map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
+                                           .map(SearchHits::getHits).flatMap(Arrays::stream)
+                                           .map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
+        }
+        return null;
+    }
+
+    public List<Dataset> queryIds(DatasetCriteria criteria) throws IOException {
+        if (this.getClient() != null) {
+            SearchRequest searchRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+
+            /*CountRequest countRequest = new CountRequest("dmps").routing("datasets").routing("id");
+            countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
+            CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT);
+            Long count = countResponse.getCount();*/
+
+            SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
+            NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
+            FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
+            nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
+            SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
+            countSourceBuilder.aggregation(nestedAggregationBuilder);
+            countRequest.source(countSourceBuilder);
+            SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
+            Long count = ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
+
+
+            searchSourceBuilder.size(count.intValue());
+
+            List<SortBuilder> sortBuilders = new ArrayList<>();
+            BoolQueryBuilder boolQuery = createBoolQuery(criteria);
+
+
+            if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
+                criteria.getSortCriteria().forEach(sortCriteria -> {
+                    switch(sortCriteria.getColumnType()) {
+                        case COLUMN:
+                            sortBuilders.add(SortBuilders.fieldSort("datasets." + sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
+                            break;
+                        case JOIN_COLUMN:
+                            List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
+                            fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
+                                sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
+                            });
+                            break;
+                    }
+                });
+
+            }
+
+            NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, new String[]{"datasets.id"}, null)));
+            searchSourceBuilder.query(nestedQueryBuilder)/*.from(criteria.getOffset()).size(criteria.getSize())*/.fetchSource("datasets.id", null);
+            sortBuilders.forEach(searchSourceBuilder::sort);
+            searchRequest.source(searchSourceBuilder);
+            SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
+            return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
+                    .map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
+                    .map(SearchHits::getHits).flatMap(Arrays::stream)
+                    .map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
+        }
+        return null;
+    }
+
+    @Override
+    public Long count(DatasetCriteria criteria) throws IOException {
+        if (this.getClient() != null) {
+            //CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
+
+            SearchRequest countRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index"));
+            BoolQueryBuilder boolQuery = createBoolQuery(criteria);
+            NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
+            FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", boolQuery);
+            nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
+            SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
+            countSourceBuilder.aggregation(nestedAggregationBuilder);
+            countRequest.source(countSourceBuilder);
+            SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
+            return  ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
+
+
+
+            /*NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.None).innerHit(new InnerHitBuilder());
+            countRequest.query(nestedQueryBuilder);
+            CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
+            return response.getCount();*/
+        }
+        return null;
+    }
+
+    private BoolQueryBuilder createBoolQuery(DatasetCriteria criteria) {
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
         if (criteria.isPublic()) {
             boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
             boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
@@ -162,7 +285,7 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
         }
 
         if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
-                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
+            boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
         }
 
         if (!criteria.isPublic()) {
@@ -179,40 +302,19 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
             boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
         }
 
+        if (criteria.getHasTags() != null) {
+            boolQuery = criteria.getHasTags() == true ? boolQuery.should(QueryBuilders.existsQuery("datasets.tags.id")) : boolQuery.mustNot(QueryBuilders.existsQuery("datasets.tags.id"));
+        }
+
         if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
             boolQuery.should(QueryBuilders.matchAllQuery());
         } else {
             boolQuery.minimumShouldMatch(boolQuery.should().size());
         }
 
-            if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
-                criteria.getSortCriteria().forEach(sortCriteria -> {
-                    switch(sortCriteria.getColumnType()) {
-                        case COLUMN:
-                            sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
-                            break;
-                        case JOIN_COLUMN:
-                            List<String> fields = Arrays.asList(sortCriteria.getFieldName().split(":"));
-                            fields.stream().filter(name -> !name.startsWith("dmp")).forEach(field -> {
-                                sortBuilders.add(SortBuilders.fieldSort(field).order(SortOrder.fromString(sortCriteria.getOrderByType().name())));
-                            });
-                            break;
-                    }
-                });
 
-            }
 
-            NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
-            searchSourceBuilder.query(nestedQueryBuilder).from(criteria.getOffset()).size(criteria.getSize());
-            sortBuilders.forEach(searchSourceBuilder::sort);
-            searchRequest.source(searchSourceBuilder);
-            SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
-            return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
-                                           .map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
-                                           .map(SearchHits::getHits).flatMap(Arrays::stream)
-                                           .map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
-        }
-        return null;
+        return boolQuery;
     }
 
     @Override

dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DmpRepository.java

@@ -99,60 +99,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
 
 			searchSourceBuilder.size(count.intValue());
 
-			BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
 			List<SortBuilder> sortBuilders = new ArrayList<>();
-			if (criteria.isPublic()) {
-				boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
-				boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
-			}
-			if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
-				boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
-						{Dmp.MapKey.LABEL.getName(), 1.0f},
-						{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
-				}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
-			}
-
-			if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) {
-				boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
-			}
-
-			if (criteria.getStatus() != null) {
-				boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue()));
-			}
-
-			if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
-				boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
-			}
-
-			if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
-				boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
-			}
-
-			if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
-				boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
-			}
-
-			if (criteria.getRoles() != null && criteria.getRoles().size() > 0) {
-				boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".role.keyword", criteria.getRoles()));
-			}
-
-			if (!criteria.isAllowAllVersions()) {
-				boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true));
-			}
-
-			if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) {
-				boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList())));
-			}
-
-			if (criteria.getGrantStatus() != null) {
-				boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus()));
-			}
-
-			if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
-				boolQuery = boolQuery.should(QueryBuilders.matchAllQuery());
-			} else {
-				boolQuery.minimumShouldMatch(boolQuery.should().size());
-			}
+			BoolQueryBuilder boolQuery = createBoolQuery(criteria);
 
 			if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) {
 				criteria.getSortCriteria().forEach(sortCriteria -> {
@@ -170,8 +118,8 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
 				});
 
 			}
-			searchSourceBuilder.query(boolQuery).from(criteria.getOffset());
-			if (criteria.getSize() != null) {
+			searchSourceBuilder.query(boolQuery).from(criteria.getOffset()).fetchSource("id", null);
+			if (criteria.getSize() != null && criteria.getSize() > 0) {
 				searchSourceBuilder.size(criteria.getSize());
 			}
 			sortBuilders.forEach(searchSourceBuilder::sort);
@@ -182,10 +130,21 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
 		return null;
 	}
 
+	@Override
 	public Long count(DmpCriteria criteria) throws IOException {
 		if (this.getClient() != null) {
 			CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index"));
 
+			BoolQueryBuilder boolQuery = createBoolQuery(criteria);
+
+			countRequest.query(boolQuery);
+			CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
+			return response.getCount();
+		}
+		return null;
+	}
+
+	private BoolQueryBuilder createBoolQuery(DmpCriteria criteria) {
 		BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())));
 		if (criteria.isPublic()) {
 			boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true));
@@ -215,7 +174,7 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
 		}
 
 		if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
-				boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
+			boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().filter(Objects::nonNull).map(UUID::toString).collect(Collectors.toList())));
 		}
 
 		if (!criteria.isAllowAllVersions()) {
@@ -235,12 +194,7 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
 		} else {
 			boolQuery.minimumShouldMatch(boolQuery.should().size());
 		}
-
-			countRequest.query(boolQuery);
-			CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT);
-			return response.getCount();
-		}
-		return null;
+		return boolQuery;
 	}
 
 	public boolean createIndex() {

dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/Repository.java

@@ -18,6 +18,8 @@ public interface Repository<ET extends ElasticEntity, C extends Criteria> {
 
     List<ET> query(C criteria) throws ExecutionException, InterruptedException, IOException;
 
+    Long count(C criteria) throws ExecutionException, InterruptedException, IOException;
+
     boolean exists() throws IOException;
 
     void clear() throws IOException;

dmp-backend/pom.xml

@@ -10,7 +10,7 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>1.5.9.RELEASE</version>
+        <version>2.5.2</version>
     </parent>
 
     <modules>
@@ -27,25 +27,26 @@
         <java.version>1.8</java.version>
 
         <dmp-backend-commons.version>0.0.1-SNAPSHOT</dmp-backend-commons.version>
-        <org.springframework.version>4.3.8.RELEASE</org.springframework.version>
+        <org.springframework.version>5.3.8</org.springframework.version>
         <!-- 		<org.springframeweu.eudat.logic.securityrity.version>3.2.10.RELEASE</org.springframeweu.eudat.logic.securityrity.version> -->
-        <org.springframework.security.version>4.2.3.RELEASE</org.springframework.security.version>
-        <com.sun.jersey.version>1.19.1</com.sun.jersey.version>
+        <org.springframework.security.version>5.3.10.RELEASE</org.springframework.security.version>
+        <!--<com.sun.jersey.version>1.19.1</com.sun.jersey.version>-->
+<!--
         <org.apache.tomcat.tomcat-jdbc.version>7.0.35</org.apache.tomcat.tomcat-jdbc.version>
-        <com.fasterxml.jackson>2.8.4</com.fasterxml.jackson>
+-->
+        <!--<com.fasterxml.jackson>2.9.0</com.fasterxml.jackson>-->
 
-        <hibernate.version>5.2.11.Final</hibernate.version>
+        <hibernate.version>5.5.3.Final</hibernate.version>
 
         <commons-codec.version>1.9</commons-codec.version>
         <org.junit.version>4.11</org.junit.version>
         <log4j.version>1.2.17</log4j.version>
         <slf4j.version>1.7.12</slf4j.version>
-        <jetty.version>9.0.7.v20131107
-        </jetty.version> <!-- Adapt this to a version found on http://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-maven-plugin/ -->
+        <!--<jetty.version>11.0.5
+        </jetty.version>--> <!-- Adapt this to a version found on http://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-maven-plugin/ -->
         <logback.version>1.2.3</logback.version>
-        <javax.inject.version>1</javax.inject.version>
-        <javax.servlet.servlet-api.version>3.0.1</javax.servlet.servlet-api.version>
-        <docker.image.prefix>ikalyvas</docker.image.prefix>
+       <!-- <javax.inject.version>1</javax.inject.version>-->
+        <!--<javax.servlet.servlet-api.version>3.0.1</javax.servlet.servlet-api.version>-->
     </properties>
 
     <dependencies>
@@ -58,7 +59,7 @@
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-context-support</artifactId>
-            <version>4.0.0.RELEASE</version>
+            <version>${org.springframework.version}</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -139,7 +140,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-xml</artifactId>
-            <version>2.9.3</version>
+            <version>2.12.3</version>
         </dependency>
 
 
@@ -147,13 +148,13 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
-            <version>2.9.4</version>
+            <version>2.12.3</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/com.fasterxml.jackeu.eudat.corecore/jackson-databind -->
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.9.4</version>
+            <version>2.12.3</version>
         </dependency>
 
         <!-- g/a spring -->
@@ -186,6 +187,12 @@
             <artifactId>fop</artifactId>
             <version>2.3</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.14.3</version>
+        </dependency>
         <!-- https://mvnrepository.com/artifact/fr.opensagres.xdocreport/org.apache.poi.xwpf.converter.pdf -->
         <dependency>
             <groupId>fr.opensagres.xdocreport</groupId>
@@ -257,36 +264,53 @@
             <version>1.3.1</version>
         </dependency>
 
+        <!-- https://mvnrepository.com/artifact/javax.validation/validation-api -->
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+            <version>2.0.1.Final</version>
+        </dependency>
+
+
         <!-- The client -->
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient</artifactId>
-            <version>0.10.0</version>
+            <version>0.11.0</version>
         </dependency>
         <!-- Hotspot JVM metrics-->
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient_hotspot</artifactId>
-            <version>0.10.0</version>
+            <version>0.11.0</version>
         </dependency>
         <!-- Exposition HTTPServer-->
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient_httpserver</artifactId>
-            <version>0.10.0</version>
+            <version>0.11.0</version>
         </dependency>
         <!-- Pushgateway exposition-->
         <dependency>
             <groupId>io.prometheus</groupId>
             <artifactId>simpleclient_pushgateway</artifactId>
-            <version>0.10.0</version>
+            <version>0.11.0</version>
         </dependency>
 
-        <!-- Pushgateway exposition-->
+        <!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus -->
         <dependency>
-            <groupId>io.prometheus</groupId>
-            <artifactId>simpleclient_spring_boot</artifactId>
-            <version>0.10.0</version>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <version>1.7.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+        <!-- Micormeter core dependecy  -->
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-core</artifactId>
         </dependency>
 
     </dependencies>

dmp-backend/queryable/src/main/java/eu/eudat/queryable/QueryableList.java

@@ -44,6 +44,8 @@ public interface QueryableList<T extends DataEntity> {
 
     QueryableList<T> orderBy(OrderByPredicate<T> predicate);
 
+    QueryableList<T> groupBy(GroupByPredicate<T> predicate);
+
     QueryableList<T> withHint(String hint);
 
     Long count();

dmp-backend/queryable/src/main/java/eu/eudat/queryable/collector/Collector.java

@@ -1,6 +1,9 @@
 package eu.eudat.queryable.collector;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import javax.persistence.Tuple;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -9,9 +12,13 @@ import java.util.stream.Collectors;
 public class Collector {
 
 	public List<Map> buildFromTuple(List<Tuple> results, Map<Object, List<Tuple>> groupedResults, List<String> fields, String key) {
-		return groupedResults.keySet().stream()
+		return results.stream().map(tuple -> {
+			Map<String, Object> parsedResult = new HashMap<>();
+			tuple.getElements().forEach(tupleElement -> parsedResult.put(tupleElement.getAlias(), tuple.get(tupleElement.getAlias())));
+			return parsedResult;
+		}).collect(Collectors.toList());/*groupedResults.keySet().stream()
 				.map(x -> buildOne(groupedResults.get(x), fields, key))
-				.collect(Collectors.toList());
+				.collect(Collectors.toList());*/
 	}
 
 	private Map buildOne(List<Tuple> tuples, List<String> fields, String key) {

dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/hibernatequeryablelist/QueryableHibernateList.java

@@ -1,5 +1,6 @@
 package eu.eudat.queryable.jpa.hibernatequeryablelist;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.queryable.QueryableList;
 import eu.eudat.queryable.collector.Collector;
 import eu.eudat.queryable.exceptions.NotSingleResultException;
@@ -33,6 +34,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 	private List<NestedQuerySinglePredicate<T>> nestedPredicates = new LinkedList<>();
 	private boolean distinct = false;
 	private List<OrderByPredicate<T>> orderings = new LinkedList<>();
+	private List<GroupByPredicate<T>> groupings = new LinkedList<>();
 	private List<String> fields = new LinkedList<>();
 	private Integer length;
 	private Integer offset;
@@ -150,6 +152,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		return this;
 	}
 
+	public QueryableList<T> groupBy(GroupByPredicate<T> predicate) {
+		this.groupings.add(predicate);
+		return this;
+	}
+
 	public Long count() {
 		CriteriaBuilder criteriaBuilder = this.manager.getCriteriaBuilder();
 		CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
@@ -157,8 +164,16 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		if (distinct) criteriaQuery.select(criteriaBuilder.countDistinct(this.root.get("id")));
 		else criteriaQuery.select(criteriaBuilder.count(this.root));
 		criteriaQuery.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
+		if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
 		//if (distinct) criteriaQuery.distinct(true);
+
+		//GK: Group By special case. When group by is used, since it will result in a list of how many elements have the grouped field common
+		// then it will instead return the number of the distinct values of the grouped field
+		if (this.groupings.isEmpty()) {
 			return this.manager.createQuery(criteriaQuery).getSingleResult();
+		} else {
+			return (long) this.manager.createQuery(criteriaQuery).getResultList().size();
+		}
 	}
 
 	@Async
@@ -169,8 +184,15 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		if (distinct) criteriaQuery.select(criteriaBuilder.countDistinct(this.root.get("id")));
 		else criteriaQuery.select(criteriaBuilder.count(this.root));
 		criteriaQuery.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
+		if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
 		//if (distinct) criteriaQuery.distinct(true);
-		return CompletableFuture.supplyAsync(() -> this.manager.createQuery(criteriaQuery).getSingleResult());
+		return CompletableFuture.supplyAsync(() -> {
+			if (this.groupings.isEmpty()) {
+				return this.manager.createQuery(criteriaQuery).getSingleResult();
+			} else {
+				return (long) this.manager.createQuery(criteriaQuery).getResultList().size();
+			}
+		});
 	}
 
 
@@ -205,6 +227,14 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		return predicates.toArray(new Order[predicates.size()]);
 	}
 
+	private Expression[] generateGroupPredicates(List<GroupByPredicate<T>> groupByPredicates, Root<T> root) {
+		List<Expression> predicates = new LinkedList<>();
+		for (GroupByPredicate<T> groupPredicate : groupByPredicates) {
+			predicates.add(groupPredicate.applyPredicate(this.manager.getCriteriaBuilder(), root));
+		}
+		return predicates.toArray(new Expression[predicates.size()]);
+	}
+
 	public List<T> toList() {
 		CriteriaBuilder builder = this.manager.getCriteriaBuilder();
 		if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
@@ -212,22 +242,20 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		this.root = this.query.from(this.tClass);
 		this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
 		if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
+		if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
 		if (!this.fields.isEmpty()) this.selectFields();
 		if (distinct) this.query.distinct(true);
-		//if (!this.fields.isEmpty()) return this.toListWithFields();
+		//if (!this.fields.isEmpty()) this.query.multiselect(this.parseFields(this.fields));
+		ObjectMapper mapper = new ObjectMapper();
+		if (!this.fields.isEmpty()) return this.toListWithFields().stream().map(m -> mapper.convertValue(m, this.tClass)).collect(Collectors.toList());
 		return this.toListWithOutFields();
 	}
 
 	public List<Map> toListWithFields() {
-		CriteriaBuilder builder = this.manager.getCriteriaBuilder();
-		if (!this.fields.isEmpty()) this.query = builder.createTupleQuery();
-		else this.query = builder.createQuery(this.tClass);
-		this.root = this.query.from(this.tClass);
-		this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
-		if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
-		if (!this.fields.isEmpty()) this.selectFields();
-		if (distinct) this.query.distinct(true);
-		List<Tuple> results = this.manager.createQuery(query).getResultList();
+		TypedQuery<Tuple> typedQuery = this.manager.createQuery(this.query);
+		if (this.offset != null) typedQuery.setFirstResult(this.offset);
+		if (this.length != null) typedQuery.setMaxResults(this.length);
+		List<Tuple> results = typedQuery.getResultList();
 		Map<Object, List<Tuple>> groupedResults = results.stream()
 				.collect(Collectors.groupingBy(x -> x.get("id")));
 		return this.collector.buildFromTuple(results, groupedResults, this.fields, "id");
@@ -252,6 +280,7 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		this.root = this.query.from(this.tClass);
 		this.query.where(this.generateWherePredicates(this.singlePredicates, this.root, this.nestedPredicates, this.nestedQueryRoot));
 		if (!this.orderings.isEmpty()) this.query.orderBy(this.generateOrderPredicates(this.orderings, this.root));
+		if (!this.groupings.isEmpty()) this.query.groupBy(this.generateGroupPredicates(this.groupings, this.root));
 		if (!this.fields.isEmpty()) this.selectFields();
 		if (distinct) this.query.distinct(true);
 		if (!this.fields.isEmpty()) return this.toListAsyncWithFields();
@@ -356,6 +385,8 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 		if (!this.orderings.isEmpty())
 			criteriaQuery.orderBy(this.generateOrderPredicates(this.orderings, criteriaRoot));
 
+		if (!this.groupings.isEmpty()) criteriaQuery.groupBy(this.generateGroupPredicates(this.groupings, this.root));
+
 		TypedQuery<T> typedQuery = this.manager.createQuery(criteriaQuery);
 		typedQuery.setHint("javax.persistence.fetchgraph", this.manager.getEntityGraph(this.hint));
 		return typedQuery;
@@ -466,4 +497,11 @@ public class QueryableHibernateList<T extends DataEntity> implements QueryableLi
 				.createQuery(update)
 				.executeUpdate();
 	}
+
+	private Path[] parseFields(List<String> selectedFields) {
+		List<Path> paths = new ArrayList<>();
+		selectedFields.forEach(s -> paths.add(root.get(s)));
+
+		return paths.toArray(new Path[0]);
+	}
 }

dmp-backend/queryable/src/main/java/eu/eudat/queryable/jpa/predicates/GroupByPredicate.java

@@ -0,0 +1,10 @@
+package eu.eudat.queryable.jpa.predicates;
+
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.Expression;
+import javax.persistence.criteria.Root;
+
+public interface GroupByPredicate<T> {
+    Expression<T> applyPredicate(CriteriaBuilder builder, Root<T> root);
+
+}

dmp-backend/web/pom.xml

@@ -50,31 +50,37 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>
-            <version>2.9.0</version>
+            <version>2.12.3</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/org.springframework/spring-context -->
-        <dependency>
+        <!--<dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-context</artifactId>
-            <version>4.3.10.RELEASE</version>
-        </dependency>
+            <version>5.3.8</version>
+        </dependency>-->
 
         <!-- guava eu.eudat.cache -->
-        <dependency>
+       <!-- <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
-            <version>23.0</version>
-        </dependency>
+            <version>30.1.1-jre</version>
+        </dependency>-->
         <!-- https://mvnrepository.com/artifact/org.springframework/spring-context-support -->
 
+        <dependency>
+            <groupId>com.github.ben-manes.caffeine</groupId>
+            <artifactId>caffeine</artifactId>
+            <version>2.9.0</version>
+        </dependency>
+
 
 
         <!-- https://mvnrepository.com/artifact/junit/junit -->
         <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>5.7.2</version>
             <scope>test</scope>
         </dependency>
 
@@ -87,7 +93,7 @@
         <dependency>
             <groupId>io.springfox</groupId>
             <artifactId>springfox-swagger-ui</artifactId>
-            <version>2.7.0</version>
+            <version>3.0.0</version>
         </dependency>
 
         <!-- facebook Login -->
@@ -101,7 +107,7 @@
         <dependency>
             <groupId>org.springframework.social</groupId>
             <artifactId>spring-social-linkedin</artifactId>
-            <version>1.0.0.RELEASE</version>
+            <version>1.0.2.RELEASE</version>
         </dependency>
 
         <!-- tweeter login-->

dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java

@@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.boot.web.support.SpringBootServletInitializer;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
 import org.springframework.scheduling.annotation.EnableAsync;
 
 @SpringBootApplication

dmp-backend/web/src/main/java/eu/eudat/cache/ExternalUrlsKeyGenerator.java

@@ -0,0 +1,22 @@
+package eu.eudat.cache;
+
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
+import org.springframework.cache.interceptor.KeyGenerator;
+
+import java.lang.reflect.Method;
+
+public class ExternalUrlsKeyGenerator implements KeyGenerator {
+    @Override
+    public Object generate(Object o, Method method, Object... params) {
+        StringBuffer stringBuffer = new StringBuffer();
+        stringBuffer.append(o.getClass().getSimpleName()).append("_");
+        stringBuffer.append(method.getName()).append("_");
+        for (Object param: params) {
+            if (param instanceof ExternalUrlCriteria) {
+                ExternalUrlCriteria externalUrlCriteria = (ExternalUrlCriteria) param;
+                stringBuffer.append(externalUrlCriteria);
+            }
+        }
+        return stringBuffer.toString();
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java

@@ -1,12 +1,13 @@
 package eu.eudat.cache;
 
 
-import com.google.common.cache.CacheBuilder;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.cache.CacheManager;
 import org.springframework.cache.annotation.EnableCaching;
-import org.springframework.cache.guava.GuavaCache;
+import org.springframework.cache.caffeine.CaffeineCache;
+import org.springframework.cache.interceptor.KeyGenerator;
 import org.springframework.cache.support.SimpleCacheManager;
 import org.springframework.context.annotation.Bean;
 import org.springframework.stereotype.Component;
@@ -29,22 +30,27 @@ public class ResponsesCache {
     public CacheManager cacheManager() {
         logger.info("Loading ResponsesCache...");
         SimpleCacheManager simpleCacheManager = new SimpleCacheManager();
-        List<GuavaCache> caches = new ArrayList<GuavaCache>();
-        caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("grants", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("funders", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("currencies", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
-        caches.add(new GuavaCache("licenses", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        List<CaffeineCache> caches = new ArrayList<CaffeineCache>();
+        caches.add(new CaffeineCache("repositories", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("grants", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("projects", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("funders", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("organisations", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("registries", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("services", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("tags", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("researchers", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("externalDatasets", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("currencies", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+        caches.add(new CaffeineCache("licenses", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
         simpleCacheManager.setCaches(caches);
         logger.info("OK");
         return simpleCacheManager;
     }
 
+    @Bean(name = "externalUrlsKeyGenerator")
+    private KeyGenerator externalUrlsKeyGenerator() {
+        return new ExternalUrlsKeyGenerator();
+    }
+
 }

dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java

@@ -2,8 +2,8 @@ package eu.eudat.configurations;
 
 
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
 import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.jdbc.DataSourceBuilder;
 import org.springframework.context.annotation.*;
 import org.springframework.core.env.Environment;
 import org.springframework.dao.annotation.PersistenceExceptionTranslationPostProcessor;

dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java

@@ -4,7 +4,6 @@ import eu.eudat.controllers.interceptors.RequestInterceptor;
 import eu.eudat.logic.handlers.PrincipalArgumentResolver;
 import eu.eudat.logic.services.ApiContext;
 import eu.eudat.logic.services.operations.authentication.AuthenticationService;
-import io.prometheus.client.spring.boot.EnablePrometheusEndpoint;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.scheduling.annotation.EnableAsync;
@@ -18,7 +17,6 @@ import java.util.List;
 @EnableAsync
 @Configuration
 @EnableScheduling
-@EnablePrometheusEndpoint
 public class WebMVCConfiguration extends WebMvcConfigurerAdapter {
 
 	private ApiContext apiContext;

dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java

@@ -20,6 +20,7 @@ import eu.eudat.models.data.listingmodels.UserInfoListingModel;
 import eu.eudat.models.data.security.Principal;
 import eu.eudat.models.data.user.composite.PagedDatasetProfile;
 import eu.eudat.types.ApiMessageCode;
+import eu.eudat.types.MetricNames;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
@@ -44,13 +45,15 @@ public class Admin extends BaseController {
 	private DatasetProfileManager datasetProfileManager;
 	private UserManager userManager;
 	private ConfigLoader configLoader;
+	private final MetricsManager metricsManager;
 
 	@Autowired
-	public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader) {
+	public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader, MetricsManager metricsManager) {
 		super(apiContext);
 		this.datasetProfileManager = datasetProfileManager;
 		this.userManager = userManager;
 		this.configLoader = configLoader;
+		this.metricsManager = metricsManager;
 	}
 
 	@Transactional
@@ -70,7 +73,7 @@ public class Admin extends BaseController {
 		userDatasetProfile.setRole(0);
 		getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
 		datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile);
-		MetricsManager.increaseValue("argos_dataset_templates", 1, "total");
+		metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
 		return ResponseEntity.status(HttpStatus.OK).body(modelDefinition.getId());
 	}
 
@@ -89,7 +92,7 @@ public class Admin extends BaseController {
 		eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(datasetprofile);
 		datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile);
 		if (datasetProfile.getStatus() == 1 && oldStatus == 0) {
-			MetricsManager.increaseValue("argos_dataset_templates", 1, "active");
+			metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.ACTIVE);
 		}
 		return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<UUID>().status(ApiMessageCode.NO_MESSAGE));
 	}

dmp-backend/web/src/main/java/eu/eudat/controllers/DMPs.java

@@ -20,6 +20,7 @@ import eu.eudat.logic.security.claims.ClaimedAuthorities;
 import eu.eudat.logic.services.ApiContext;
 import eu.eudat.logic.services.operations.DatabaseRepository;
 import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
+import eu.eudat.logic.utilities.documents.pdf.PDFUtils;
 import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel;
 import eu.eudat.models.data.datasetwizard.DatasetsToBeFinalized;
 import eu.eudat.models.data.dmp.DataManagementPlan;
@@ -69,17 +70,15 @@ public class DMPs extends BaseController {
     private DynamicGrantConfiguration dynamicGrantConfiguration;
     private Environment environment;
     private DataManagementPlanManager dataManagementPlanManager;
-    private DatasetManager datasetManager;
     private ConfigLoader configLoader;
 
     @Autowired
     public DMPs(ApiContext apiContext, DynamicGrantConfiguration dynamicGrantConfiguration, Environment environment,
-                DataManagementPlanManager dataManagementPlanManager, DatasetManager datasetManager, ConfigLoader configLoader) {
+                DataManagementPlanManager dataManagementPlanManager, ConfigLoader configLoader) {
         super(apiContext);
         this.dynamicGrantConfiguration = dynamicGrantConfiguration;
         this.environment = environment;
         this.dataManagementPlanManager = dataManagementPlanManager;
-        this.datasetManager = datasetManager;
         this.configLoader = configLoader;
     }
 
@@ -175,7 +174,7 @@ public class DMPs extends BaseController {
     public @ResponseBody
     ResponseEntity getRDAJsonDocument(@PathVariable String id, @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) {
     	try {
-        return this.dataManagementPlanManager.getRDAJsonDocument(id, datasetManager, principal);
+        return this.dataManagementPlanManager.getRDAJsonDocument(id, principal);
     	} catch (Exception e) {
     		return ResponseEntity.status(HttpStatus.EXPECTATION_FAILED).body(new ResponseItem<>().message(e.getMessage()).status(ApiMessageCode.ERROR_MESSAGE));
     	}
@@ -186,8 +185,8 @@ public class DMPs extends BaseController {
     ResponseEntity<byte[]> getPDFDocument(@PathVariable String id, @RequestHeader("Content-Type") String contentType,
                                           @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) throws IllegalAccessException, IOException, InstantiationException, InterruptedException {
         FileEnvelope file = this.dataManagementPlanManager.getWordDocument(id, principal, configLoader);
-        String name = file.getFilename().substring(0, file.getFilename().length() - 5);
-        File pdffile = datasetManager.convertToPDF(file, environment);
+        String name = file.getFilename().substring(0, file.getFilename().length() - 5).replace(" ", "_").replace(",", "_");
+        File pdffile = PDFUtils.convertToPDF(file, environment);
         InputStream resource = new FileInputStream(pdffile);
         logger.info("Mime Type of " + name + " is " +
                 new MimetypesFileTypeMap().getContentType(file.getFile()));

dmp-backend/web/src/main/java/eu/eudat/controllers/DashBoardController.java

@@ -54,6 +54,7 @@ public class DashBoardController extends BaseController {
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<RecentActivityModel>>().status(ApiMessageCode.NO_MESSAGE).payload(statistics));
     }
 
+    @Deprecated
     @RequestMapping(method = RequestMethod.GET, value = {"/user/recentActivity"}, produces = "application/json")
     public ResponseEntity<ResponseItem<RecentActivity>> getRecentActivity(@RequestParam(name = "numOfActivities", required = false, defaultValue = "5") Integer numberOfActivities, Principal principal) {
         RecentActivity statistics = dashBoardManager.getRecentActivity(principal, numberOfActivities);

dmp-backend/web/src/main/java/eu/eudat/controllers/Datasets.java

@@ -16,6 +16,7 @@ import eu.eudat.logic.security.claims.ClaimedAuthorities;
 import eu.eudat.logic.services.ApiContext;
 import eu.eudat.logic.services.forms.VisibilityRuleService;
 import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
+import eu.eudat.logic.utilities.documents.pdf.PDFUtils;
 import eu.eudat.models.data.dataset.DatasetOverviewModel;
 import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel;
 import eu.eudat.models.data.datasetwizard.DataManagentPlanListingModel;
@@ -38,11 +39,11 @@ import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
-import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.persistence.NoResultException;
+import javax.transaction.Transactional;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -120,7 +121,7 @@ public class Datasets extends BaseController {
 //        }
     }
 
-    @javax.transaction.Transactional
+    @Transactional
     @RequestMapping(method = RequestMethod.GET, value = {"{id}"}, produces = "application/json")
     public @ResponseBody
     ResponseEntity getSingle(@PathVariable String id, @RequestHeader("Content-Type") String contentType, @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) throws IllegalAccessException, IOException, InstantiationException {
@@ -134,7 +135,8 @@ public class Datasets extends BaseController {
                 HttpHeaders responseHeaders = new HttpHeaders();
                 responseHeaders.setContentLength(file.getFile().length());
                 responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
-                responseHeaders.set("Content-Disposition", "attachment;filename=" + file.getFilename());
+                String fileName = file.getFilename().replace(" ", "_").replace(",", "_");
+                responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName);
                 responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
                 responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
 
@@ -177,7 +179,7 @@ public class Datasets extends BaseController {
     @RequestMapping(method = RequestMethod.POST, value = {"/getAvailableProfiles"}, produces = "application/json")
     public @ResponseBody
     ResponseEntity<ResponseItem<List<AssociatedProfile>>> getAvailableProfiles(@RequestBody DatasetProfileWizardAutocompleteRequest datasetProfileWizardAutocompleteRequest, @ClaimedAuthorities(claims = {ANONYMOUS}) Principal principal) throws IllegalAccessException, InstantiationException {
-        List<AssociatedProfile> dataManagementPlans = DatasetWizardManager.getAvailableProfiles(this.getApiContext().getOperationsContext().getDatabaseRepository().getDmpDao(), datasetProfileWizardAutocompleteRequest);
+        List<AssociatedProfile> dataManagementPlans = DatasetWizardManager.getAvailableProfiles(this.getApiContext().getOperationsContext().getDatabaseRepository().getDmpDao(), this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao(), datasetProfileWizardAutocompleteRequest);
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<AssociatedProfile>>().status(ApiMessageCode.NO_MESSAGE).payload(dataManagementPlans));
     }
 
@@ -217,11 +219,11 @@ public class Datasets extends BaseController {
     public @ResponseBody
     ResponseEntity<byte[]> getPDFDocument(@PathVariable String id, @ClaimedAuthorities(claims = {Authorities.ADMIN, Authorities.MANAGER, Authorities.USER, Authorities.ANONYMOUS}) Principal principal) throws IllegalAccessException, IOException, InstantiationException, InterruptedException {
         FileEnvelope file = datasetManager.getWordDocumentFile(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal);
-        String fileName = file.getFilename();
+        String fileName = file.getFilename().replace(" ", "_").replace(",", "_");
         if (fileName.endsWith(".docx")){
             fileName = fileName.substring(0, fileName.length() - 5);
         }
-        File pdffile = datasetManager.convertToPDF(file, environment);
+        File pdffile = PDFUtils.convertToPDF(file, environment);
         InputStream resource = new FileInputStream(pdffile);
 
         HttpHeaders responseHeaders = new HttpHeaders();
@@ -244,7 +246,7 @@ public class Datasets extends BaseController {
     * Data Management
     * */
 
-    @javax.transaction.Transactional
+    @Transactional
     @RequestMapping(method = RequestMethod.POST, consumes = "application/json", produces = "application/json")
     public @ResponseBody
     ResponseEntity<ResponseItem<DatasetWizardModel>> createOrUpdate(@RequestBody DatasetWizardModel profile, Principal principal) throws Exception {
@@ -260,7 +262,7 @@ public class Datasets extends BaseController {
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Dataset>().status(ApiMessageCode.SUCCESS_MESSAGE).message(this.getApiContext().getHelpersService().getMessageSource().getMessage("dataset.public", new Object[]{}, locale)));
     }
 
-    @javax.transaction.Transactional
+    @Transactional
     @RequestMapping(method = RequestMethod.DELETE, value = {"/delete/{id}"}, produces = "application/json")
     public @ResponseBody
     ResponseEntity<ResponseItem<Dataset>> delete(@PathVariable(value = "id") UUID id, Principal principal) throws Exception {
@@ -268,7 +270,7 @@ public class Datasets extends BaseController {
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Dataset>().status(ApiMessageCode.SUCCESS_MESSAGE).message("Deleted"));
     }
 
-    @javax.transaction.Transactional
+    @Transactional
     @RequestMapping(method = RequestMethod.GET, value = {"/{id}/unlock"}, produces = "application/json")
     public @ResponseBody
     ResponseEntity<ResponseItem<Dataset>> unlock(@PathVariable(value = "id") UUID id, Principal principal) throws Exception {
@@ -316,7 +318,7 @@ public class Datasets extends BaseController {
     * Data Index
     * */
 
-    @javax.transaction.Transactional
+    @Transactional
     @RequestMapping(method = RequestMethod.POST, value = {"/index"})
     public @ResponseBody
     ResponseEntity<ResponseItem<Dataset>> generateIndex(Principal principal) throws Exception {
@@ -324,7 +326,7 @@ public class Datasets extends BaseController {
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<eu.eudat.data.entities.Dataset>().status(ApiMessageCode.SUCCESS_MESSAGE).message("Generated").payload(null));
     }
 
-    @javax.transaction.Transactional
+    @Transactional
     @RequestMapping(method = RequestMethod.DELETE, value = {"/index"})
     public @ResponseBody
     ResponseEntity<ResponseItem<Dataset>> clearIndex(Principal principal) throws Exception {

dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java

@@ -35,6 +35,7 @@ import eu.eudat.models.data.login.LoginInfo;
 import eu.eudat.models.data.principal.PrincipalModel;
 import eu.eudat.models.data.security.Principal;
 import eu.eudat.types.ApiMessageCode;
+import eu.eudat.types.MetricNames;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -64,6 +65,7 @@ public class Login {
     private ConfigurableProviderTokenValidator configurableProviderTokenValidator;
     private ZenodoTokenValidator zenodoTokenValidator;
     private ConfigLoader configLoader;
+    private final MetricsManager metricsManager;
 
 //    private Logger logger;
 
@@ -75,7 +77,8 @@ public class Login {
                  B2AccessTokenValidator b2AccessTokenValidator, ORCIDTokenValidator orcidTokenValidator,
                  LinkedInTokenValidator linkedInTokenValidator, OpenAIRETokenValidator openAIRETokenValidator,
                  ConfigurableProviderTokenValidator configurableProviderTokenValidator, ZenodoTokenValidator zenodoTokenValidator,
-                 ConfigLoader configLoader, UserManager userManager) {
+                 ConfigLoader configLoader, UserManager userManager,
+                 MetricsManager metricsManager) {
         this.customAuthenticationProvider = customAuthenticationProvider;
         this.nonVerifiedUserAuthenticationService = nonVerifiedUserAuthenticationService;
         this.twitterTokenValidator = twitterTokenValidator;
@@ -87,6 +90,7 @@ public class Login {
         this.zenodoTokenValidator = zenodoTokenValidator;
         this.configLoader = configLoader;
         this.userManager = userManager;
+        this.metricsManager = metricsManager;
     }
 
 
@@ -95,7 +99,7 @@ public class Login {
     public @ResponseBody
     ResponseEntity<ResponseItem<PrincipalModel>> externallogin(@RequestBody LoginInfo credentials) throws GeneralSecurityException, NullEmailException {
         logger.info("Trying To Login With " + credentials.getProvider());
-        MetricsManager.increaseValue("argos_users", 1, "loggedin");
+        metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<PrincipalModel>().payload(customAuthenticationProvider.authenticate(credentials)).status(ApiMessageCode.SUCCESS_MESSAGE));
     }
 
@@ -104,7 +108,7 @@ public class Login {
     public @ResponseBody
     ResponseEntity<ResponseItem<PrincipalModel>> nativelogin(@RequestBody Credentials credentials) throws NullEmailException {
         logger.info(credentials.getUsername() + " Trying To Login");
-        MetricsManager.increaseValue("argos_users", 1, "loggedin");
+        metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<PrincipalModel>().payload(userManager.authenticate(this.nonVerifiedUserAuthenticationService, credentials)).status(ApiMessageCode.SUCCESS_MESSAGE));
     }
 
@@ -165,7 +169,7 @@ public class Login {
     ResponseEntity<ResponseItem<Principal>> logout(Principal principal) {
         this.nonVerifiedUserAuthenticationService.Logout(principal.getToken());
         logger.info(principal + " Logged Out");
-        MetricsManager.decreaseValue("argos_users", 1, "loggedin");
+        metricsManager.decreaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Principal>().status(ApiMessageCode.NO_MESSAGE));
     }
 

dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java

@@ -1,10 +1,8 @@
 package eu.eudat.controllers;
 
 import eu.eudat.elastic.criteria.DatasetCriteria;
-import eu.eudat.elastic.criteria.TagCriteria;
 import eu.eudat.elastic.entities.Dataset;
 import eu.eudat.elastic.entities.Tag;
-import eu.eudat.elastic.repository.Repository;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.services.ApiContext;
@@ -49,7 +47,9 @@ public class TagController extends BaseController {
         /*List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type);
         TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);*/
         if (this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().exists()) {
-            List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
+            DatasetCriteria criteria = new DatasetCriteria();
+            criteria.setHasTags(true);
+            List<Tag> tags = this.getApiContext().getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tag.getName().toLowerCase().startsWith(query.toLowerCase())).collect(Collectors.toList());
 
             return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<Tag>>().payload(tags).status(ApiMessageCode.NO_MESSAGE));
         } else {

dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java

@@ -6,6 +6,7 @@ import eu.eudat.models.data.helpers.responses.ResponseItem;
 import eu.eudat.models.data.security.Principal;
 import eu.eudat.models.data.userguide.UserGuide;
 import eu.eudat.types.ApiMessageCode;
+import eu.eudat.types.MetricNames;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.env.Environment;
 import org.springframework.http.HttpHeaders;
@@ -30,16 +31,18 @@ import static eu.eudat.types.Authorities.ADMIN;
 public class UserGuideController {
 
 	private Environment environment;
+	private final MetricsManager metricsManager;
 
 	@Autowired
-	public UserGuideController(Environment environment) {
+	public UserGuideController(Environment environment, MetricsManager metricsManager) {
 		this.environment = environment;
+		this.metricsManager = metricsManager;
 	}
 
 	@RequestMapping(path = "{lang}", method = RequestMethod.GET )
 	public ResponseEntity getUserGuide(@PathVariable(name = "lang") String lang) throws IOException {
 		long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count();
-		MetricsManager.calculateValue("argos_languages", (int) files, null);
+		metricsManager.calculateValue(MetricNames.LANGUAGES, (int) files, null);
 		Stream<Path> walk = Files.walk(Paths.get(this.environment.getProperty("userguide.path")));
 		List<String> result = walk.filter(Files::isRegularFile)
 								  .map(Path::toString).collect(Collectors.toList());
@@ -50,13 +53,12 @@ public class UserGuideController {
 		}
 		InputStream is = new FileInputStream(fileName);
 
-		String[] filepath = fileName.split("\\.")[0].split("\\\\");
-		String simplename = filepath[filepath.length - 1];
+		Path path = Paths.get(fileName);
 
 		HttpHeaders responseHeaders = new HttpHeaders();
 		responseHeaders.setContentLength(is.available());
 		responseHeaders.setContentType(MediaType.TEXT_HTML);
-		responseHeaders.set("Content-Disposition", "attachment;filename=" + simplename);
+		responseHeaders.set("Content-Disposition", "attachment;filename=" + path.getFileName().toString());
 		responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
 		responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
 
@@ -71,7 +73,7 @@ public class UserGuideController {
 	@RequestMapping(value = "current", method = RequestMethod.POST)
 	public @ResponseBody
 	ResponseEntity<ResponseItem<String>> updateGuide(@RequestBody UserGuide guide, @ClaimedAuthorities(claims = {ADMIN}) Principal principal) throws Exception {
-		String fileName =  this.environment.getProperty("userguide.path") + guide.getName() + ".html";
+		String fileName =  this.environment.getProperty("userguide.path") + guide.getName();
 		OutputStream os = new FileOutputStream(fileName);
 		os.write(guide.getHtml().getBytes());
 		os.close();

dmp-backend/web/src/main/java/eu/eudat/controllers/UserInvitationController.java

@@ -15,6 +15,7 @@ import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.bind.annotation.*;
 
 import javax.xml.bind.JAXBException;
+import java.io.IOException;
 import java.util.List;
 import java.util.UUID;
 
@@ -42,15 +43,18 @@ public class UserInvitationController extends BaseController {
     @Transactional
     @RequestMapping(method = RequestMethod.GET, value = {"/exchange/{invitationID}"}, produces = "application/json")
     public @ResponseBody
-    ResponseEntity<ResponseItem<UUID>> exchange(@PathVariable UUID invitationID, Principal principal) throws JAXBException {
+    ResponseEntity<ResponseItem<UUID>> exchange(@PathVariable UUID invitationID, Principal principal) throws JAXBException, IOException {
         UUID dmpId = invitationsManager.assignUserAcceptedInvitation(invitationID, principal);
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<UUID>().status(ApiMessageCode.SUCCESS_MESSAGE).payload(dmpId));
     }
 
     @RequestMapping(method = RequestMethod.POST, value = {"/getUsers"}, consumes = "application/json", produces = "application/json")
     public @ResponseBody
-    ResponseEntity<ResponseItem<List<UserInfoInvitationModel>>> getUsers(Principal principal) throws IllegalAccessException, InstantiationException {
-        List<UserInfoInvitationModel> users = invitationsManager.getUsers(principal);
+//    ResponseEntity<ResponseItem<List<UserInfoInvitationModel>>> getUsers(Principal principal) throws IllegalAccessException, InstantiationException {
+    ResponseEntity<ResponseItem<List<UserInfoInvitationModel>>> getUsers(Principal principal, @RequestBody UserInfoRequestItem userInfoRequestItem) throws IllegalAccessException, InstantiationException {
+        System.out.println(userInfoRequestItem.getCriteria().getLike());
+//        List<UserInfoInvitationModel> users = invitationsManager.getUsers(principal);
+        List<UserInfoInvitationModel> users = invitationsManager.getUsersWithCriteria(principal, userInfoRequestItem);
         return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<UserInfoInvitationModel>>().status(ApiMessageCode.SUCCESS_MESSAGE).payload(users));
     }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DashBoardManager.java

@@ -8,13 +8,13 @@ import eu.eudat.data.dao.entities.DMPDao;
 import eu.eudat.data.dao.entities.DatasetDao;
 import eu.eudat.data.dao.entities.GrantDao;
 import eu.eudat.data.dao.entities.OrganisationDao;
-import eu.eudat.data.entities.DMP;
-import eu.eudat.data.entities.Dataset;
-import eu.eudat.data.entities.Grant;
-import eu.eudat.data.entities.UserInfo;
+import eu.eudat.data.entities.*;
 import eu.eudat.data.query.PaginationService;
+import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
+import eu.eudat.elastic.criteria.SortCriteria;
 import eu.eudat.elastic.entities.Dmp;
 import eu.eudat.logic.builders.model.models.RecentActivityDataBuilder;
+import eu.eudat.logic.mapper.elastic.criteria.DmpCriteriaMapper;
 import eu.eudat.logic.services.ApiContext;
 import eu.eudat.logic.services.operations.DatabaseRepository;
 import eu.eudat.models.HintedModelFactory;
@@ -26,6 +26,8 @@ import eu.eudat.models.data.dashboard.recent.model.RecentDmpModel;
 import eu.eudat.models.data.dashboard.recent.tablerequest.RecentActivityTableRequest;
 import eu.eudat.models.data.dashboard.searchbar.SearchBarItem;
 import eu.eudat.models.data.dashboard.statistics.DashBoardStatistics;
+import eu.eudat.models.data.listingmodels.DataManagementPlanListingModel;
+import eu.eudat.models.data.listingmodels.DatasetListingModel;
 import eu.eudat.models.data.security.Principal;
 import eu.eudat.queryable.QueryableList;
 import eu.eudat.types.searchbar.SearchBarItemType;
@@ -36,9 +38,11 @@ import org.springframework.stereotype.Component;
 
 import javax.transaction.Transactional;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 @Component
@@ -97,8 +101,8 @@ public class DashBoardManager {
     }
 
     public DashBoardStatistics getMeStatistics(Principal principal) throws IOException {
-        List<eu.eudat.elastic.entities.Dataset> datasets = null;
-        List<eu.eudat.elastic.entities.Dmp> dmps = null;
+        Long datasets = 0L;
+        Long dmps = 0L;
         DashBoardStatistics statistics = new DashBoardStatistics();
         DMPDao dataManagementPlanRepository = databaseRepository.getDmpDao();
         DatasetDao datasetRepository = databaseRepository.getDatasetDao();
@@ -112,7 +116,8 @@ public class DashBoardManager {
                 eu.eudat.elastic.criteria.DatasetCriteria datasetElasticCriteria = new eu.eudat.elastic.criteria.DatasetCriteria();
                 datasetElasticCriteria.setAllowAllVersions(false);
                 datasetElasticCriteria.setPublic(false);
-                datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
+                datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
+                datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().count(datasetElasticCriteria);
             }catch (Exception e) {
                 logger.warn(e.getMessage(), e);
                 datasets = null;
@@ -126,33 +131,44 @@ public class DashBoardManager {
                 eu.eudat.elastic.criteria.DmpCriteria dmpElasticCriteria = new eu.eudat.elastic.criteria.DmpCriteria();
                 dmpElasticCriteria.setAllowAllVersions(false);
                 dmpElasticCriteria.setPublic(false);
-                dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
+                dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
+                dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().count(dmpElasticCriteria);
             }catch (Exception e) {
                 logger.warn(e.getMessage(), e);
                 dmps = null;
             }
         }
         dataManagementPlanCriteria.setAllVersions(false);
-        GrantCriteria grantCriteria = new GrantCriteria();
+        dataManagementPlanCriteria.setOnlyPublic(false);
+        dataManagementPlanCriteria.setIsPublic(false);
 
+        GrantCriteria grantCriteria = new GrantCriteria();
         grantCriteria.setActive(true);
 
+        OrganisationCriteria organisationCriteria = new OrganisationCriteria();
+        organisationCriteria.setActive(true);
+
         List<Integer> roles = new LinkedList<>();
-        List<Dmp> finalDmps = dmps;
-        CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated((dmps != null && !dmps.isEmpty()) ? dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))) : dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
+        if ((dmps == null || dmps == 0L) && (datasets == null || datasets == 0L)) {
+            CompletableFuture dmpFuture = dataManagementPlanRepository.getAuthenticated(dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria), principal.getId(), roles).distinct().countAsync()
                     .whenComplete((dmpsStats, throwable) -> statistics.setTotalDataManagementPlanCount(dmpsStats));
-        List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
-        CompletableFuture datasetFuture = datasetRepository.getAuthenticated((datasets != null && !datasets.isEmpty()) ? datasetRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))) : datasetRepository.getWithCriteria(datasetCriteria), user, roles).countAsync()
+            CompletableFuture datasetFuture = datasetRepository.getAuthenticated( datasetRepository.getWithCriteria(datasetCriteria), user, roles).distinct().countAsync()
                     .whenComplete((datasetsStats, throwable) -> statistics.setTotalDataSetCount(datasetsStats));
+            CompletableFuture.allOf(dmpFuture, datasetFuture).join();
+        } else {
+            statistics.setTotalDataManagementPlanCount(dmps);
+            statistics.setTotalDataSetCount(datasets);
+        }
         CompletableFuture grantFuture = grantRepository.getAuthenticated(grantRepository.getWithCriteria(grantCriteria), user).countAsync()
                 .whenComplete((grantsStats, throwable) -> statistics.setTotalGrantCount(grantsStats));
-        CompletableFuture orgnanisationFuture = organisationRepository.getAuthenticated(organisationRepository.asQueryable().withHint("organisationRecentActivity"), user).countAsync()
+        CompletableFuture orgnanisationFuture = organisationRepository.getAuthenticated(organisationRepository.getWithCriteria(organisationCriteria).withHint("organisationRecentActivity"), user).countAsync()
                 .whenComplete((organisationStats, throwable) -> statistics.setTotalOrganisationCount(organisationStats));
 
-        CompletableFuture.allOf(dmpFuture, datasetFuture, grantFuture, orgnanisationFuture).join();
+        CompletableFuture.allOf( grantFuture, orgnanisationFuture).join();
         return statistics;
     }
 
+    @Deprecated
     public RecentActivity getRecentActivity(Principal principal, Integer numberofactivities) {
         RecentActivity activity = new RecentActivity();
         DMPDao dataManagementPlanRepository = databaseRepository.getDmpDao();
@@ -175,7 +191,7 @@ public class DashBoardManager {
                 .selectAsync(item -> recentActivityDataBuilder.label(item.getLabel()).timestamp(item.getModified()).id(item.getId().toString()).build())
                 .whenComplete((dmpActivities, throwable) -> activity.setRecentDmpActivities(dmpActivities));
 
-        CompletableFuture<List<RecentActivityData>> datasets = datasetRepository.getAuthenticated(datasetRepository.getWithCriteria(datasetCriteria), user, roles)
+        CompletableFuture<List<RecentActivityData>> datasets = datasetRepository.getAuthenticated(datasetRepository.getWithCriteria(datasetCriteria), user, roles).distinct()
                 .withHint("datasetRecentActivity")
                 .orderBy((builder, root) -> builder.desc(root.get("modified")))
                 .take(numberofactivities)
@@ -225,7 +241,17 @@ public class DashBoardManager {
                 datasetElasticCriteria.setLike(tableRequest.getCriteria().getLike());
                 datasetElasticCriteria.setAllowAllVersions(false);
                 datasetElasticCriteria.setPublic(!isAuthenticated);
-                datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(datasetElasticCriteria);
+                datasetElasticCriteria.setOffset(tableRequest.getDatasetOffset());
+                datasetElasticCriteria.setSize(tableRequest.getLength());
+                if (isAuthenticated) {
+                    datasetElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
+                }
+                datasetElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
+                datasetElasticCriteria.getSortCriteria().stream().filter(sortCriteria -> sortCriteria.getFieldName().equals("publishedAt")).forEach(sortCriteria -> {
+                    sortCriteria.setFieldName("dmp:" + sortCriteria.getFieldName());
+                    sortCriteria.setColumnType(SortCriteria.ColumnType.JOIN_COLUMN);
+                });
+                datasets = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().queryIds(datasetElasticCriteria);
             }catch (Exception e) {
                 logger.warn(e.getMessage(), e);
                 datasets = null;
@@ -238,10 +264,16 @@ public class DashBoardManager {
                 dmpElasticCriteria.setLike(tableRequest.getCriteria().getLike());
                 dmpElasticCriteria.setAllowAllVersions(false);
                 dmpElasticCriteria.setPublic(!isAuthenticated);
+                dmpElasticCriteria.setOffset(tableRequest.getDmpOffset());
+                dmpElasticCriteria.setSize(tableRequest.getLength());
+                if (isAuthenticated) {
+                    dmpElasticCriteria.setCollaborators(Collections.singletonList(principal.getId()));
+                }
+                dmpElasticCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(tableRequest.getOrderings()));
                 dmps = apiContext.getOperationsContext().getElasticRepository().getDmpRepository().query(dmpElasticCriteria);
             }catch (Exception e) {
                 logger.warn(e.getMessage(), e);
-                datasets = null;
+                dmps = null;
             }
         }
 
@@ -250,6 +282,8 @@ public class DashBoardManager {
             dmpList = dataManagementPlanRepository.asQueryable().where((builder, root) -> root.get("id").in(finalDmps.stream().map(Dmp::getId).collect(Collectors.toList()))).distinct();
         } else {
             dmpList = dataManagementPlanRepository.getWithCriteria(dataManagementPlanCriteria).distinct();
+            PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
+            dmpList.skip(tableRequest.getDmpOffset()).take(tableRequest.getLength());
         }
 
         if (datasets != null  && !datasets.isEmpty()) {
@@ -258,53 +292,57 @@ public class DashBoardManager {
         } else {
             datasetList = datasetRepository.getWithCriteria(datasetCriteria);
         }
+        IntStream.range(0, tableRequest.getOrderings().getFields().size()).filter(i -> tableRequest.getOrderings().getFields().get(i).contains("publishedAt"))
+                .forEach(i -> tableRequest.getOrderings().getFields().set(i, tableRequest.getOrderings().getFields().get(i).toCharArray()[0] + "dmp:publishedAt|join|"));
+       // tableRequest.getOrderings().getFields().stream().filter(s -> s.contains("publishedAt")).forEach(s -> s = s.toCharArray()[0] + "dmp:publishedAt|join|" );
+        /*for (int i = 0; i< tableRequest.getOrderings().getFields().size(); i++) {
+            if (tableRequest.getOrderings().getFields().get(i).contains("publishedAt")) {
+                String newField = tableRequest.getOrderings().getFields().get(i).toCharArray()[0] + "dmp:publishedAt|join|";
+                tableRequest.getOrderings().getFields().set(i, newField);
+            }
+        }*/
+        /*if (tableRequest.getOrderings().getFields().get(0).contains("publishedAt")) {
+            tableRequest.getOrderings().getFields().set(0, tableRequest.getOrderings().getFields().get(0).charAt(0) + "dmp:" + tableRequest.getOrderings().getFields().get(0).substring(1) + "|join|");
+        }*/
+        PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
+        datasetList.skip(tableRequest.getDatasetOffset()).take(tableRequest.getLength());
 
         if (isAuthenticated) {
 
             List<Integer> roles = new LinkedList<>();
+            roles.add(UserDMP.UserDMPRoles.USER.getValue());
+            roles.add(UserDMP.UserDMPRoles.OWNER.getValue());
             dmpList = dataManagementPlanRepository.getAuthenticated(dmpList, principal.getId(), roles);
-            datasetList = datasetRepository.getAuthenticated(datasetList, user, roles);
+            datasetList = datasetRepository.getAuthenticated(datasetList, user, roles).distinct();
 
         }
 
-        PaginationService.applyOrder(dmpList, tableRequest.getOrderings());
-        for (int i = 0; i< tableRequest.getOrderings().getFields().length; i++) {
-            if (tableRequest.getOrderings().getFields()[i].contains("publishedAt")) {
-                String newField = tableRequest.getOrderings().getFields()[i].toCharArray()[0] + "dmp:publishedAt|join|";
-                tableRequest.getOrderings().getFields()[i] = newField;
-            }
-        }
-        PaginationService.applyOrder(datasetList, tableRequest.getOrderings());
+
 
         /*CompletableFuture future = CompletableFuture.runAsync(() -> */{
-            recentActivityModels.addAll(dmpList
-                    .withHint(HintedModelFactory.getHint(RecentDmpModel.class))
-                   // .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
-                    .skip(tableRequest.getDmpOffset())
-                    .take(tableRequest.getLength())
-                    .select(item -> {
-                        item.setDataset(
-                                item.getDataset().stream()
-                                    .filter(dataset -> !dataset.getStatus().equals(Dataset.Status.DELETED.getValue()) && !dataset.getStatus().equals(Dataset.Status.CANCELED.getValue())).collect(Collectors.toList()).stream()
-                                    .filter(dataset ->  dataset.getDmp().getUsers().stream()
-                                                               .filter(x -> x.getUser().getId().equals(principal.getId()))
-                                                               .collect(Collectors.toList()).size() > 0)
-                                    .collect(Collectors.toSet()));
-                        return new RecentDmpModel().fromEntity(item);
-                    }));
+            List<DMP> dmps1 = dmpList.withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))
+                    .distinct().toList();
+            recentActivityModels.addAll(dmps1.stream().map(dmp -> {
+                DatasetCriteria datasetCriteria1 = new DatasetCriteria();
+                datasetCriteria1.setDmpIds(Collections.singletonList(dmp.getId()));
+                datasetCriteria1.setAllVersions(false);
+                if (isAuthenticated) {
+                    dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1, principal.getId()));
+                } else {
+                    datasetCriteria1.setIsPublic(true);
+                    dmp.setDataset(retrieveRelevantDatasets(datasetCriteria1));
+                }
+                return new RecentDmpModel().fromDataModel(dmp);
+            }).collect(Collectors.toList()));
 
-            recentActivityModels.addAll(datasetList
-                    .withHint(HintedModelFactory.getHint(RecentDatasetModel.class))
-                  //  .orderBy((builder, root) -> builder.desc(root.get(tableRequest.getCriteria().getOrder())))
-                    .skip(tableRequest.getDatasetOffset())
-                    .take(tableRequest.getLength())
-                    .select(item -> {
-                        return new RecentDatasetModel().fromEntity(item);
-                    }));
+            List<RecentActivityModel> recentDatasetModels = datasetList
+                    .withHint(HintedModelFactory.getHint(DatasetListingModel.class))
+                    .select(item -> new RecentDatasetModel().fromEntity(item));
+            recentActivityModels.addAll(recentDatasetModels);
         }/*);*/
 
         //GK: Shuffle the deck otherwise we will summon the DMPodia when sorting with status
-        int pos = -1;
+        /*int pos = -1;
         for (int i = (recentActivityModels.size() / 2); i < recentActivityModels.size(); i++) {
             RecentActivityModel recentActivityModel = recentActivityModels.remove(i);
             while (pos < recentActivityModels.size()) {
@@ -314,12 +352,29 @@ public class DashBoardManager {
                 }
             }
             recentActivityModels.add(pos, recentActivityModel);
+        }*/
+        //GK: No one likes to play shuffle with the recent activities. So just re-sort them based on how they have been sorted already
+
+        recentActivityModels = recentActivityModels.stream().sorted((o1, o2) -> {
+            try {
+                String order = tableRequest.getOrderings().getFields().get(0).toCharArray()[0] + "";
+                String field = tableRequest.getOrderings().getFields().get(0).substring(1);
+                if (field.contains(":") && field.contains("|")) {
+                    field = field.substring(field.lastIndexOf(":") + 1, field.indexOf("|"));
                 }
+                field = field.equals("label") ? "title" : field;
+                field = field.substring(0, 1).toUpperCase() + field.substring(1);
+                return (order.equals("+") ? 1 : -1 ) * ((Comparable)o1.getClass().getMethod("get" + field).invoke(o1)).compareTo(o2.getClass().getMethod("get" + field).invoke(o2));
+                } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+                    logger.error(e.getLocalizedMessage(), e);
+                }
+            return 0;
+        }).collect(Collectors.toList());
 
         //CompletableFuture.allOf(future).join();
        // CompletableFuture.allOf(dmps, datasets).join();
 
-        return recentActivityModels.stream().sorted(this.comparators.get(tableRequest.getCriteria().getOrder())).limit(tableRequest.getLength()).collect(Collectors.toList());
+        return recentActivityModels;
     }
 
     public List<SearchBarItem> searchUserData(String like, Principal principal) {
@@ -380,4 +435,38 @@ public class DashBoardManager {
 
         return searchBarItems;
     }
+
+    private Set<Dataset> retrieveRelevantDatasets(DatasetCriteria datasetCriteria) {
+        return retrieveRelevantDatasets(datasetCriteria, null);
+    }
+
+    private Set<Dataset> retrieveRelevantDatasets (DatasetCriteria datasetCriteria, UUID principal) {
+        QueryableList<Dataset> datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria);
+        if (principal != null) {
+            UserInfo userInfo = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal);
+            List<Integer> roles = new ArrayList<>();
+            roles.add(0);
+            roles.add(1);
+            datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getAuthenticated(datasetItems, userInfo, roles);
+        }
+        Long maxDatasets = datasetItems.distinct().count();
+        DatasetTableRequest datasetTableRequest = new DatasetTableRequest();
+        datasetTableRequest.setOffset(0);
+        datasetTableRequest.setLength(3);
+        Set<Dataset> datasetsSet = new LinkedHashSet<>();
+        try {
+            datasetItems = PaginationManager.applyPaging(datasetItems, datasetTableRequest);
+            List<Dataset> datasets = datasetItems.distinct().toList();
+            datasetsSet.addAll(datasets);
+            for (int i = 0; i < maxDatasets - datasets.size(); i++) {
+                Dataset fakedataset = new Dataset();
+                fakedataset.setId(UUID.randomUUID());
+                datasetsSet.add(fakedataset);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+
+        return datasetsSet;
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java

@@ -37,6 +37,7 @@ import eu.eudat.logic.services.operations.DatabaseRepository;
 import eu.eudat.logic.services.utilities.UtilitiesService;
 import eu.eudat.logic.utilities.builders.XmlBuilder;
 import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
+import eu.eudat.logic.utilities.documents.pdf.PDFUtils;
 import eu.eudat.logic.utilities.documents.types.ParagraphStyle;
 import eu.eudat.logic.utilities.documents.word.WordBuilder;
 import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder;
@@ -59,6 +60,7 @@ import eu.eudat.models.data.user.composite.PagedDatasetProfile;
 import eu.eudat.models.data.userinfo.UserListingModel;
 import eu.eudat.queryable.QueryableList;
 import eu.eudat.types.Authorities;
+import eu.eudat.types.MetricNames;
 import org.apache.poi.xwpf.usermodel.XWPFDocument;
 import org.apache.poi.xwpf.usermodel.XWPFParagraph;
 import org.apache.poi.xwpf.usermodel.XWPFRun;
@@ -77,6 +79,7 @@ import org.springframework.web.multipart.MultipartFile;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 
+import javax.transaction.Transactional;
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Unmarshaller;
@@ -108,9 +111,10 @@ public class DataManagementPlanManager {
     private Environment environment;
     private RDAManager rdaManager;
     private UserManager userManager;
+    private final MetricsManager metricsManager;
 
     @Autowired
-    public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager) {
+    public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager, MetricsManager metricsManager) {
         this.apiContext = apiContext;
         this.datasetManager = datasetManager;
         this.utilitiesService = apiContext.getUtilitiesService();
@@ -118,6 +122,7 @@ public class DataManagementPlanManager {
         this.environment = environment;
         this.rdaManager = rdaManager;
         this.userManager = userManager;
+        this.metricsManager = metricsManager;
     }
 
     /*
@@ -149,8 +154,10 @@ public class DataManagementPlanManager {
                 items = null;
             }
         }
+
         if (items == null) {
             items = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dataManagementPlanTableRequest.getCriteria());
+        }
             List<Integer> roles = new LinkedList<>();
             if (!dataManagementPlanTableRequest.getCriteria().isOnlyPublic()) {
                 if (dataManagementPlanTableRequest.getCriteria().getRole() != null)
@@ -159,8 +166,11 @@ public class DataManagementPlanManager {
             } else {
                 authItems = items;
             }
+        if (dmps == null) {
             totalData = authItems.count();
             items = PaginationManager.applyPaging(authItems, dataManagementPlanTableRequest);
+        } else {
+            items = authItems;
         }
 
 
@@ -175,6 +185,8 @@ public class DataManagementPlanManager {
                 dataTable.setData(dmps1.stream().map(dmp -> {
                     DatasetCriteria datasetCriteria = new DatasetCriteria();
                     datasetCriteria.setDmpIds(Collections.singletonList(dmp.getId()));
+                    datasetCriteria.setAllVersions(false);
+                    datasetCriteria.setIsPublic(dataManagementPlanTableRequest.getCriteria().getIsPublic());
                     dmp.setDataset(retrieveRelevantDatasets(datasetCriteria, principalID));
 
 
@@ -240,14 +252,14 @@ public class DataManagementPlanManager {
             roles.add(1);
             datasetItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getAuthenticated(datasetItems, userInfo, roles);
         }
-        Long maxDatasets = datasetItems.count();
+        Long maxDatasets = datasetItems.distinct().count();
         DatasetTableRequest datasetTableRequest = new DatasetTableRequest();
         datasetTableRequest.setOffset(0);
         datasetTableRequest.setLength(3);
         Set<Dataset> datasetsSet = new LinkedHashSet<>();
         try {
             datasetItems = PaginationManager.applyPaging(datasetItems, datasetTableRequest);
-            List<Dataset> datasets = datasetItems.toList();
+            List<Dataset> datasets = datasetItems.distinct().toList();
             datasetsSet.addAll(datasets);
             for (int i = 0; i < maxDatasets - datasets.size(); i++) {
                 Dataset fakedataset = new Dataset();
@@ -441,7 +453,7 @@ public class DataManagementPlanManager {
 
             setNotification = true;
         } else {
-            MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
+            metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
         }
 
         DMP newDmp = dataManagementPlan.toDataModel();
@@ -532,6 +544,9 @@ public class DataManagementPlanManager {
         if (dataManagementPlan.getAssociatedUsers().size() == 0)
             assignUser(newDmp, user);
 
+        UUID dmpId = newDmp.getId();
+        newDmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
+
         this.updateIndex(newDmp);
 
         if (setNotification) {
@@ -565,7 +580,7 @@ public class DataManagementPlanManager {
             if (dataManagementPlan.getStatus() == (int) DMP.DMPStatus.FINALISED.getValue() && dmp1.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()))
                 throw new Exception("DMP is finalized, therefore cannot be edited.");
         } else {
-            MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
+            metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
         }
         List<Dataset> datasets = new ArrayList<>();
         DMP tempDMP = dataManagementPlan.toDataModel();
@@ -586,6 +601,11 @@ public class DataManagementPlanManager {
             datasets.add(dataset);
         }
 
+        UUID dmpId = result.getId();
+        result.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
+
+        this.updateIndex(result);
+
         return result;
     }
 
@@ -623,15 +643,31 @@ public class DataManagementPlanManager {
             assignGrandUserIfInternal(newDmp, user);
             assignFunderUserIfInternal(newDmp, user);
             assignProjectUserIfInternal(newDmp, user);
+            if (newDmp.getGrant().getStartdate() == null) {
+                newDmp.getGrant().setStartdate(new Date());
+            }
+            if (newDmp.getGrant().getEnddate() == null) {
+                newDmp.getGrant().setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS)));
+            }
             databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant());
             DMP tempDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp);
             newDmp.setId(tempDmp.getId());
 
             // Assign creator.
-            assignUser(newDmp, user);
+            //assignUser(newDmp, user);
 
             copyDatasets(newDmp, databaseRepository.getDatasetDao());
 
+           databaseRepository
+                    .getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), oldDmp.getId()))
+                    .toList().stream().forEach(userDMP -> {
+                        UserDMP temp = new UserDMP();
+                        temp.setUser(userDMP.getUser());
+                        temp.setRole(userDMP.getRole());
+                        temp.setDmp(newDmp);
+                        apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().createOrUpdate(temp);
+                    });
+
             newDmp.setUsers(new HashSet<>(databaseRepository.getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), newDmp.getId())).toList()));
 
             DatasetCriteria criteria1 = new DatasetCriteria();
@@ -679,9 +715,11 @@ public class DataManagementPlanManager {
         DatasetCriteria criteria1 = new DatasetCriteria();
         criteria1.setDmpIds(Collections.singletonList(newDmp.getId()));
         newDmp.setDataset(new HashSet<>(databaseRepository.getDatasetDao().getWithCriteria(criteria1).toList()));
+        UUID dmpId = newDmp.getId();
+        newDmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
 
         this.updateIndex(newDmp);
-        MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
+        metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
 
         return newDmp.getId();
     }
@@ -695,26 +733,30 @@ public class DataManagementPlanManager {
         DMP oldDmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(uuid);
         switch (oldDmp.getStatus()) {
             case 0:
-                MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft");
+                metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
                 break;
             case 1:
                 if (oldDmp.getDoi() != null) {
-                    MetricsManager.decreaseValue("argos_managed_dmps", 1, "doied");
+                    metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DOIED);
                 }
                 if (oldDmp.isPublic()) {
-                    MetricsManager.decreaseValue("argos_managed_dmps", 1, "published");
+                    metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
                 }
-                MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized");
+                metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
                 break;
         }
         oldDmp.setStatus(DMP.DMPStatus.DELETED.getValue());
         apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(oldDmp);
+        UUID dmpId = oldDmp.getId();
+        oldDmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
         this.updateIndex(oldDmp);
         DataManagementPlanCriteria criteria1 = new DataManagementPlanCriteria();
         criteria1.setAllVersions(true);
         criteria1.setGroupIds(Collections.singletonList(oldDmp.getGroupId()));
         apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria1).toList().forEach(dmp -> {
             try {
+                UUID tdmpId = dmp.getId();
+                dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), tdmpId)).toList()));
                 this.updateIndex(dmp);
             } catch (IOException e) {
                 logger.error(e.getMessage(), e);
@@ -726,6 +768,7 @@ public class DataManagementPlanManager {
                     if (elastic != null) {
                         tags = elastic.getTags();
                     }
+                    dataset.setDmp(dmp);
                     this.datasetManager.updateTags(dataset, tags);
                 } catch (Exception e) {
                     logger.error(e.getMessage(), e);
@@ -744,7 +787,7 @@ public class DataManagementPlanManager {
                 else {
                     researcher.setCreationUser(user);
                     researcherRepository.createOrUpdate(researcher);
-                    MetricsManager.increaseValue("argos_researchers", 1, null);
+                    metricsManager.increaseValue(MetricNames.RESEARCHER, 1, null);
                 }
             }
         }
@@ -785,6 +828,12 @@ public class DataManagementPlanManager {
                 else {
                     grant.setType(Grant.GrantType.EXTERNAL.getValue());
                     grant.setCreationUser(null);
+                    if (grant.getStartdate() == null) {
+                        grant.setStartdate(new Date());
+                    }
+                    if (grant.getEnddate() == null) {
+                        grant.setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS)));
+                    }
                     grantDao.createOrUpdate(grant);
                 }
             }
@@ -823,19 +872,22 @@ public class DataManagementPlanManager {
                 eu.eudat.data.entities.Project projectEntity = projectDao.getWithCritetia(criteria).toList().stream().max(Comparator.comparing(project1 -> project1.getModified().getTime())).orElse(null);
                 if (projectEntity != null) project.setId(projectEntity.getId());
                 else {
-                    project.setType(Project.ProjectType.EXTERNAL.getValue());
-                    if (project.getId() == null) project.setId(UUID.randomUUID());
-                    projectDao.createOrUpdate(project);
-                    MetricsManager.increaseValue("argos_projects", 1, null);
+                    createExternalProject(project, projectDao);
                 }
             }
             else {
+                createExternalProject(project, projectDao);
+            }
+        }
+    }
+
+    private void createExternalProject(Project project, ProjectDao projectDao) {
+        if (project.getStartdate() == null) project.setStartdate(new Date());
+        if (project.getEnddate() == null) project.setEnddate(new Date());
         project.setType(Project.ProjectType.EXTERNAL.getValue());
         if (project.getId() == null) project.setId(UUID.randomUUID());
         projectDao.createOrUpdate(project);
-                MetricsManager.increaseValue("argos_projects", 1, null);
-            }
-        }
+        metricsManager.increaseValue(MetricNames.PROJECT, 1, null);
     }
 
     private void copyDatasets(DMP newDmp, DatasetDao datasetDao) {
@@ -851,7 +903,8 @@ public class DataManagementPlanManager {
                 if (elastic != null) {
                     tags = elastic.getTags();
                 }
-
+                UUID dmpId = tempDataset.getDmp().getId();
+                tempDataset.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
                 this.datasetManager.updateTags(tempDataset, tags);
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
@@ -910,7 +963,7 @@ public class DataManagementPlanManager {
                           return newDataset;
                       }).thenApplyAsync(item -> {
                 futures.add(datasetDao.createOrUpdateAsync(item).whenComplete(((dataset1, throwable) -> {
-                    MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft");
+                    metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
                     eu.eudat.elastic.entities.Dataset datasetElastic = new eu.eudat.elastic.entities.Dataset();
                     datasetElastic.setId(dataset1.getId().toString());
                     datasetElastic.setLabel(dataset1.getLabel());
@@ -968,8 +1021,10 @@ public class DataManagementPlanManager {
             throw new Exception("DMP is not finalized");
         dmp.setPublic(true);
         apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
+        UUID dmpId = dmp.getId();
+        dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
         this.updateIndex(dmp);
-        MetricsManager.increaseValue("argos_managed_dmps", 1, "published");
+        metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
         DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
         criteria.setGroupIds(Collections.singletonList(dmp.getGroupId()));
         criteria.setAllVersions(true);
@@ -981,8 +1036,10 @@ public class DataManagementPlanManager {
                     if (elastic != null) {
                         tags = elastic.getTags();
                     }
+                    UUID tmdmpId = dataset.getDmp().getId();
+                    dataset.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), tmdmpId)).toList()));
                     this.datasetManager.updateTags(dataset, tags);
-                    MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published");
+                    metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
                 } catch (Exception e) {
                     logger.error(e.getMessage(), e);
                 }
@@ -995,6 +1052,7 @@ public class DataManagementPlanManager {
         }
     }
 
+    @Transactional
     public void makeFinalize(UUID id, Principal principal, DatasetsToBeFinalized datasetsToBeFinalized) throws Exception {
         DMP dmp = this.apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(id);
         if (!isUserOwnerOfDmp(dmp, principal))
@@ -1065,14 +1123,16 @@ public class DataManagementPlanManager {
         }
         dmp.setStatus(DMP.DMPStatus.FINALISED.getValue());
         apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
+        UUID dmpId = dmp.getId();
+        dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
         this.updateIndex(dmp);
         UserInfo user = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId());
         sendNotification(dmp, user, NotificationType.DMP_FINALISED);
-        MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft");
-        MetricsManager.increaseValue("argos_managed_dmps", 1, "finalized");
+        metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
+        metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
         this.updateDatasetsIndex(indexDatasets);
-        MetricsManager.decreaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "draft");
-        MetricsManager.increaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "finalized");
+        metricsManager.decreaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.DRAFT);
+        metricsManager.increaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.FINALIZED);
     }
 
     public void undoFinalize(UUID id, Principal principal) throws Exception {
@@ -1083,9 +1143,11 @@ public class DataManagementPlanManager {
             throw new Exception("DMP is already Active");
         dmp.setStatus(DMP.DMPStatus.ACTIVE.getValue());
         apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
+        UUID dmpId = dmp.getId();
+        dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
         this.updateIndex(dmp);
-        MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized");
-        MetricsManager.increaseValue("argos_managed_dmps", 1, "draft");
+        metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
+        metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
     }
 
     public void updateUsers(UUID id, List<UserInfoListingModel> users, Principal principal) throws Exception {
@@ -1122,7 +1184,7 @@ public class DataManagementPlanManager {
         wordBuilder.addParagraphContent("Data Management Plan Information", document, ParagraphStyle.HEADER1, BigInteger.ZERO);
         // DMP title custom style.
         wordBuilder.addParagraphContent(dmpEntity.getLabel(), document, ParagraphStyle.HEADER2, BigInteger.ZERO);
-        wordBuilder.addParagraphContent(dmpEntity.getDescription(), document, ParagraphStyle.TEXT, BigInteger.ZERO);
+        wordBuilder.addParagraphContent(dmpEntity.getDescription(), document, ParagraphStyle.HTML, BigInteger.ZERO);
 
         wordBuilder.addParagraphContent("Funder", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
         if (dmpEntity.getGrant().getFunder() != null)
@@ -1161,8 +1223,14 @@ public class DataManagementPlanManager {
                  .forEach(datasetEntity -> {
                      Map<String, Object> properties = new HashMap<>();
                      if (datasetEntity.getProperties() != null) {
-                         JSONObject jObject = new JSONObject(datasetEntity.getProperties());
-                         properties = jObject.toMap();
+                         ObjectMapper objectMapper = new ObjectMapper();
+                         try {
+                             properties = objectMapper.readValue(datasetEntity.getProperties(), LinkedHashMap.class);
+                         } catch (IOException e) {
+                             logger.error(e.getLocalizedMessage(), e);
+                         }
+                         /*JSONObject jObject = new JSONObject(datasetEntity.getProperties());
+                         properties = jObject.toMap();*/
                      }
 
                      // Custom style for the Dataset title.
@@ -1226,7 +1294,7 @@ public class DataManagementPlanManager {
                     }*/
 
 
-                     wordBuilder.addParagraphContent(datasetEntity.getDescription(), document, ParagraphStyle.TEXT, BigInteger.ZERO);
+                     wordBuilder.addParagraphContent(datasetEntity.getDescription(), document, ParagraphStyle.HTML, BigInteger.ZERO);
 
                      // Dataset Description custom style.
                      XWPFParagraph datasetDescriptionParagraph = document.createParagraph();
@@ -1277,7 +1345,7 @@ public class DataManagementPlanManager {
         eu.eudat.data.entities.DMP dmp = databaseRepository.getDmpDao().find(UUID.fromString(id));
         if (!dmp.isPublic() && dmp.getUsers().stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()).collect(Collectors.toList()).size() == 0)
             throw new UnauthorisedException();
-        List<Dataset> datasets = dmp.getDataset().stream().collect(Collectors.toList());
+        List<Dataset> datasets = dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != Dmp.DMPStatus.DELETED.getValue()).collect(Collectors.toList());
         /*String fileName = dmp.getLabel();
         fileName = fileName.replaceAll("[^a-zA-Z0-9+ ]", "");*/
         String uuid = UUID.randomUUID().toString();
@@ -1292,6 +1360,22 @@ public class DataManagementPlanManager {
         dmpName.setTextContent(dmp.getLabel());
         dmpElement.appendChild(dmpName);
 
+        if (dmp.getExtraProperties() != null && !dmp.getExtraProperties().isEmpty()) {
+            Map<String, Object> extraProperties = new ObjectMapper().readValue(dmp.getExtraProperties(), HashMap.class);
+            Element language = xmlDoc.createElement("language");
+            language.setTextContent(extraProperties.get("language") != null ? extraProperties.get("language").toString() : null);
+            dmpElement.appendChild(language);
+            Element visibility = xmlDoc.createElement("visibility");
+            visibility.setTextContent(extraProperties.get("visible") != null ? extraProperties.get("visible").toString() : null);
+            dmpElement.appendChild(visibility);
+            Element publicDate = xmlDoc.createElement("publicDate");
+            publicDate.setTextContent(extraProperties.get("publicDate") != null ? extraProperties.get("publicDate").toString() : null);
+            dmpElement.appendChild(publicDate);
+            Element costs = xmlDoc.createElement("costs");
+            costs.setTextContent(extraProperties.get("costs") != null ? extraProperties.get("costs").toString() : null);
+            dmpElement.appendChild(costs);
+        }
+
         DMPProfile dmpProfile = dmp.getProfile();
         Element dmpProfileElement = xmlDoc.createElement("dmpProfile");
         Element dmpProfileName = xmlDoc.createElement("dmpProfileName");
@@ -1407,7 +1491,7 @@ public class DataManagementPlanManager {
         return fileEnvelope;
     }
 
-    public ResponseEntity<byte[]> getRDAJsonDocument(String id, DatasetManager datasetManager, Principal principal) throws Exception {
+    public ResponseEntity<byte[]> getRDAJsonDocument(String id, Principal principal) throws Exception {
         eu.eudat.data.entities.DMP dmp = databaseRepository.getDmpDao().find(UUID.fromString(id));
         if (!dmp.isPublic() && dmp.getUsers().stream().noneMatch(userInfo -> userInfo.getUser().getId() == principal.getId()))
             throw new UnauthorisedException();
@@ -1417,7 +1501,7 @@ public class DataManagementPlanManager {
         /*ObjectMapper mapper = new ObjectMapper();
         mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);*/
         String fileName = "DMP_" + dmp.getGrant().getLabel() + "_" + dmp.getVersion();//dmp.getLabel();
-        fileName = fileName.replaceAll("[^a-zA-Z0-9+ ]", "");
+        fileName = fileName.replaceAll("[^a-zA-Z0-9+ ]", "").replace(" ", "_").replace(",", "_");
         String uuid = UUID.randomUUID().toString();
         File file = new File(this.environment.getProperty("temp.temp") + uuid + ".json");
         OutputStream output = new FileOutputStream(file);
@@ -1456,11 +1540,12 @@ public class DataManagementPlanManager {
             default:
                 file = getXmlDocument(id, principal);
         }
+        String fileName = file.getFilename().replace(" ", "_").replace(",", "_");
         InputStream resource = new FileInputStream(file.getFile());
         HttpHeaders responseHeaders = new HttpHeaders();
         responseHeaders.setContentLength(file.getFile().length());
         responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
-        responseHeaders.set("Content-Disposition", "attachment;filename=" + file.getFilename());
+        responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName);
         responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
         responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
 
@@ -1580,6 +1665,21 @@ public class DataManagementPlanManager {
             dm.setAssociatedUsers(associatedUsers);  // Sets associatedUsers property.
             dm.setDynamicFields(dynamicFields);  // Sets dynamicFields property.
             dm.setDefinition(dmpProfile);
+            ObjectMapper mapper = new ObjectMapper();
+            Map<String, Object> extraPropertiesMap = new HashMap<>();
+            if (dataManagementPlans.get(0).getLanguage() != null) {
+                extraPropertiesMap.put("language", dataManagementPlans.get(0).getLanguage());
+            }
+            if (dataManagementPlans.get(0).getVisibility() != null) {
+                extraPropertiesMap.put("visible", dataManagementPlans.get(0).getVisibility());
+            }
+            if (dataManagementPlans.get(0).getPublicDate() != null) {
+                extraPropertiesMap.put("publicDate", dataManagementPlans.get(0).getPublicDate());
+            }
+            if (dataManagementPlans.get(0).getCosts() != null) {
+                extraPropertiesMap.put("costs", mapper.readValue(dataManagementPlans.get(0).getCosts(), ArrayList.class));
+            }
+            dm.setExtraProperties(extraPropertiesMap);
 
             //createOrUpdate(apiContext, dm, principal);
             DMP dmp = this.createOrUpdate(dm, principal);
@@ -1622,7 +1722,7 @@ public class DataManagementPlanManager {
                 // datasets.add(new DatasetListingModel().fromDataModel(dataset));
             }
 
-            logger.info(dm.toString());
+            //logger.info(dm.toString());
         }
 
         return dataManagementPlans;
@@ -1654,6 +1754,8 @@ public class DataManagementPlanManager {
             databaseRepository.getDmpDao().createOrUpdate(dmp);
             assignUser(dmp, me);
             if (this.apiContext.getOperationsContext().getElasticRepository().getDmpRepository().getClient() != null) {
+                UUID dmpId = dmp.getId();
+                dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
                 this.updateIndex(dmp);
             }
             dmp.getDataset().forEach(dataset -> {
@@ -1676,13 +1778,13 @@ public class DataManagementPlanManager {
                 try {
                     List<Tag> tags = new ArrayList<>();
                     eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
+                    DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset);
                     if (elastic != null) {
                         tags = elastic.getTags();
-                        DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset);
                         datasetWizardModel.setTags(tags);
+                    }
                     datasetManager.getTagsFromProfile(datasetWizardModel, dataset);
                     datasetManager.updateTags(dataset, datasetWizardModel.getTags());
-                    }
                 } catch (Exception e) {
                     logger.error(e.getMessage(), e);
                 }
@@ -1745,6 +1847,9 @@ public class DataManagementPlanManager {
             try {
                 if (dmp.getUsers() != null) {
                     logger.info(dmp.getUsers().toString());
+                } else {
+                    UUID dmpId = dmp.getId();
+                    dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
                 }
                 this.updateIndex(dmp);
             } catch (IOException e) {
@@ -1762,6 +1867,8 @@ public class DataManagementPlanManager {
                 if (elastic != null) {
                     tags = elastic.getTags();
                 }
+                UUID dmpId = dataset.getDmp().getId();
+                dataset.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
                 this.datasetManager.updateTags(dataset, tags);
             } catch (Exception e) {
                 logger.error(e.getMessage(), e);
@@ -1770,7 +1877,7 @@ public class DataManagementPlanManager {
         });
     }
 
-    private void updateIndex(DMP dmp) throws IOException {
+    public void updateIndex(DMP dmp) throws IOException {
         DmpMapper mapper = new DmpMapper(apiContext, datasetManager);
         Dmp elastic = mapper.toElastic(dmp);
         apiContext.getOperationsContext().getElasticRepository().getDmpRepository().createOrUpdate(elastic);
@@ -1782,6 +1889,8 @@ public class DataManagementPlanManager {
                 List<DMP> dmps = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList();
                 dmps.forEach(dmp -> {
                     try {
+                        UUID dmpId = dmp.getId();
+                        dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
                         this.updateIndex(dmp);
                     } catch (IOException e) {
                         logger.error(e.getMessage(), e);
@@ -2016,10 +2125,10 @@ public class DataManagementPlanManager {
         }
         dataBuilder.append("    }\n").append("}");
         createData = dataBuilder.toString();
-        ObjectMapper mapper = new ObjectMapper();
+        /*ObjectMapper mapper = new ObjectMapper();
         mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
-        JsonNode createDataJSON = mapper.readTree(createData);
-        HttpEntity<JsonNode> request = new HttpEntity<>(createDataJSON, headers);
+        JsonNode createDataJSON = mapper.readTree(createData);*/
+        HttpEntity<String> request = new HttpEntity<>(createData, headers);
         Map createResponse = null;
         LinkedHashMap<String, String> links = null;
         String previousDOI = this.getPreviousDOI(dmp.getGroupId(), dmp.getId());
@@ -2081,19 +2190,19 @@ public class DataManagementPlanManager {
                 if (unpublishedUrl == null) {
                     // Second step, add the file to the entry.
                     FileEnvelope file = getWordDocument(id.toString(), principal, configLoader);
-                    /*String name = file.getFilename().substring(0, file.getFilename().length() - 5);
-                    File pdfFile = datasetManager.convertToPDF(file, environment);
-                    String fileName = name + ".pdf";*/
-                    FileSystemResource fileSystemResource = new FileSystemResource(file.getFile());
+                    String name = file.getFilename().substring(0, file.getFilename().length() - 5);
+                    File pdfFile = PDFUtils.convertToPDF(file, environment);
+                    String fileName = name + ".pdf";
+                    FileSystemResource fileSystemResource = new FileSystemResource(pdfFile);
                     HttpEntity<FileSystemResource> addFileMapRequest = new HttpEntity<>(fileSystemResource, null);
 
-                    String addFileUrl = links.get("bucket") + "/" + file.getFilename() + "?access_token=" + zenodoToken;
+                    String addFileUrl = links.get("bucket") + "/" + fileName + "?access_token=" + zenodoToken;
                     restTemplate.put(addFileUrl, addFileMapRequest);
                     Files.deleteIfExists(file.getFile().toPath());
 
                     ResponseEntity<byte[]> jsonFile;
                     try {
-                        jsonFile = getRDAJsonDocument(id.toString(), datasetManager, principal);
+                        jsonFile = getRDAJsonDocument(id.toString(), principal);
                     } catch (Exception e) {
                         throw e;
                     }
@@ -2104,7 +2213,9 @@ public class DataManagementPlanManager {
                         jsonFos.flush();
                     }
                     fileSystemResource = new FileSystemResource(tempJsonFile);
-                    addFileMapRequest = new HttpEntity<>(fileSystemResource, null);
+                    HttpHeaders jsonHeaders = new HttpHeaders();
+                    jsonHeaders.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
+                    addFileMapRequest = new HttpEntity<>(fileSystemResource, jsonHeaders);
                     String jsonFileName = jsonFile.getHeaders().get("Content-Disposition").get(0).substring(jsonFile.getHeaders().get("Content-Disposition").get(0).lastIndexOf('=')  + 1);
                     addFileUrl = links.get("bucket") + "/" + jsonFileName + "?access_token=" + zenodoToken;
                     restTemplate.put(addFileUrl, addFileMapRequest);
@@ -2168,51 +2279,6 @@ public class DataManagementPlanManager {
 
     }
 
-    public long countAllDrafts() {
-        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
-        criteria.setStatus(0);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllFinalized() {
-        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
-        criteria.setStatus(1);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllPublished() {
-        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
-        criteria.setIsPublic(true);
-        criteria.setOnlyPublic(true);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllDoied() {
-        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
-        criteria.setHasDoi(true);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllResearchers() {
-        ResearcherCriteria criteria = new ResearcherCriteria();
-        return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllProjects() {
-        ProjectCriteria criteria = new ProjectCriteria();
-        return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
-    }
-
-    public long countAllFunders() {
-        FunderCriteria criteria = new FunderCriteria();
-        return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
-    }
-
-    public long countAllGrants() {
-        GrantCriteria criteria = new GrantCriteria();
-        return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
-    }
-
     /*public DataTableData<DataManagementPlanListingModel> getPublicPaged(DataManagmentPlanPublicTableRequest dataManagementPlanPublicTableRequest, String fieldsGroup, Principal principal) throws Exception {
         dataManagementPlanPublicTableRequest.setQuery(databaseRepository.getDmpDao().asQueryable().withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class)));
         QueryableList<DMP> items = dataManagementPlanPublicTableRequest.applyCriteria();

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementProfileManager.java

@@ -104,7 +104,8 @@ public class DataManagementProfileManager {
         HttpHeaders responseHeaders = new HttpHeaders();
         responseHeaders.setContentLength(envelope.getFile().length());
         responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
-        responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml");
+        String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
+        responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
         responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
         responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
         byte[] content = org.apache.poi.util.IOUtils.toByteArray(resource);

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java

@@ -48,6 +48,7 @@ import eu.eudat.models.data.security.Principal;
 import eu.eudat.models.data.user.composite.PagedDatasetProfile;
 import eu.eudat.queryable.QueryableList;
 import eu.eudat.types.Authorities;
+import eu.eudat.types.MetricNames;
 import org.apache.commons.io.IOUtils;
 import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
 import org.apache.poi.xwpf.usermodel.XWPFDocument;
@@ -66,6 +67,7 @@ import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.multipart.MultipartFile;
 import org.w3c.dom.Document;
+import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
@@ -87,7 +89,6 @@ import java.nio.file.Files;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
@@ -109,9 +110,10 @@ public class DatasetManager {
     private UserManager userManager;
     private ConfigLoader configLoader;
     private Environment environment;
+    private final MetricsManager metricsManager;
 
     @Autowired
-    public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment) {
+    public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager) {
         this.apiContext = apiContext;
         this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
         this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository();
@@ -119,6 +121,7 @@ public class DatasetManager {
         this.userManager = userManager;
         this.configLoader = configLoader;
         this.environment = environment;
+        this.metricsManager = metricsManager;
     }
 
     public DataTableData<DatasetListingModel> getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception {
@@ -156,7 +159,7 @@ public class DatasetManager {
         List<eu.eudat.elastic.entities.Dataset> datasets;
         try {
             datasets = datasetRepository.exists() ?
-                    datasetRepository.query(datasetCriteria) : null;
+                    datasetRepository.queryIds(datasetCriteria) : null;
         } catch (Exception ex) {
             logger.warn(ex.getMessage(), ex);
             datasets = null;
@@ -168,15 +171,16 @@ public class DatasetManager {
         if (datasets != null) {
 
             if (!datasets.isEmpty()) {
-                items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
-                List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
-                items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
+                //items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
+                final List<UUID> datasetIds = datasets.stream().map(datasetE -> UUID.fromString(datasetE.getId())).distinct().collect(Collectors.toList());
+                items = databaseRepository.getDatasetDao().filterFromElastic(datasetTableRequest.getCriteria(), datasetIds).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
+                //items.where((builder, root) -> root.get("id").in(datasetIds));
             } else {
-                items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
+                items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
                 //items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
             }
         } else {
-            items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
+            items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
         }
         List<Integer> roles = new LinkedList<>();
         QueryableList<eu.eudat.data.entities.Dataset> pagedItems;
@@ -185,9 +189,10 @@ public class DatasetManager {
             if (principal.getId() == null) {
                 throw new UnauthorisedException("You are not allowed to access those datasets");
             }
-            if (datasetTableRequest.getCriteria().getRole() != null)
+            if (datasetTableRequest.getCriteria().getRole() != null) {
                 roles.add(datasetTableRequest.getCriteria().getRole());
-            authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles);
+            }
+            authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles).distinct();
             pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
         } else {
             if (principal.getId() != null && datasetTableRequest.getCriteria().getRole() != null) {
@@ -214,6 +219,7 @@ public class DatasetManager {
     }
 
     public DataTableData<DatasetListingModel> getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception {
+        Long count = 0L;
         DatasetCriteria datasetCriteria = new DatasetCriteria();
         datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
         datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile());
@@ -227,7 +233,8 @@ public class DatasetManager {
         List<eu.eudat.elastic.entities.Dataset> datasets;
         try {
             datasets = datasetRepository.exists() ?
-                    datasetRepository.query(datasetCriteria) : new LinkedList<>();
+                    datasetRepository.queryIds(datasetCriteria) : new LinkedList<>();
+            count = datasetRepository.exists() ? datasetRepository.count(datasetCriteria) : 0L;
         } catch (Exception ex) {
             logger.warn(ex.getMessage());
             datasets = null;
@@ -253,22 +260,22 @@ public class DatasetManager {
                 return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
             });
         }
-        String[] strings = new String[1];
-        strings[0] = "-dmp:publishedAt|join|";
+        List<String> strings = new ArrayList<>();
+        strings.add("-dmp:publishedAt|join|");
         datasetTableRequest.getOrderings().setFields(strings);
+        if (count == 0L) {
+            count = items.count();
+        }
         QueryableList<eu.eudat.data.entities.Dataset> pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
         DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
 
-        CompletableFuture<List<DatasetListingModel>> itemsFuture = pagedItems.
-                selectAsync(this::mapModel).whenComplete((resultList, throwable) -> {
-            dataTable.setData(resultList.stream().filter(Objects::nonNull).collect(Collectors.toList()));
-        });
+        List<DatasetListingModel> datasetListis = pagedItems.
+                select(this::mapModel);
+
+        dataTable.setData(datasetListis.stream().filter(Objects::nonNull).collect(Collectors.toList()));
 
-        CompletableFuture countFuture = pagedItems.countAsync().whenComplete((count, throwable) -> {
         dataTable.setTotalCount(count);
-        });
-
-        CompletableFuture.allOf(itemsFuture, countFuture).join();
+        //CompletableFuture.allOf(countFuture).join();
         return dataTable;
     }
 
@@ -549,54 +556,6 @@ public class DatasetManager {
         return fileEnvelope;
     }
 
-    public File convertToPDF(FileEnvelope file, Environment environment) throws IOException, InterruptedException {
-        LinkedMultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
-        String uuid = UUID.randomUUID().toString();
-        map.add("files", new FileSystemResource(file.getFile()));
-        map.add("filename", uuid + ".pdf");
-        HttpHeaders headers = new HttpHeaders();
-        headers.setContentType(MediaType.MULTIPART_FORM_DATA);
-        headers.add("Content-disposition", "attachment; filename=" + uuid + ".pdf");
-        headers.add("Content-type", "application/pdf");
-
-        HttpEntity<LinkedMultiValueMap<String, Object>> requestEntity = new HttpEntity<LinkedMultiValueMap<String, Object>>(
-                map, headers);
-
-        byte[] queueResult = new RestTemplate().postForObject(environment.getProperty("pdf.converter.url") + "convert/office"
-                , requestEntity, byte[].class);
-
-        File resultPdf = new File(environment.getProperty("temp.temp") + uuid + ".pdf");
-        FileOutputStream output = new FileOutputStream(resultPdf);
-        IOUtils.write(queueResult, output);
-        output.close();
-        Files.deleteIfExists(file.getFile().toPath());
-
-        return resultPdf;
-    }
-
-    private File extractFromZip(File file, String filename) throws IOException {
-        byte[] buffer = new byte[1024];
-        File newFile = new File(filename);
-        ZipInputStream zis = new ZipInputStream(new FileInputStream(file));
-        ZipEntry zipEntry = zis.getNextEntry();
-        while (zipEntry != null) {
-            String zippedFileName = zipEntry.getName();
-            if (zippedFileName.equals("pdf")) {
-
-                FileOutputStream fos = new FileOutputStream(newFile);
-                int len;
-                while ((len = zis.read(buffer)) > 0) {
-                    fos.write(buffer, 0, len);
-                }
-                fos.close();
-                zipEntry = zis.getNextEntry();
-            }
-        }
-        zis.closeEntry();
-        zis.close();
-        return newFile;
-    }
-
     public eu.eudat.data.entities.Dataset createOrUpdate(DatasetWizardModel datasetWizardModel, Principal principal) throws Exception {
         Boolean sendNotification = false;
         Dataset tempDataset = null;
@@ -613,7 +572,7 @@ public class DatasetManager {
                 sendNotification = true;
             }
         } else {
-            MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft");
+            metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
         }
         if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null)
             throw new Exception("DMP is finalized, therefore Dataset cannot be edited.");
@@ -639,6 +598,8 @@ public class DatasetManager {
         }
         dataset1.setProfile(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(datasetWizardModel.getProfile().getId()));
 //        datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset1));
+        UUID dmpId = dataset1.getDmp().getId();
+        dataset1.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
         updateTags(dataset1, datasetWizardModel.getTags());
         if (sendNotification) {
             if (dataset1.getStatus() != Dataset.Status.FINALISED.getValue()) {
@@ -691,6 +652,9 @@ public class DatasetManager {
             datasetProfileValidators.add(node.getNodeValue());
         }
 
+        expression = "//validation/@type[.=1]/ancestor::fieldSet";
+        nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);
+
 
         JSONObject obj = new JSONObject(dataset.getProperties());
         VisibilityRuleService visibilityRuleService = this.apiContext.getUtilitiesService().getVisibilityRuleService();
@@ -702,14 +666,37 @@ public class DatasetManager {
 
 
         for (String validator : datasetProfileValidators) {
-            if ((obj.getString(validator) == null || obj.getString(validator).trim().isEmpty()) && visibilityRuleService.isElementVisible(validator)) {
+            if ((obj.has(validator) && (obj.getString(validator) == null || obj.getString(validator).trim().isEmpty())) && isElementVisible(nodeList, validator, visibilityRuleService)) {
                 throw new Exception("Field value of " + validator + " must be filled.");
             }
         }
     }
 
+    private boolean isElementVisible(NodeList nodeList, String id, VisibilityRuleService visibilityRuleService) {
+        Element fieldSet = null;
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            for (int j = 0; j < node.getChildNodes().getLength(); j++) {
+                Node fcnode = node.getChildNodes().item(j);
+                if (fcnode.getNodeName().equals("fields")) {
+                    for(int k = 0; k < fcnode.getChildNodes().getLength(); k++) {
+                        Node scnode = fcnode.getChildNodes().item(k);
+                        if (scnode.getNodeName().equals("field") && scnode.getAttributes().getNamedItem("id").getNodeValue().equals(id)) {
+                            fieldSet = (Element) node;
+                        }
+                    }
+                }
+            }
+        }
+        if (fieldSet != null) {
+            return visibilityRuleService.isElementVisible(id) && visibilityRuleService.isElementVisible(fieldSet.getAttribute("id"));
+        } else {
+            return visibilityRuleService.isElementVisible(id);
+        }
+    }
+
     private String propertiesModelToString(PagedDatasetProfile pagedDatasetProfile) {
-        Map<String, Object> values = new HashMap();
+        Map<String, Object> values = new LinkedHashMap<>();
         pagedDatasetProfile.toMap(values);
         JSONObject jobject = new JSONObject(values);
         return jobject.toString();
@@ -857,7 +844,7 @@ public class DatasetManager {
         if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue())
             throw new Exception("You cannot make public a Dataset That Has not Been Finalised");
         datasetDao.createOrUpdate(dataset);
-        MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published");
+        metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
     }
 
     public ResponseEntity<byte[]> getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException {
@@ -868,7 +855,8 @@ public class DatasetManager {
         HttpHeaders responseHeaders = new HttpHeaders();
         responseHeaders.setContentLength(envelope.getFile().length());
         responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
-        responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml");
+        String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
+        responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
         responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
         responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
 
@@ -946,7 +934,7 @@ public class DatasetManager {
         createServicesIfTheyDontExist(entity);
         createExternalDatasetsIfTheyDontExist(entity);
 
-        MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft");
+        metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
         return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity);
     }
 
@@ -1028,6 +1016,8 @@ public class DatasetManager {
             datasetEntities.forEach(datasetEntity -> {
                 try {
                     eu.eudat.elastic.entities.Dataset dataset = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(datasetEntity.getId().toString());
+                    UUID dmpId = datasetEntity.getDmp().getId();
+                    datasetEntity.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
                     updateTags(datasetEntity, dataset != null ? dataset.getTags() : null);
                 } catch (Exception e) {
                     logger.error(e.getMessage(), e);
@@ -1052,14 +1042,17 @@ public class DatasetManager {
         ObjectMapper mapper = new ObjectMapper();
         String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition());
         JsonNode propertiesJson = mapper.readTree(json);
-        List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
+        DatasetCriteria criteria = new DatasetCriteria();
+        criteria.setHasTags(true);
+        List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
         Set<JsonNode> tagNodes = new HashSet<>();
         tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true));
         tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "rdaProperty", "dataset.keyword"));
         if (!tagNodes.isEmpty()) {
             tagNodes.forEach(node -> {
                 JsonNode value = node.get("value");
-                String stringValue = value.asText().replaceAll("=", ":");
+                if (!value.toString().equals("\"\"")) {
+                    String stringValue = value.toString().replaceAll("=", ":");
                     JSONArray values = new JSONArray(stringValue);
                     if (values != null) {
                         values.iterator().forEachRemaining(element -> {
@@ -1069,6 +1062,7 @@ public class DatasetManager {
                     } else {
                         this.addTag(tags, wizardModel.getTags(), "", value.asText());
                     }
+                }
             });
         }
     }
@@ -1088,10 +1082,10 @@ public class DatasetManager {
 
     @Transactional
     private DatasetListingModel mapModel(Dataset item) {
-        if (item.getProfile() == null)
-            return null;
+        /*if (item.getProfile() == null)
+            return null;*/
         DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item);
-        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
+        /*DatasetProfileCriteria criteria = new DatasetProfileCriteria();
         criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId()));
         List<DatasetProfile> profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
         boolean islast = false;
@@ -1099,31 +1093,7 @@ public class DatasetManager {
             profiles = profiles.stream().sorted(Comparator.comparing(DatasetProfile::getVersion)).collect(Collectors.toList());
             islast = profiles.get(0).getId().equals(item.getProfile().getId());
         }
-        listingModel.setProfileLatestVersion(islast);
+        listingModel.setProfileLatestVersion(islast);*/
         return listingModel;
     }
-
-    public long countAllDraft() {
-        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
-        criteria.setStatus(0);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllFinalized() {
-        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
-        criteria.setStatus(1);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllPublic() {
-        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
-        criteria.setIsPublic(true);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
-    }
-
-    public long countAllWithDoi() {
-        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
-        criteria.setHasDoi(true);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
-    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java

@@ -24,9 +24,11 @@ import eu.eudat.models.data.entities.xmlmodels.datasetprofiledefinition.Field;
 import eu.eudat.models.data.externaldataset.ExternalAutocompleteFieldModel;
 import eu.eudat.models.data.helpers.common.DataTableData;
 import eu.eudat.models.data.listingmodels.UserInfoListingModel;
+import eu.eudat.models.data.mail.SimpleMail;
 import eu.eudat.models.data.security.Principal;
 import eu.eudat.queryable.QueryableList;
 import eu.eudat.types.Authorities;
+import eu.eudat.types.MetricNames;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -48,6 +50,7 @@ import javax.xml.xpath.*;
 import java.io.*;
 import java.nio.file.Files;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 
 
@@ -59,12 +62,14 @@ public class DatasetProfileManager {
     private DatabaseRepository databaseRepository;
     private Environment environment;
     private List<String> cache;
+    private final MetricsManager metricsManager;
 
     @Autowired
-    public DatasetProfileManager(ApiContext apiContext, Environment environment) {
+    public DatasetProfileManager(ApiContext apiContext, Environment environment, MetricsManager metricsManager) {
         this.apiContext = apiContext;
         this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
         this.environment = environment;
+        this.metricsManager = metricsManager;
         this.cache = new ArrayList<>();
     }
 
@@ -92,7 +97,7 @@ public class DatasetProfileManager {
         DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(UUID.fromString(id));
         apiContext.getOperationsContext().getDatabaseRepository().detachEntity(profile);
         profile.setId(null);
-        MetricsManager.increaseValue("argos_dataset_templates", 1, "total");
+        metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
         return profile;
     }
 
@@ -102,7 +107,8 @@ public class DatasetProfileManager {
         if (principal.getAuthz().contains(Authorities.ADMIN)) {
             authItems = items;
         } else if (principal.getAuthz().contains(Authorities.DATASET_PROFILE_MANAGER)) {
-            authItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getAuthenticated(items, principal.getId(), null);
+            List<Integer> roles = Collections.singletonList(1);
+            authItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getAuthenticated(items, principal.getId(), roles);
         }
         QueryableList<DatasetProfile> pagedItems = PaginationManager.applyPaging(authItems, datasetProfileTableRequestItem);
         List<DatasetProfileListingModel> datasetProfiles = pagedItems.select(item -> new DatasetProfileListingModel().fromDataModel(item));
@@ -170,7 +176,7 @@ public class DatasetProfileManager {
                     response = restTemplate.exchange(url, HttpMethod.GET, entity, Object.class);
                     jsonContext = JsonPath.parse(response.getBody());
                     jsonItems = jsonContext.read(singleData.getOptionsRoot() + "['" + singleData.getAutoCompleteOptions().getLabel() + "','" + singleData.getAutoCompleteOptions().getValue() + "','" + singleData.getAutoCompleteOptions().getSource() + "','" + "uri" + "']");
-                    jsonItems.forEach(item -> result.add(new ExternalAutocompleteFieldModel(item.get(singleData.getAutoCompleteOptions().getValue()), item.get(singleData.getAutoCompleteOptions().getLabel()), item.get(singleData.getAutoCompleteOptions().getSource())  != null ? item.get(singleData.getAutoCompleteOptions().getSource()) : singleData.getAutoCompleteOptions().getSource(), item.get("uri"))));
+                    jsonItems.forEach(item -> result.add(new ExternalAutocompleteFieldModel(parseItem(item.get(singleData.getAutoCompleteOptions().getValue())), parseItem(item.get(singleData.getAutoCompleteOptions().getLabel())), item.get(singleData.getAutoCompleteOptions().getSource())  != null ? parseItem(item.get(singleData.getAutoCompleteOptions().getSource())) : singleData.getAutoCompleteOptions().getSource(), parseItem(item.get("uri")))));
                     break;
                 case CACHED:
                     headers.setAccept(Collections.singletonList(MediaType.valueOf("text/plain; charset=utf-8")));
@@ -196,6 +202,20 @@ public class DatasetProfileManager {
         //return result;
     }
 
+    private String parseItem(Object item) {
+        if (item instanceof String) {
+            return (String) item;
+        }
+        if (item instanceof List) {
+            List listedItems = (List) item;
+            return parseItem(listedItems.get(0));
+        }
+        if (item instanceof Map) {
+            return (String) ((Map)item).get("$");
+        }
+        return item.toString();
+    }
+
     public ResponseEntity<byte[]> getDocument(eu.eudat.models.data.user.composite.DatasetProfile datasetProfile, String label) throws IllegalAccessException, IOException, InstantiationException {
         FileEnvelope envelope = getXmlDocument(datasetProfile, label);
         InputStream resource = new FileInputStream(envelope.getFile());
@@ -204,7 +224,8 @@ public class DatasetProfileManager {
         HttpHeaders responseHeaders = new HttpHeaders();
         responseHeaders.setContentLength(envelope.getFile().length());
         responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
-        responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml");
+        String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
+        responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
         responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
         responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");
 
@@ -232,7 +253,7 @@ public class DatasetProfileManager {
             File localFile = convert(multiPartFile);
             eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileModel.DatasetProfile profile = xmlBuilder.build(localFile);
             Files.deleteIfExists(localFile.toPath());
-            MetricsManager.increaseValue("argos_dataset_templates", 1, "total");
+            metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
             return profile;
         } catch (IOException e) {
             logger.error(e.getMessage(), e);
@@ -281,17 +302,29 @@ public class DatasetProfileManager {
             if (entity.getUsers() == null) {
                 entity.setUsers(new HashSet<>());
             }
-            model.getUsers().stream().filter(userInfoListingModel -> entity.getUsers().stream().filter(userDatasetProfile -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0).forEach(userInfoListingModel -> {
+            model.getUsers().stream().filter(userInfoListingModel -> entity.getUsers().stream()
+                    .filter(userDatasetProfile -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0)
+                    .forEach(userInfoListingModel -> {
                 UserDatasetProfile userDatasetProfile1 = new UserDatasetProfile();
                 userDatasetProfile1.setDatasetProfile(entity);
                 UserInfo userInfo1 = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(userInfoListingModel.getId());
                 userDatasetProfile1.setUser(userInfo1);
                 userDatasetProfile1.setRole(1);
                 apiContext.getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile1);
+                sendJoinMail(userDatasetProfile1);
+            });
+            entity.getUsers().stream().filter(userDatasetProfile -> model.getUsers().stream()
+                    .filter(userInfoListingModel -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() > 0
+                    && userDatasetProfile.getRole() == 2).forEach(userDatasetProfile -> {
+                userDatasetProfile.setRole(1);
+                apiContext.getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
+                sendJoinMail(userDatasetProfile);
             });
         }
         if (entity.getUsers() != null && !entity.getUsers().isEmpty()) {
-            entity.getUsers().stream().filter(userDatasetProfile -> model.getUsers().stream().filter(userInfoListingModel -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0).forEach(userDatasetProfile -> {
+            entity.getUsers().stream().filter(userDatasetProfile -> model.getUsers().stream()
+                    .filter(userInfoListingModel -> userDatasetProfile.getUser().getId().equals(userInfoListingModel.getId())).count() == 0)
+                    .forEach(userDatasetProfile -> {
                 userDatasetProfile.setRole(2);
                 apiContext.getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
             });
@@ -312,24 +345,21 @@ public class DatasetProfileManager {
         }
     }
 
-    public long countAlldraft() {
-        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
-        criteria.setStatus(0);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
+    private void sendJoinMail(UserDatasetProfile userDatasetProfile) {
+            SimpleMail mail = new SimpleMail();
+            mail.setSubject(environment.getProperty("admin.mail.subject").replace( "{templateName}", userDatasetProfile.getDatasetProfile().getLabel()));
+            String content = apiContext.getUtilitiesService().getMailService().getMailTemplateContent(environment.getProperty("email.dataset.template"));
+            content = content.replace("{recipient}", userDatasetProfile.getUser().getName());
+            content = content.replace("{templateName}", userDatasetProfile.getDatasetProfile().getLabel());
+            content = content.replace("{host}", this.environment.getProperty("dmp.domain"));
+            content = content.replace("{templateID}", userDatasetProfile.getDatasetProfile().getId().toString());
+            mail.setContent(content);
+            mail.setTo(userDatasetProfile.getUser().getEmail());
+            try {
+                apiContext.getUtilitiesService().getMailService().sendSimpleMail(mail);
+            } catch (Exception ex) {
+                logger.error(ex.getMessage(), ex);
             }
 
-    public long countAllFinalized() {
-        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
-        criteria.setStatus(1);
-        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
-    }
-
-    @Transactional
-    public long countAllUsed() {
-        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
-        criteria.setStatus(1);
-        criteria.setAllVersions(false);
-        List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
-        return datasetProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).filter(datasetProfile -> !datasetProfile.getDataset().isEmpty()).count();
     }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetWizardManager.java

@@ -1,15 +1,17 @@
 package eu.eudat.logic.managers;
 
+import eu.eudat.data.dao.criteria.DatasetProfileCriteria;
 import eu.eudat.data.dao.entities.DMPDao;
-
+import eu.eudat.data.dao.entities.DatasetProfileDao;
 import eu.eudat.data.entities.DMP;
 import eu.eudat.data.entities.Dataset;
+import eu.eudat.data.entities.DatasetProfile;
 import eu.eudat.data.entities.UserInfo;
+import eu.eudat.data.query.items.item.dataset.DatasetWizardAutocompleteRequest;
+import eu.eudat.data.query.items.item.datasetprofile.DatasetProfileWizardAutocompleteRequest;
 import eu.eudat.exceptions.datasetwizard.DatasetWizardCannotUnlockException;
 import eu.eudat.logic.services.ApiContext;
 import eu.eudat.models.data.datasetwizard.DataManagentPlanListingModel;
-import eu.eudat.data.query.items.item.datasetprofile.DatasetProfileWizardAutocompleteRequest;
-import eu.eudat.data.query.items.item.dataset.DatasetWizardAutocompleteRequest;
 import eu.eudat.models.data.dmp.AssociatedProfile;
 import eu.eudat.models.data.dmp.DataManagementPlan;
 import eu.eudat.models.data.security.Principal;
@@ -19,6 +21,7 @@ import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 
 public class DatasetWizardManager {
@@ -31,12 +34,18 @@ public class DatasetWizardManager {
         return dataManagementPlans;
     }
 
-    public static List<AssociatedProfile> getAvailableProfiles(DMPDao dmpRepository, DatasetProfileWizardAutocompleteRequest datasetProfileWizardAutocompleteRequest) throws InstantiationException, IllegalAccessException {
+    public static List<AssociatedProfile> getAvailableProfiles(DMPDao dmpRepository, DatasetProfileDao profileDao, DatasetProfileWizardAutocompleteRequest datasetProfileWizardAutocompleteRequest) throws InstantiationException, IllegalAccessException {
         DataManagementPlan dataManagementPlan = new DataManagementPlan().fromDataModel(dmpRepository.find(datasetProfileWizardAutocompleteRequest.getCriteria().getId()));
         if (dataManagementPlan.getProfiles() == null || dataManagementPlan.getProfiles().isEmpty()) {
             return new LinkedList<>();
         }
-        List<AssociatedProfile> profiles = dataManagementPlan.getProfiles();
+        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
+        criteria.setIds(dataManagementPlan.getProfiles().stream().map(AssociatedProfile::getId).collect(Collectors.toList()));
+        List<DatasetProfile> datasetProfiles = profileDao.getWithCriteria(criteria).toList();
+        criteria.setIds(null);
+        criteria.setGroupIds(datasetProfiles.stream().map(DatasetProfile::getGroupId).collect(Collectors.toList()));
+        datasetProfiles = profileDao.getWithCriteria(criteria).toList();
+        List<AssociatedProfile> profiles = datasetProfiles.stream().map(profile -> new AssociatedProfile().fromData(profile)).collect(Collectors.toList());
         return profiles;
     }
 

dmp-backend/web/src/main/java/eu/eudat/logic/managers/DocumentManager.java

@@ -1,159 +0,0 @@
-package eu.eudat.logic.managers;
-
-import eu.eudat.data.dao.entities.DatasetDao;
-import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
-import eu.eudat.logic.services.ApiContext;
-import eu.eudat.logic.services.forms.VisibilityRuleService;
-import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
-import eu.eudat.logic.utilities.documents.word.WordBuilder;
-import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder;
-import eu.eudat.models.HintedModelFactory;
-import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
-import eu.eudat.models.data.user.composite.PagedDatasetProfile;
-import org.apache.commons.io.IOUtils;
-import org.apache.poi.xwpf.usermodel.XWPFDocument;
-import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.core.env.Environment;
-import org.springframework.core.io.FileSystemResource;
-import org.springframework.http.*;
-import org.springframework.http.converter.ByteArrayHttpMessageConverter;
-import org.springframework.stereotype.Service;
-import org.springframework.util.LinkedMultiValueMap;
-import org.springframework.web.client.RestTemplate;
-
-import java.io.*;
-import java.net.URL;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-/**
- * Created by ikalyvas on 10/16/2018.
- */
-@Service
-public class DocumentManager {
-    private static final Logger logger = LoggerFactory.getLogger(DocumentManager.class);
-
-    private ApiContext context;
-    private DatasetManager datasetManager;
-    private ConfigLoader configLoader;
-    private Environment environment;
-
-    public DocumentManager(ApiContext context, DatasetManager datasetManager, ConfigLoader configLoader, Environment environment) {
-        this.context = context;
-        this.datasetManager = datasetManager;
-        this.configLoader = configLoader;
-        this.environment = environment;
-    }
-
-    public File getWordDocument(ConfigLoader configLoader, DatasetDao datatasetRepository, String id, VisibilityRuleService visibilityRuleService) throws InstantiationException, IllegalAccessException, IOException {
-        WordBuilder wordBuilder = new WordBuilder();
-        DatasetWizardModel dataset = new DatasetWizardModel();
-        XWPFDocument document = configLoader.getDocument();
-        eu.eudat.data.entities.Dataset datasetEntity = datatasetRepository.find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
-        Map<String, Object> properties = new HashMap<>();
-        if (datasetEntity.getProperties() != null) {
-            JSONObject jobject = new JSONObject(datasetEntity.getProperties());
-            properties = jobject.toMap();
-        }
-        PagedDatasetProfile pagedDatasetProfile = datasetManager.getPagedProfile(dataset, datasetEntity);
-        visibilityRuleService.setProperties(properties);
-        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
-        wordBuilder.build(document, pagedDatasetProfile, visibilityRuleService);
-        File exportFile = new File(dataset.getLabel() + ".docx");
-        FileOutputStream out = new FileOutputStream(exportFile);
-        document.write(out);
-        out.close();
-        return exportFile;
-    }
-
-    public FileEnvelope getXmlDocument(eu.eudat.data.entities.Dataset datasetEntity, String id, VisibilityRuleService visibilityRuleService) throws InstantiationException, IllegalAccessException, IOException {
-        ExportXmlBuilder xmlBuilder = new ExportXmlBuilder();
-        DatasetWizardModel dataset = new DatasetWizardModel();
-        Map<String, Object> properties = new HashMap<>();
-        if (datasetEntity.getProperties() != null) {
-            JSONObject jobject = new JSONObject(datasetEntity.getProperties());
-            properties = jobject.toMap();
-        }
-        PagedDatasetProfile pagedDatasetProfile = datasetManager.getPagedProfile(dataset, datasetEntity);
-        visibilityRuleService.setProperties(properties);
-        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
-        File file = xmlBuilder.build(pagedDatasetProfile, UUID.fromString(id), visibilityRuleService, environment);
-        FileEnvelope fileEnvelope = new FileEnvelope();
-        fileEnvelope.setFile(file);
-        fileEnvelope.setFilename(datasetEntity.getLabel());
-        return fileEnvelope;
-    }
-
-    public File convertToPDF(File file, Environment environment, String label) throws IOException, InterruptedException {
-        LinkedMultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
-        map.add("file", new FileSystemResource(file));
-        HttpHeaders headers = new HttpHeaders();
-        headers.setContentType(MediaType.MULTIPART_FORM_DATA);
-        headers.add("Content-disposition", "attachment; filename=" + label + ".docx");
-
-        headers.add("Content-type", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
-
-        HttpEntity<LinkedMultiValueMap<String, Object>> requestEntity = new HttpEntity<LinkedMultiValueMap<String, Object>>(
-                map, headers);
-
-        Map queueResult = new RestTemplate().postForObject(
-                environment.getProperty("pdf.converter.url") +
-                        "api/v1/", requestEntity, Map.class);
-
-        Map mediaResult = new RestTemplate().getForObject(environment.getProperty("pdf.converter.url") +
-                "/api/v1/" + queueResult.get("id"), Map.class);
-        logger.info("Status: " + mediaResult.get("status"));
-        while (!mediaResult.get("status").equals("finished")) {
-            Thread.sleep(500);
-            mediaResult = new RestTemplate().getForObject(environment.getProperty("pdf.converter.url") +
-                    "api/v1/" + queueResult.get("id"), Map.class);
-            logger.info("Polling");
-        }
-        RestTemplate restTemplate = new RestTemplate();
-        restTemplate.getMessageConverters().add(new ByteArrayHttpMessageConverter());
-        HttpHeaders headers2 = new HttpHeaders();
-        headers.setAccept(Arrays.asList(MediaType.APPLICATION_OCTET_STREAM));
-        HttpEntity<String> entity = new HttpEntity<String>(headers2);
-
-        ResponseEntity<byte[]> response = restTemplate.exchange(environment.getProperty("pdf.converter.url") +
-                mediaResult.get("result_url"), HttpMethod.GET, entity, byte[].class, "1");
-
-        UUID uuid = UUID.randomUUID();
-        File zip = new File(uuid + ".zip");
-        if (response.getStatusCode().equals(HttpStatus.OK)) {
-            FileOutputStream output = new FileOutputStream(zip);
-            IOUtils.write(response.getBody(), output);
-        }
-        return extractFromZip(zip, label + ".pdf");
-    }
-
-    private File extractFromZip(File file, String filename) throws IOException {
-        byte[] buffer = new byte[1024];
-        File newFile = new File(filename);
-        ZipInputStream zis = new ZipInputStream(new FileInputStream(file));
-        ZipEntry zipEntry = zis.getNextEntry();
-        while (zipEntry != null) {
-            String zippedFileName = zipEntry.getName();
-            if (zippedFileName.equals("pdf")) {
-
-                FileOutputStream fos = new FileOutputStream(newFile);
-                int len;
-                while ((len = zis.read(buffer)) > 0) {
-                    fos.write(buffer, 0, len);
-                }
-                fos.close();
-                zipEntry = zis.getNextEntry();
-            }
-        }
-        zis.closeEntry();
-        zis.close();
-        return newFile;
-    }
-}

dmp-backend/web/src/main/java/eu/eudat/logic/managers/InvitationsManager.java

@@ -3,6 +3,7 @@ package eu.eudat.logic.managers;
 import eu.eudat.data.entities.DMP;
 import eu.eudat.data.entities.UserDMP;
 import eu.eudat.data.entities.UserInfo;
+import eu.eudat.data.query.items.item.userinfo.UserInfoRequestItem;
 import eu.eudat.exceptions.security.UnauthorisedException;
 import eu.eudat.logic.services.ApiContext;
 import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
@@ -17,6 +18,7 @@ import org.springframework.stereotype.Component;
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Unmarshaller;
+import java.io.IOException;
 import java.io.StringReader;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -26,10 +28,12 @@ import java.util.stream.Stream;
 public class InvitationsManager {
 
     private ApiContext apiContext;
+    private DataManagementPlanManager dataManagementPlanManager;
 
     @Autowired
-    public InvitationsManager(ApiContext apiContext) {
+    public InvitationsManager(ApiContext apiContext, DataManagementPlanManager dataManagementPlanManager) {
         this.apiContext = apiContext;
+        this.dataManagementPlanManager = dataManagementPlanManager;
     }
 
     public void inviteUsers(Invitation invitation, Principal principal) throws Exception {
@@ -79,7 +83,24 @@ public class InvitationsManager {
         return userModels;
     }
 
-    public UUID assignUserAcceptedInvitation(UUID invitationID, Principal principal) throws UnauthorisedException, JAXBException {
+    public List<UserInfoInvitationModel> getUsersWithCriteria(Principal principal, UserInfoRequestItem userInfoRequestItem) throws IllegalAccessException, InstantiationException {
+        List<UserInfo> users = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao()
+                .getAuthenticated(apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable()
+                        .where(((builder, root) ->
+                                builder.notEqual(root.get("status"), DMP.DMPStatus.DELETED.getValue()))), principal.getId(), Stream.of(0, 1).collect(Collectors.toList()))
+                .toList().stream().map(DMP::getUsers).flatMap(Collection::stream).map(UserDMP::getUser)
+                .filter(userInfo -> !userInfo.getId().equals(principal.getId())).filter(StreamDistinctBy.distinctByKey(UserInfo::getId))
+                .filter(userInfo -> (userInfoRequestItem == null || userInfoRequestItem.getCriteria() == null || userInfoRequestItem.getCriteria().getLike() == null
+                        || userInfo.getName().toLowerCase().contains(userInfoRequestItem.getCriteria().getLike().toLowerCase())
+                        || (userInfo.getEmail().toLowerCase().contains(userInfoRequestItem.getCriteria().getLike().toLowerCase()))))
+                .collect(Collectors.toList());
+//                                .where((builder, root) -> builder.like(builder.upper(root.get("name")), "%" + userInfoRequestItem.getCriteria().getLike().toUpperCase() + "%"))
+
+        List<UserInfoInvitationModel> userModels = users.stream().map(userInfo -> new UserInfoInvitationModel().fromDataModel(userInfo)).collect(Collectors.toList());
+        return userModels;
+    }
+
+    public UUID assignUserAcceptedInvitation(UUID invitationID, Principal principal) throws UnauthorisedException, JAXBException, IOException {
         eu.eudat.data.entities.Invitation invitation = apiContext.getOperationsContext().getDatabaseRepository().getInvitationDao().find(invitationID);
         if (invitation == null)
             throw new UnauthorisedException("There is no Data Management Plan assigned to this Link");
@@ -98,6 +119,8 @@ public class InvitationsManager {
                 apiContext.getUtilitiesService().getInvitationService().assignToDmp(apiContext.getOperationsContext().getDatabaseRepository().getDmpDao(), existingUserDMP, datamanagementPlan);
                 invitation.setAcceptedInvitation(true);
                 apiContext.getOperationsContext().getDatabaseRepository().getInvitationDao().createOrUpdate(invitation);
+                datamanagementPlan.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), datamanagementPlan.getId())).toList()));
+                dataManagementPlanManager.updateIndex(datamanagementPlan);
                 return datamanagementPlan.getId();
             }
         } else {
@@ -121,6 +144,8 @@ public class InvitationsManager {
             apiContext.getUtilitiesService().getInvitationService().assignToDmp(apiContext.getOperationsContext().getDatabaseRepository().getDmpDao(), userDMP, datamanagementPlan);
             invitation.setAcceptedInvitation(true);
             apiContext.getOperationsContext().getDatabaseRepository().getInvitationDao().createOrUpdate(invitation);
+            datamanagementPlan.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), datamanagementPlan.getId())).toList()));
+            dataManagementPlanManager.updateIndex(datamanagementPlan);
             return datamanagementPlan.getId();
         }
         return invitation.getDmp().getId();

dmp-backend/web/src/main/java/eu/eudat/logic/managers/LicenseManager.java

@@ -42,7 +42,7 @@ public class LicenseManager {
 
         ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         licenseModels.addAll(remoteRepos.stream().map(item -> mapper.convertValue(item, LicenseModel.class)).collect(Collectors.toList()));
-        licenseModels = licenseModels.stream().filter(licenseModel -> licenseModel.getName().contains(query)).collect(Collectors.toList());
+        licenseModels = licenseModels.stream().filter(licenseModel -> licenseModel.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
         return licenseModels;
     }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/LocalFetchManager.java

@@ -22,7 +22,7 @@ public class LocalFetchManager {
 	public List<LocalFetchModel> getCurrency(String query) throws Exception {
 		List<Map<String, String>> data = localFetcher.retrieveCurrency();
 		List<LocalFetchModel> result = data.stream().map(entry -> new LocalFetchModel(entry.get("name"), entry.get("value"))).collect(Collectors.toList());
-		result = result.stream().filter(localFetchModel -> localFetchModel.getValue() != null).filter(StreamDistinctBy.distinctByKey(LocalFetchModel::getValue)).filter(localFetchModel -> localFetchModel.getName().contains(query)).collect(Collectors.toList());
+		result = result.stream().filter(localFetchModel -> localFetchModel.getValue() != null).filter(StreamDistinctBy.distinctByKey(LocalFetchModel::getValue)).filter(localFetchModel -> localFetchModel.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
 		return result;
 	}
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java

@@ -1,5 +1,11 @@
 package eu.eudat.logic.managers;
 
+import eu.eudat.data.dao.criteria.*;
+import eu.eudat.data.entities.DatasetProfile;
+import eu.eudat.logic.services.ApiContext;
+import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
+import eu.eudat.types.MetricNames;
+import io.micrometer.prometheus.PrometheusMeterRegistry;
 import io.prometheus.client.Gauge;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -13,32 +19,20 @@ import javax.transaction.Transactional;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.util.Map;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 @Component
 public class MetricsManager {
     private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class);
-    private static Map<String, Gauge> gauges = Stream.of( new Object[][]{
-            {"argos_managed_dmps", Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register()},
+    private final Map<String, Gauge> gauges;
 
-            {"argos_funders", Gauge.build().name("argos_funders").help("Number of registered Funders").register()},
-            {"argos_grants", Gauge.build().name("argos_grants").help("Number of registered Grants").register()},
-            {"argos_projects", Gauge.build().name("argos_projects").help("Number of registered Projects").register()},
-            {"argos_researchers", Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register()},
-
-            {"argos_managed_dataset_descriptions", Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register()},
-
-            {"argos_dataset_templates", Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register()},
-
-            {"argos_users", Gauge.build().name("argos_users").help("Number of users").labelNames("type").register()},
-
-            {"argos_languages", Gauge.build().name("argos_languages").help("Number of Languages").register()},
-
-    }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
-    
-    public static void increaseValue(String name, int amount,  String label) {
+    public void increaseValue(String name, int amount,  String label) {
 
         if(label != null) {
             gauges.get(name).labels(label).inc(amount);
@@ -47,7 +41,7 @@ public class MetricsManager {
         }
     }
     
-    public static void decreaseValue(String name, int amount, String label) {
+    public void decreaseValue(String name, int amount, String label) {
         if(label != null) {
             gauges.get(name).labels(label).dec(amount);
         } else {
@@ -55,7 +49,7 @@ public class MetricsManager {
         }
     }
 
-    public static Integer getValue(String name, String label) {
+    public Integer getValue(String name, String label) {
         if(label != null) {
             return Double.valueOf(gauges.get(name).labels(label).get()).intValue();
         } else {
@@ -63,7 +57,7 @@ public class MetricsManager {
         }
     }
 
-    public static void calculateValue(String name, int amount, String label) {
+    public void calculateValue(String name, int amount, String label) {
         Integer orig = getValue(name, label);
         int diff = orig - amount;
         if (diff != 0) {
@@ -75,19 +69,51 @@ public class MetricsManager {
         }
     }
 
-    private final DatasetManager datasetManager;
-    private final DataManagementPlanManager dataManagementPlanManager;
-    private final DatasetProfileManager datasetProfileManager;
+    private final ApiContext apiContext;
     private final UserManager userManager;
     private final Environment environment;
 
     @Autowired
-    public MetricsManager(DatasetManager datasetManager, DataManagementPlanManager dataManagementPlanManager, DatasetProfileManager datasetProfileManager, UserManager userManager, Environment environment) {
-        this.datasetManager = datasetManager;
-        this.dataManagementPlanManager = dataManagementPlanManager;
-        this.datasetProfileManager = datasetProfileManager;
+    public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) {
+        this.apiContext = apiContext;
         this.userManager = userManager;
         this.environment = environment;
+        registry.clear();
+        this.gauges = Stream.of( new Object[][]{
+                {MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
+                {MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())},
+                {MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())},
+                {MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())},
+                {MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())},
+                {MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())},
+                {MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())},
+                {MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
+
+                {MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
+                {MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
+
+        }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
+
     }
 
     @PostConstruct
@@ -95,32 +121,293 @@ public class MetricsManager {
     @Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60)
     public void init() throws IOException {
         logger.info("Start calculating Metrics");
-        calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDrafts(), "draft");
-        calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllFinalized(), "finalized");
-        calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllPublished(), "published");
-        calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDoied(), "doied");
+        calculateValue(MetricNames.DMP, (int) countAllDraftDMPs(), MetricNames.DRAFT);
+        calculateValue(MetricNames.DMP, (int) countAllFinalizedDMPs(), MetricNames.FINALIZED);
+        calculateValue(MetricNames.DMP, (int) countAllPublishedDMPs(), MetricNames.PUBLISHED);
+        calculateValue(MetricNames.DMP, (int) countAllDoiedDMPs(), MetricNames.DOIED);
 
-        calculateValue("argos_funders", (int) dataManagementPlanManager.countAllFunders(), null);
-        calculateValue("argos_grants", (int) dataManagementPlanManager.countAllGrants(), null);
-        calculateValue("argos_projects", (int) dataManagementPlanManager.countAllProjects(), null);
-        calculateValue("argos_researchers", (int) dataManagementPlanManager.countAllResearchers(), null);
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDraftDMPs(true), MetricNames.DRAFT);
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllFinalizedDMPs(true), MetricNames.FINALIZED);
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllPublishedDMPs(true), MetricNames.PUBLISHED);
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDoiedDMPs(true), MetricNames.DOIED);
 
-        calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllDraft(), "draft");
-        calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllFinalized(), "finalized");
-        calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllPublic(), "published");
-        calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllWithDoi(), "doied");
+        calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(), MetricNames.DRAFT);
+        calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(), MetricNames.FINALIZED);
+        calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(), MetricNames.PUBLISHED);
+        calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(), MetricNames.DOIED);
 
-        calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAlldraft(), "draft");
-        calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllFinalized(), "active");
-        calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllUsed(), "used");
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(true), MetricNames.DRAFT);
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(true), MetricNames.FINALIZED);
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(true), MetricNames.PUBLISHED);
+        calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(true), MetricNames.DOIED);
 
-        calculateValue("argos_users", (int) userManager.countActiveUsers().intValue(), "loggedin");
-        calculateValue("argos_users", (int) userManager.countAllUsers().intValue(), "total");
+        calculateValue(MetricNames.FUNDERS, (int) countAllFunders(), null);
+        calculateValue(MetricNames.GRANTS, (int) countAllGrants(), null);
+        calculateValue(MetricNames.PROJECT, (int) countAllProjects(), null);
+        calculateValue(MetricNames.RESEARCHER, (int) countAllResearchers(), null);
+
+        calculateValue(MetricNames.NEXUS + MetricNames.FUNDERS, (int) countAllFunders(true), null);
+        calculateValue(MetricNames.NEXUS + MetricNames.GRANTS, (int) countAllGrants(true), null);
+        calculateValue(MetricNames.NEXUS + MetricNames.PROJECT, (int) countAllProjects(true), null);
+        calculateValue(MetricNames.NEXUS + MetricNames.RESEARCHER, (int) countAllResearchers(true), null);
+
+        calculateValue(MetricNames.DATASET, (int) countAllDraftDatasets(), MetricNames.DRAFT);
+        calculateValue(MetricNames.DATASET, (int) countAllFinalizedDatasets(), MetricNames.FINALIZED);
+        calculateValue(MetricNames.DATASET, (int) countAllPublicDatasets(), MetricNames.PUBLISHED);
+        calculateValue(MetricNames.DATASET, (int) countAllDatasetsWithDoi(), MetricNames.DOIED);
+
+        calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDraftDatasets(true), MetricNames.DRAFT);
+        calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllFinalizedDatasets(true), MetricNames.FINALIZED);
+        calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllPublicDatasets(true), MetricNames.PUBLISHED);
+        calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDatasetsWithDoi(true), MetricNames.DOIED);
+
+        calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(), MetricNames.DRAFT);
+        calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(), MetricNames.ACTIVE);
+        calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(), MetricNames.USED);
+
+        calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(true), MetricNames.DRAFT);
+        calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(true), MetricNames.ACTIVE);
+        calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(true), MetricNames.USED);
+
+        calculateValue(MetricNames.USERS, (int) userManager.countActiveUsers().intValue(), MetricNames.LOGGEDIN);
+        calculateValue(MetricNames.USERS, (int) userManager.countAllUsers().intValue(), MetricNames.TOTAL);
 
         long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count();
-        calculateValue("argos_languages", (int) files, null);
+        calculateValue(MetricNames.LANGUAGES, (int) files, null);
+
+        calculateValue(MetricNames.INSTALLATIONS, 1, null);
+        calculateValue(MetricNames.NEXUS + MetricNames.INSTALLATIONS, 1, null);
 
         logger.info("Metrics calculation Completed");
     }
 
+    private Date getNexusDate() {
+        try {
+            return new SimpleDateFormat("yyyy-MM-dd").parse("2021-01-01");
+        } catch (ParseException e) {
+            logger.error(e.getLocalizedMessage(), e);
+        }
+        return Date.from(LocalDate.of(2021, 1, 1).atStartOfDay(ZoneId.systemDefault()).toInstant());
+    }
+
+    private long countAllDraftDMPs(){
+        return countAllDraftDMPs(false);
+    }
+
+    private long countAllDraftDMPs(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setStatus(0);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
+    }
+
+    private long countAllFinalizedDMPs() {
+        return countAllFinalizedDMPs(false);
+    }
+
+    private long countAllFinalizedDMPs(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setStatus(1);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
+    }
+
+    private long countAllPublishedDMPs() {
+        return countAllPublishedDMPs(false);
+    }
+
+    private long countAllPublishedDMPs(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setIsPublic(true);
+        criteria.setOnlyPublic(true);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
+    }
+
+    private long countAllDoiedDMPs() {
+        return countAllDoiedDMPs(false);
+    }
+
+    private long countAllDoiedDMPs(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setHasDoi(true);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
+    }
+
+    private long countAllDraftDMPsWithGrantId() {
+        return countAllDraftDMPsWithGrantId(false);
+    }
+
+    private long countAllDraftDMPsWithGrantId(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setStatus(0);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
+    }
+
+    private long countAllFinalizedDMPsWithGrantId() {
+        return countAllFinalizedDMPsWithGrantId(false);
+    }
+
+    private long countAllFinalizedDMPsWithGrantId(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setStatus(1);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
+    }
+
+    private long countAllPublishedDMPsWithGrantId() {
+        return countAllPublishedDMPsWithGrantId(false);
+    }
+
+    private long countAllPublishedDMPsWithGrantId(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setIsPublic(true);
+        criteria.setOnlyPublic(true);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
+    }
+
+    private long countAllDoiedDMPsWithGrantId() {
+        return countAllDoiedDMPsWithGrantId(false);
+    }
+
+    private long countAllDoiedDMPsWithGrantId(boolean countNexus) {
+        DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
+        criteria.setHasDoi(true);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
+    }
+
+    private long countAllResearchers() {
+        return countAllResearchers(false);
+    }
+
+    private long countAllResearchers(boolean countNexus) {
+        ResearcherCriteria criteria = new ResearcherCriteria();
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
+    }
+
+    private long countAllProjects() {
+        return countAllProjects(false);
+    }
+
+    private long countAllProjects(boolean countNexus) {
+        ProjectCriteria criteria = new ProjectCriteria();
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
+    }
+
+    private long countAllFunders() {
+        return countAllFunders(false);
+    }
+
+    private long countAllFunders(boolean countNexus) {
+        FunderCriteria criteria = new FunderCriteria();
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
+    }
+
+    private long countAllGrants() {
+        return countAllGrants(false);
+    }
+
+    private long countAllGrants(boolean countNexus) {
+        GrantCriteria criteria = new GrantCriteria();
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
+    }
+
+    public long countAllDraftDatasets() {
+        return countAllDraftDatasets(false);
+    }
+
+    public long countAllDraftDatasets(boolean countNexus) {
+        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
+        criteria.setStatus(0);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
+    }
+
+    public long countAllFinalizedDatasets() {
+        return countAllFinalizedDatasets(false);
+    }
+
+    public long countAllFinalizedDatasets(boolean countNexus) {
+        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
+        criteria.setStatus(1);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
+    }
+
+    public long countAllPublicDatasets() {
+        return countAllPublicDatasets(false);
+    }
+
+    public long countAllPublicDatasets(boolean countNexus) {
+        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
+        criteria.setIsPublic(true);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
+    }
+
+    public long countAllDatasetsWithDoi() {
+        return countAllDatasetsWithDoi(false);
+    }
+
+    public long countAllDatasetsWithDoi(boolean countNexus) {
+        eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
+        criteria.setHasDoi(true);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
+    }
+
+    public long countAllDraftTemplates() {
+        return countAllDraftTemplates(false);
+    }
+
+    public long countAllDraftTemplates(boolean countNexus) {
+        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
+        criteria.setStatus(0);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
+    }
+
+    public long countAllFinalizedTemplates() {
+        return countAllFinalizedTemplates(false);
+    }
+
+    public long countAllFinalizedTemplates(boolean countNexus) {
+        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
+        criteria.setStatus(1);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
+    }
+
+    @Transactional
+    public long countAllUsedTemplates() {
+        return countAllUsedTemplates(false);
+    }
+
+    @Transactional
+    public long countAllUsedTemplates(boolean countNexus) {
+        DatasetProfileCriteria criteria = new DatasetProfileCriteria();
+        criteria.setStatus(1);
+        criteria.setAllVersions(false);
+        if (countNexus) criteria.setPeriodStart(getNexusDate());
+        List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
+        List<DatasetProfile> filteredProfiles = new ArrayList<>();
+        for (DatasetProfile datasetProfile : datasetProfiles) {
+            DatasetCriteria datasetCriteria = new DatasetCriteria();
+            datasetCriteria.setDatasetTemplates(Collections.singletonList(datasetProfile.getId()));
+            Long datasetCount = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).count();
+            if (datasetCount > 0) {
+                filteredProfiles.add(datasetProfile);
+            }
+        }
+        return filteredProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).count();
+    }
+
 }

dmp-backend/web/src/main/java/eu/eudat/logic/managers/RDAManager.java

@@ -1,6 +1,5 @@
 package eu.eudat.logic.managers;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.data.entities.DMP;
@@ -12,7 +11,6 @@ import org.springframework.stereotype.Component;
 
 import javax.transaction.Transactional;
 import java.io.IOException;
-import java.io.Serializable;
 import java.text.SimpleDateFormat;
 
 @Component
@@ -43,7 +41,6 @@ public class RDAManager {
 
 	public DMP convertToEntity(String json, String[] profiles) throws IOException {
 		ObjectMapper mapper = new ObjectMapper();
-		mapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"));
 
 		Dmp rda = mapper.readValue(json, RDAModel.class).getDmp();
 		return dmpRDAMapper.toEntity(rda, profiles);

dmp-backend/web/src/main/java/eu/eudat/logic/managers/UserManager.java

@@ -167,7 +167,7 @@ public class UserManager {
 			try {
 				this.updateDOIToken(ZenodoAccessType.REFRESH_TOKEN, principal.getZenodoRefresh(), this.environment.getProperty("zenodo.login.redirect_uri"), principal);
 				return true;
-			}catch (IOException e) {
+			}catch (Exception e) {
 				this.deleteDOIToken(principal);
 				throw new ExpiredTokenException("Zenodo Token is expired.");
 			}

dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/DatasetMapper.java

@@ -31,6 +31,7 @@ public class DatasetMapper {
 		if (tags != null && !tags.isEmpty()) {
 			DatasetCriteria criteria = new DatasetCriteria();
 			criteria.setTags(tags);
+			criteria.setHasTags(true);
 			List<Tag> tags1 = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
 									   .filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> tags.stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
 			if (tags1.isEmpty()) {

dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/DmpMapper.java

@@ -67,6 +67,7 @@ public class DmpMapper {
 					if (dataset1 != null) {
 						tags = dataset1.getTags();
 					}
+					dataset.setDmp(dmp);
 					return datasetMapper.toElastic(dataset, tags);
 				} catch (Exception e) {
 					logger.error(e.getMessage(), e);

dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/criteria/DmpCriteriaMapper.java

@@ -45,7 +45,7 @@ public class DmpCriteriaMapper {
 
 	public static List<SortCriteria> toElasticSorting(ColumnOrderings columnOrderings) {
 		List<SortCriteria> sortCriteria = new ArrayList<>();
-		if (columnOrderings.getFieldOrderings() != null && columnOrderings.getFieldOrderings().length > 0) {
+		if (columnOrderings.getFieldOrderings() != null && !columnOrderings.getFieldOrderings().isEmpty()) {
 			for (Ordering ordering: columnOrderings.getFieldOrderings()) {
 				SortCriteria sortCriteria1 = new SortCriteria();
 				sortCriteria1.setFieldName(ordering.getFieldName() + (ordering.getFieldName().contains("label") ?".keyword" : ""));

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/ExternalUrlCriteria.java

@@ -58,4 +58,16 @@ public class ExternalUrlCriteria {
 
 	public ExternalUrlCriteria() {
 	}
+
+	@Override
+	public String toString() {
+		return "{" +
+				"like='" + like + '\'' +
+				", page='" + page + '\'' +
+				", pageSize='" + pageSize + '\'' +
+				", funderId='" + funderId + '\'' +
+				", path='" + path + '\'' +
+				", host='" + host + '\'' +
+				'}';
+	}
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/configloaders/DefaultConfigLoader.java

@@ -8,7 +8,6 @@ import org.apache.poi.xwpf.usermodel.XWPFDocument;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Profile;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Document;
@@ -26,16 +25,13 @@ import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 import java.io.*;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 @Service("configLoader")
-@Profile("devel")
-public class DevelConfigLoader implements ConfigLoader {
-    private static final Logger logger = LoggerFactory.getLogger(DevelConfigLoader.class);
+//@Profile("devel")
+public class DefaultConfigLoader implements ConfigLoader {
+    private static final Logger logger = LoggerFactory.getLogger(DefaultConfigLoader.class);
 
     private ExternalUrls externalUrls;
     private List<String> rdaProperties;
@@ -53,10 +49,10 @@ public class DevelConfigLoader implements ConfigLoader {
         try {
             JAXBContext jaxbContext = JAXBContext.newInstance(ExternalUrls.class);
             Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
-            is = getClass().getClassLoader().getResource(fileUrl).openStream();
+            is = getStreamFromPath(fileUrl);
             externalUrls = (ExternalUrls) jaxbUnmarshaller.unmarshal(is);
         } catch (Exception ex) {
-           logger.error("Cannot find resource in classpath", ex);
+           logger.error("Cannot find resource", ex);
         } finally {
             try {
                 if (is != null) is.close();
@@ -68,10 +64,11 @@ public class DevelConfigLoader implements ConfigLoader {
 
     private void setRdaProperties() {
         String filePath = environment.getProperty("configuration.rda");
+        logger.info("Loaded also config file: " + filePath);
         BufferedReader reader;
         List<String> rdaList = new LinkedList<>();
         try {
-            reader = new BufferedReader(new FileReader(getClass().getClassLoader().getResource(filePath).getFile()));
+            reader = new BufferedReader(new InputStreamReader(getStreamFromPath(filePath)));
             String line = reader.readLine();
             while (line != null) {
                 rdaList.add(line);
@@ -87,9 +84,10 @@ public class DevelConfigLoader implements ConfigLoader {
 
     private void setDocument() {
         String filePath = environment.getProperty("configuration.h2020template");
+        logger.info("Loaded also config file: " + filePath);
         InputStream is = null;
         try {
-            is = getClass().getClassLoader().getResource(filePath).openStream();
+            is = getStreamFromPath(filePath);
             this.document = new XWPFDocument(is);
         } catch (IOException | NullPointerException e) {
             logger.error(e.getMessage(), e);
@@ -104,9 +102,10 @@ public class DevelConfigLoader implements ConfigLoader {
 
     private void setConfigurableProviders() {
         String filePath = environment.getProperty("configuration.configurable_login_providers");
+        logger.info("Loaded also config file: " + filePath);
         InputStream is = null;
         try {
-            is = getClass().getClassLoader().getResource(filePath).openStream();
+            is = getStreamFromPath(filePath);
             ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
             this.configurableProviders = mapper.readValue(is, ConfigurableProviders.class);
         } catch (IOException | NullPointerException e) {
@@ -145,12 +144,34 @@ public class DevelConfigLoader implements ConfigLoader {
 
 
     public ExternalUrls getExternalUrls() {
+        if (externalUrls == null) {
+            externalUrls = new ExternalUrls();
             this.setExternalUrls();
+        } else {
+            while (externalUrls.getMaxresults() == null) {
+                try {
+                    Thread.sleep(100L);
+                } catch (InterruptedException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
         return externalUrls;
     }
 
     public List<String> getRdaProperties() {
+        if (rdaProperties == null) {
+            rdaProperties = new ArrayList<>();
             this.setRdaProperties();
+        } else {
+            while (rdaProperties.size() == 0) {
+                try {
+                    Thread.sleep(100L);
+                } catch (InterruptedException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
         return rdaProperties;
     }
 
@@ -160,12 +181,34 @@ public class DevelConfigLoader implements ConfigLoader {
     }
 
     public ConfigurableProviders getConfigurableProviders() {
+        if (configurableProviders == null) {
+            configurableProviders = new ConfigurableProviders();
             this.setConfigurableProviders();
+        } else {
+            while (configurableProviders.getProviders().size() == 0) {
+                try {
+                    Thread.sleep(100L);
+                } catch (InterruptedException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
         return configurableProviders;
     }
 
     public Map<String, String> getKeyToSourceMap() {
+        if (keyToSourceMap == null) {
+            keyToSourceMap = new HashMap<>();
             this.setKeyToSourceMap();
+        } else {
+            while (keyToSourceMap.size() == 0) {
+                try {
+                    Thread.sleep(100L);
+                } catch (InterruptedException e) {
+                    logger.error(e.getMessage(), e);
+                }
+            }
+        }
         return keyToSourceMap;
     }
 
@@ -173,7 +216,7 @@ public class DevelConfigLoader implements ConfigLoader {
         InputStream is = null;
         Document doc;
         try {
-            is = getClass().getClassLoader().getResource(filePath).openStream();
+            is = getStreamFromPath(filePath);
             DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
             DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
             doc = documentBuilder.parse(is);
@@ -211,4 +254,13 @@ public class DevelConfigLoader implements ConfigLoader {
         }
         return values;
     }
+
+    private InputStream getStreamFromPath(String filePath) {
+        try {
+            return new FileInputStream(filePath);
+        } catch (FileNotFoundException e) {
+            logger.info("loading from classpath");
+            return getClass().getClassLoader().getResourceAsStream(filePath);
+        }
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/configloaders/ProductionConfigLoader.java

@@ -33,6 +33,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+/*
 @Service("configLoader")
 @Profile({ "production", "staging" })
 public class ProductionConfigLoader implements ConfigLoader {
@@ -236,4 +237,4 @@ public class ProductionConfigLoader implements ConfigLoader {
         }
         return values;
     }
-}
+}*/

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/LocalFetcher.java

@@ -8,7 +8,6 @@ import org.springframework.stereotype.Component;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Unmarshaller;
-import java.beans.PropertyDescriptor;
 import java.io.InputStream;
 import java.lang.reflect.Method;
 import java.util.*;
@@ -22,6 +21,7 @@ public class LocalFetcher {
 		JAXBContext context = JAXBContext.newInstance(Config.class);
 		Unmarshaller unmarshaller = context.createUnmarshaller();
 		Config config = (Config) unmarshaller.unmarshal(is);
+		is.close();
 		ConfigSingle currencyConfig = config.getConfigs().stream().filter(configSingle -> configSingle.getType().equals("currency")).findFirst().get();
 
 		return retrieveData(currencyConfig);
@@ -39,10 +39,11 @@ public class LocalFetcher {
 				Unmarshaller unmarshaller = context.createUnmarshaller();
 
 				Object object = unmarshaller.unmarshal(is);
-
+				is.close();
 				Method reader = null;
 				if (configSingle.getParseField() != null && !configSingle.getParseField().isEmpty()) {
-					reader = new PropertyDescriptor(configSingle.getParseField(), aClass).getReadMethod();
+					String camelCaseGetter =configSingle.getParseField() != null && !configSingle.getParseField().isEmpty() ? "get" + configSingle.getParseField().substring(0, 1).toUpperCase() + configSingle.getParseField().substring(1) : "";
+					reader = aClass.getMethod(camelCaseGetter);
 				}
 				ObjectMapper objectMapper = new ObjectMapper();
 				List<Map<String, String>> values = new ArrayList<>();

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java

@@ -12,29 +12,20 @@ import eu.eudat.logic.proxy.config.UrlConfiguration;
 import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
-import net.minidev.json.JSONArray;
+import eu.eudat.logic.proxy.fetching.entities.Results;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.cache.annotation.Cacheable;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.HttpStatus;
+import org.springframework.http.*;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Unmarshaller;
-import java.beans.PropertyDescriptor;
-import java.io.*;
+import java.io.File;
+import java.io.StringReader;
 import java.lang.reflect.Method;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLEncoder;
 import java.nio.file.Paths;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -50,7 +41,7 @@ public class RemoteFetcher {
         this.configLoader = configLoader;
     }
 
-    @Cacheable("repositories")
+    @Cacheable(value = "repositories", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@@ -59,28 +50,28 @@ public class RemoteFetcher {
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("grants")
+    @Cacheable(value = "grants", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getGrants().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode();
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("projects")
+    @Cacheable(value = "projects", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("funders")
+    @Cacheable(value = "funders", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
         FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode();
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("organisations")
+    @Cacheable(value = "organisations", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@@ -89,7 +80,7 @@ public class RemoteFetcher {
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("registries")
+    @Cacheable(value = "registries", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@@ -98,7 +89,7 @@ public class RemoteFetcher {
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("services")
+    @Cacheable(value = "services", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@@ -107,7 +98,7 @@ public class RemoteFetcher {
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("researchers")
+    @Cacheable(value = "researchers", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@@ -125,7 +116,7 @@ public class RemoteFetcher {
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }*/
 
-    @Cacheable("externalDatasets")
+    @Cacheable(value = "externalDatasets", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@@ -134,7 +125,7 @@ public class RemoteFetcher {
         return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
     }
 
-    @Cacheable("licenses")
+    @Cacheable(value = "licenses", keyGenerator = "externalUrlsKeyGenerator")
     public List<Map<String, String>> getlicenses(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs =
                 key != null && !key.isEmpty() ? configLoader.getExternalUrls().getLicenses().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
@@ -153,22 +144,36 @@ public class RemoteFetcher {
     }
 
 
-    private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
+    private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) {
 
         List<Map<String, String>> results = new LinkedList<>();
 
-        if (urlConfigs == null || urlConfigs.isEmpty()) return results;
+        if (urlConfigs == null || urlConfigs.isEmpty()) {
+            return results;
+        }
 //            throw new NoURLFound("No Repository urls found in configuration");
 
         urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
-        for (UrlConfiguration urlConfig : urlConfigs) {
+        urlConfigs.forEach(urlConfiguration -> {
+            ifFunderQueryExist(urlConfiguration, externalUrlCriteria);
+            if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
+                try {
+                    results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalUrlCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType()));
+                } catch (Exception e) {
+                    logger.error(e.getLocalizedMessage(), e);
+                }
+            } else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) {
+                results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalUrlCriteria.getLike()));
+            }
+        });
+       /* for (UrlConfiguration urlConfig : urlConfigs) {
             ifFunderQueryExist(urlConfig, externalUrlCriteria);
             if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
                 results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
             } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
                 results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
             }
-        }
+        }*/
         return results;
     }
 
@@ -183,7 +188,7 @@ public class RemoteFetcher {
         }
     }
 
-    private String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria, String firstPage) {
+    protected String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria, String firstPage) {
         String completedPath = path;
         if (externalUrlCriteria.getLike() != null) {
             if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror")) && externalUrlCriteria.getLike().equals("")) {
@@ -233,7 +238,7 @@ public class RemoteFetcher {
         return completedPath;
     }
 
-    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws HugeResultSet {
+    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws Exception {
         Set<Integer> pages = new HashSet<>();
 
         String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
@@ -253,6 +258,7 @@ public class RemoteFetcher {
 
         Optional<Results> optionalResults = pages.parallelStream()
                 .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType))
+                .filter(Objects::nonNull)
                 .reduce((result1, result2) -> {
                     result1.getResults().addAll(result2.getResults());
                     return result1;
@@ -264,7 +270,7 @@ public class RemoteFetcher {
     }
 
 
-    private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) {
+    protected Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) {
 
         try {
         	RestTemplate restTemplate = new RestTemplate();
@@ -292,103 +298,23 @@ public class RemoteFetcher {
                     DocumentContext jsonContext = JsonPath.parse(response.getBody());
 
                     if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) {
-                        results = new Results(jsonContext.read(jsonDataPath.getPath()
-                                + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
-                                + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
-                                + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
-                                new HashMap<>(1, 1));
+                        results = RemoteFetcherUtils.getFromJsonWithSource(jsonContext, jsonDataPath);
                     } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu
-                        results = new Results(jsonContext.read(jsonDataPath.getPath()
-                                + "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
-                                + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
-                                new HashMap<>(1, 1));
-                        List<Map<String, String>> fixedResults = results.getResults().stream().map(item -> {
-                            for (int i = 0; i < 2; i++) {
-                                String id;
-                                if (i == 0) {
-                                    id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", "");
-                                } else {
-                                    id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", "");
-                                }
-                                if (!(item.get(id) instanceof String)) {
-                                    Object obj = item.get(id);
-                                    if (obj instanceof JSONArray) {
-                                        JSONArray jarr = (JSONArray) obj;
-                                        if (jarr.get(0) instanceof String) {
-                                            item.put(id, jarr.get(0).toString());
-                                        } else {
-                                            for (int j = 0; j < jarr.size(); j++) {
-                                                mapToMap(id, (Map<String, String>)jarr.get(j), item, i == 1);
-                                            }
-                                        }
-                                    } else {
-                                        if (obj instanceof Map) {
-                                            mapToMap(id, (Map<String, String>) obj, item, i == 1);
-                                        } else if (obj != null){
-                                            item.put(id, obj.toString());
-                                        }
-                                    }
-                                }
-                            }
-                            return item;
-                        }).collect(Collectors.toList());
-
-                        results = new Results(fixedResults, new HashMap<>(1, 1));
+                        results = RemoteFetcherUtils.getFromJsonWithParsedData(jsonContext, jsonDataPath);
                     } else if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
-                        results = new Results(jsonContext.read(jsonDataPath.getPath()
-                                + "[" + jsonDataPath.getFieldsUrlConfiguration().getPath()
-                                + "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"),
-                                new HashMap<>(1, 1));
-
-                        List<Map<String, String>> multiResults = results.results.stream().map(result -> {
-                            ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
-                            externalUrlCriteria.setPath(result.get("path"));
-                            externalUrlCriteria.setHost(result.get("host"));
-                            String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
-                            return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType);
-                        }).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList());
-                        results = new Results(multiResults, new HashMap<>(1, 1));
+                        results = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType);
                     } else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) {
-                        List<Map<String, Object>> tempRes = jsonContext.read(jsonDataPath.getPath()
-                                + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName()
-                                + "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]");
-                        List<Map<String, String>> finalRes = new ArrayList<>();
-                        tempRes.forEach(map -> {
-                            Map<String, String> resMap = new HashMap<>();
-                            map.forEach((key, value) -> {
-                                if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
-                                    resMap.put("tags", ((JSONArray) value).toJSONString());
-                                } else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
-                                    resMap.put(key, ((JSONArray) value).toJSONString());
-                                } else {
-                                    resMap.put(key, (String) value);
-                                }
-                            });
-
-                            finalRes.add(resMap);
-                        });
-
-                        results = new Results(finalRes,
-                                new HashMap<>(1, 1));
+                        results = RemoteFetcherUtils.getFromJsonWithType(jsonContext, jsonDataPath);
                     } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
-                        results = new Results(jsonContext.read(jsonDataPath.getPath()
-                                + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName()
-                                + "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"),
-                                new HashMap<>(1, 1));
-                        results.getResults().stream().forEach(entry -> {
-                            String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
-                            entry.put("name", name);
-                            entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
-                            entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
-                        });
+                        results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
                     } else {
                         results = new Results(jsonContext.read(jsonDataPath.getPath()
                                 + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
                                 + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
                                 new HashMap<>(1, 1));
                     }
-                    results.results = results.results.stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
-                                                     .collect(Collectors.toList());
+                    results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
+                                                     .collect(Collectors.toList()));
                 }
                 else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) {
                     Class<?> aClass = Class.forName(jsonDataPath.getParseClass());
@@ -398,7 +324,8 @@ public class RemoteFetcher {
                     Object data = unmarshaller.unmarshal(stringReader);
                     Method reader = null;
                     if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) {
-                        reader = new PropertyDescriptor(jsonDataPath.getParseField(), aClass).getReadMethod();
+                        String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : "";
+                        reader = aClass.getMethod(camelCaseGetter);
                     }
                     ObjectMapper objectMapper = new ObjectMapper();
                     List<Map<String, String>> values = new ArrayList<>();
@@ -448,8 +375,6 @@ public class RemoteFetcher {
         } catch (Exception exception) {
             logger.error(exception.getMessage(), exception);
         } //maybe print smth...
-        finally {
-        }
 
         return null;
     }
@@ -459,7 +384,7 @@ public class RemoteFetcher {
         try {
             String filePath = Paths.get(path).toUri().toURL().toString();
             ObjectMapper mapper = new ObjectMapper();
-            internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, Object>>>(){});
+            internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
             return searchListMap(internalResults, query);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
@@ -494,48 +419,7 @@ public class RemoteFetcher {
     }
 
 
-    static class Results {
-        List<Map<String, String>> results;
-        Map<String, Integer> pagination;
 
-        Results() {
-            this.results = new ArrayList<>();
-            this.pagination = new HashMap<>();
-        }
 
-        Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
-            this.results = results;
-            this.pagination = pagination;
-        }
 
-        List<Map<String, String>> getResults() {
-            return results;
-        }
-
-        public void setResults(List<Map<String, String>> results) {
-            this.results = results;
-        }
-
-        Map<String, Integer> getPagination() {
-            return pagination;
-        }
-
-        public void setPagination(Map<String, Integer> pagination) {
-            this.pagination = pagination;
-        }
-    }
-
-    private void mapToMap(String key, Map<String, String> source, Map<String, String> destination, boolean isTitle) {
-        if (source != null) {
-            String content = source.get("content");
-            if (isTitle) {
-                String classId = source.get("classid");
-                if (classId.equals("main title")) {
-                    destination.put(key, content);
-                }
-            } else {
-                destination.put(key, content);
-            }
-        }
-    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java

@@ -0,0 +1,128 @@
+package eu.eudat.logic.proxy.fetching;
+
+import com.jayway.jsonpath.DocumentContext;
+import eu.eudat.logic.proxy.config.DataUrlConfiguration;
+import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
+import eu.eudat.logic.proxy.fetching.entities.Results;
+import net.minidev.json.JSONArray;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class RemoteFetcherUtils {
+
+    public static Results getFromJsonWithSource(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
+        return new Results(jsonContext.read(jsonDataPath.getPath()
+                + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
+                + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
+                + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
+                new HashMap<>(1, 1));
+    }
+
+    public static Results getFromJsonWithParsedData(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
+        Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+                + "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+                + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
+                new HashMap<>(1, 1));
+        List<Map<String, String>> fixedResults = results.getResults().stream().map(item -> {
+            for (int i = 0; i < 2; i++) {
+                String id;
+                if (i == 0) {
+                    id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", "");
+                } else {
+                    id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", "");
+                }
+                if (!(item.get(id) instanceof String)) {
+                    Object obj = item.get(id);
+                    if (obj instanceof JSONArray) {
+                        JSONArray jarr = (JSONArray) obj;
+                        if (jarr.get(0) instanceof String) {
+                            item.put(id, jarr.get(0).toString());
+                        } else {
+                            for (int j = 0; j < jarr.size(); j++) {
+                                mapToMap(id, (Map<String, String>)jarr.get(j), item, i == 1);
+                            }
+                        }
+                    } else {
+                        if (obj instanceof Map) {
+                            mapToMap(id, (Map<String, String>) obj, item, i == 1);
+                        } else if (obj != null){
+                            item.put(id, obj.toString());
+                        }
+                    }
+                }
+            }
+            return item;
+        }).collect(Collectors.toList());
+
+        return new Results(fixedResults, new HashMap<>(1, 1));
+    }
+
+    public static Results getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcher remoteFetcher, String requestBody, String requestType) {
+        Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+                + "[" + jsonDataPath.getFieldsUrlConfiguration().getPath()
+                + "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"),
+                new HashMap<>(1, 1));
+
+        List<Map<String, String>> multiResults = results.getResults().stream().map(result -> {
+            ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
+            externalUrlCriteria.setPath(result.get("path"));
+            externalUrlCriteria.setHost(result.get("host"));
+            String replacedPath = remoteFetcher.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
+            return remoteFetcher.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType);
+        }).filter(Objects::nonNull).map(results1 -> results1.getResults().get(0)).collect(Collectors.toList());
+        return new Results(multiResults, new HashMap<>(1, 1));
+    }
+
+    public static Results getFromJsonWithType(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
+        List<Map<String, Object>> tempRes = jsonContext.read(jsonDataPath.getPath()
+                + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName()
+                + "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]");
+        List<Map<String, String>> finalRes = new ArrayList<>();
+        tempRes.forEach(map -> {
+            Map<String, String> resMap = new HashMap<>();
+            map.forEach((key, value) -> {
+                if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
+                    resMap.put("tags", ((JSONArray) value).toJSONString());
+                } else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
+                    resMap.put(key, ((JSONArray) value).toJSONString());
+                } else {
+                    resMap.put(key, (String) value);
+                }
+            });
+
+            finalRes.add(resMap);
+        });
+
+        return new Results(finalRes,
+                new HashMap<>(1, 1));
+    }
+
+    public static Results getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
+        Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+                + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName()
+                + "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"),
+                new HashMap<>(1, 1));
+        results.getResults().stream().forEach(entry -> {
+            String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
+            entry.put("name", name);
+            entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
+            entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
+        });
+        return results;
+    }
+
+    private static void mapToMap(String key, Map<String, String> source, Map<String, String> destination, boolean isTitle) {
+        if (source != null) {
+            String content = source.get("content");
+            /*if (isTitle) {
+                String classId = source.get("classid");
+                if (classId.equals("main title")) {
+                    destination.put(key, content);
+                }
+            } else {*/
+            destination.put(key, content);
+//            }
+        }
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/entities/Results.java

@@ -0,0 +1,37 @@
+package eu.eudat.logic.proxy.fetching.entities;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Results {
+    List<Map<String, String>> results;
+    Map<String, Integer> pagination;
+
+    public Results() {
+        this.results = new ArrayList<>();
+        this.pagination = new HashMap<>();
+    }
+
+    public Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
+        this.results = results;
+        this.pagination = pagination;
+    }
+
+    public List<Map<String, String>> getResults() {
+        return results;
+    }
+
+    public void setResults(List<Map<String, String>> results) {
+        this.results = results;
+    }
+
+    public Map<String, Integer> getPagination() {
+        return pagination;
+    }
+
+    public void setPagination(Map<String, Integer> pagination) {
+        this.pagination = pagination;
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/logic/services/forms/VisibilityRuleService.java

@@ -9,7 +9,7 @@ import java.util.Map;
  * Created by ikalyvas on 3/5/2018.
  */
 public interface VisibilityRuleService {
-    Boolean isElementVisible(String id);
+    boolean isElementVisible(String id);
 
     void buildVisibilityContext(List<Rule> sources);
 

dmp-backend/web/src/main/java/eu/eudat/logic/services/forms/VisibilityRuleServiceImpl.java

@@ -3,9 +3,7 @@ package eu.eudat.logic.services.forms;
 import eu.eudat.models.data.user.components.commons.Rule;
 import org.springframework.stereotype.Service;
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * Created by ikalyvas on 3/5/2018.
@@ -16,13 +14,22 @@ public class VisibilityRuleServiceImpl implements VisibilityRuleService {
     private VisibilityContext visibilityContext;
     private Map<String, Object> properties;
 
-    public Boolean isElementVisible(String id) {
+    public boolean isElementVisible(String id) {
         if (!this.elementVisibility.containsKey(id) || this.elementVisibility.get(id)) return true;
         return false;
     }
 
     public void setProperties(Map<String, Object> properties) {
         this.properties = properties;
+        this.properties.entrySet().stream()
+                .filter(stringObjectEntry -> stringObjectEntry.getValue() instanceof String && ((String) stringObjectEntry.getValue()).startsWith("[") && ((String) stringObjectEntry.getValue()).endsWith("]")).forEach(stringObjectEntry -> {
+            stringObjectEntry.setValue(parseArray((String) stringObjectEntry.getValue()));
+        });
+    }
+
+    private List<String> parseArray(String original) {
+        String parsed = original.replace("[", "").replace("\"", "").replace("]", "");
+        return Arrays.asList(parsed.split(","));
     }
 
     public void buildVisibilityContext(List<Rule> sources) {
@@ -34,12 +41,19 @@ public class VisibilityRuleServiceImpl implements VisibilityRuleService {
     private void evaluateVisibility(VisibilityRule rule) {
         List<VisibilityRuleSource> sources = rule.getVisibilityRuleSources();
         for(int i = 0; i < sources.size(); i++){
-            if (!properties.containsKey(sources.get(i).getVisibilityRuleSourceId()) || !properties.get(sources.get(i).getVisibilityRuleSourceId()).equals(sources.get(i).getVisibilityRuleSourceValue())) {
+            if (properties.containsKey(sources.get(i).getVisibilityRuleSourceId()) && (isContained(properties.get(sources.get(i).getVisibilityRuleSourceId()), sources.get(i).getVisibilityRuleSourceValue()) || properties.get(sources.get(i).getVisibilityRuleSourceId()).equals(sources.get(i).getVisibilityRuleSourceValue()))) {
+                this.elementVisibility.put(rule.getVisibilityRuleTargetId(), true);
+            }else{
                 this.elementVisibility.put(rule.getVisibilityRuleTargetId(), false);
                 return;
-            }else{
-                this.elementVisibility.put(rule.getVisibilityRuleTargetId(), true);
             }
         }
     }
+
+    private Boolean isContained(Object values, String source) {
+        if (values instanceof List) {
+            return ((Collection<?>) values).contains(source);
+        }
+        return false;
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/services/operations/authentication/AbstractAuthenticationService.java

@@ -100,7 +100,7 @@ public abstract class AbstractAuthenticationService implements AuthenticationSer
 
 		UserToken userToken = this.apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserTokenBuilder.class)
 				.issuedAt(new Date()).user(credential.getUserInfo())
-				.token(UUID.randomUUID()).expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(1)))
+				.token(UUID.randomUUID()).expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(10)))
 				.build();
 
 		userToken = apiContext.getOperationsContext().getDatabaseRepository().getUserTokenDao().createOrUpdate(userToken);
@@ -200,7 +200,7 @@ public abstract class AbstractAuthenticationService implements AuthenticationSer
 
 		UserToken userToken = this.apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserTokenBuilder.class)
 				.token(UUID.randomUUID()).user(userInfo)
-				.expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(1))).issuedAt(new Date())
+				.expiresAt(Timestamp.valueOf(LocalDateTime.now().plusDays(10))).issuedAt(new Date())
 				.build();
 
 		apiContext.getOperationsContext().getDatabaseRepository().getUserTokenDao().createOrUpdate(userToken);

dmp-backend/web/src/main/java/eu/eudat/logic/services/utilities/ConfirmationEmailServiceImpl.java

@@ -61,7 +61,7 @@ public class ConfirmationEmailServiceImpl implements ConfirmationEmailService {
 	}
 
 	private String createContent(UUID confirmationToken, MailService mailService) {
-		String content = mailService.getMailTemplateContent("classpath:templates/email/emailConfirmation.html");
+		String content = mailService.getMailTemplateContent(this.environment.getProperty("email.confirmation"));
 		content = content.replace("{confirmationToken}", confirmationToken.toString());
 		content = content.replace("{expiration_time}", secondsToTime(Integer.parseInt(this.environment.getProperty("conf_email.expiration_time_seconds"))));
 		content = content.replace("{host}", this.environment.getProperty("dmp.domain"));
@@ -85,7 +85,7 @@ public class ConfirmationEmailServiceImpl implements ConfirmationEmailService {
 	}
 
 	private String createMergeContent(UUID confirmationToken, MailService mailService, String userName) {
-		String content = mailService.getMailTemplateContent("classpath:templates/email/emailMergeConfirmation.html");
+		String content = mailService.getMailTemplateContent(this.environment.getProperty("email.merge"));
 		content = content.replace("{userName}", userName);
 		content = content.replace("{confirmationToken}", confirmationToken.toString());
 		content = content.replace("{expiration_time}", secondsToTime(Integer.parseInt(this.environment.getProperty("conf_email.expiration_time_seconds"))));

dmp-backend/web/src/main/java/eu/eudat/logic/services/utilities/InvitationServiceImpl.java

@@ -80,7 +80,7 @@ public class InvitationServiceImpl implements InvitationService {
         return CompletableFuture.runAsync(() -> {
             SimpleMail mail = new SimpleMail();
             mail.setSubject(createSubject(dmp, mailService.getMailTemplateSubject()));
-            mail.setContent(createContent(invitation.getId(), dmp, recipient, mailService.getMailTemplateContent("classpath:templates/email/email.html"), role));
+            mail.setContent(createContent(invitation.getId(), dmp, recipient, mailService.getMailTemplateContent(this.environment.getProperty("email.invite")), role));
             mail.setTo(invitation.getInvitationEmail());
             try {
                 mailService.sendSimpleMail(mail);

dmp-backend/web/src/main/java/eu/eudat/logic/services/utilities/MailServiceImpl.java

@@ -86,6 +86,7 @@ public class MailServiceImpl implements MailService {
             InputStream inputStream = resource.getInputStream();
             StringWriter writer = new StringWriter();
             IOUtils.copy(inputStream, writer, "UTF-8");
+            inputStream.close();
             return writer.toString();
         } catch (IOException e) {
             logger.error(e.getMessage(), e);

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/builders/ModelBuilder.java

@@ -84,6 +84,7 @@ public class ModelBuilder {
         if (type.equals("checkBox")) return (FieldData<U>) new CheckBoxData().fromData(data);
         if (type.equals("freetext")) return (FieldData<U>) new FreeTextData().fromData(data);
         if (type.equals("textarea")) return (FieldData<U>) new TextAreaData().fromData(data);
+        if (type.equals("richTextarea")) return (FieldData<U>) new RichTextAreaData().fromData(data);
         if (type.equals("datePicker")) return (FieldData<U>) new DatePickerData().fromData(data);
         if (type.equals("externalDatasets")) return (FieldData<U>) new ExternalDatasetsData().fromData(data);
         if (type.equals("dataRepositories")) return (FieldData<U>) new DataRepositoriesData().fromData(data);
@@ -123,6 +124,7 @@ public class ModelBuilder {
         if (type.equals("checkBox")) return (FieldData<U>) new CheckBoxData().fromData(data);
         if (type.equals("freetext")) return (FieldData<U>) new FreeTextData().fromData(data);
         if (type.equals("textarea")) return (FieldData<U>) new TextAreaData().fromData(data);
+        if (type.equals("richTextarea")) return (FieldData<U>) new RichTextAreaData().fromData(data);
         if (type.equals("datePicker")) return (FieldData<U>) new DatePickerData().fromData(data);
         if (type.equals("externalDatasets")) return (FieldData<U>) new ExternalDatasetsData().fromData(data);
         if (type.equals("dataRepositories")) return (FieldData<U>) new DataRepositoriesData().fromData(data);

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/pdf/PDFUtils.java

@@ -0,0 +1,45 @@
+package eu.eudat.logic.utilities.documents.pdf;
+
+import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
+import org.apache.commons.io.IOUtils;
+import org.springframework.core.env.Environment;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.web.client.RestTemplate;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.UUID;
+
+public class PDFUtils {
+
+    public static File convertToPDF(FileEnvelope file, Environment environment) throws IOException {
+        LinkedMultiValueMap<String, Object> map = new LinkedMultiValueMap<>();
+        String uuid = UUID.randomUUID().toString();
+        map.add("files", new FileSystemResource(file.getFile()));
+        map.add("filename", uuid + ".pdf");
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.MULTIPART_FORM_DATA);
+        headers.add("Content-disposition", "attachment; filename=" + uuid + ".pdf");
+        headers.add("Content-type", "application/pdf");
+
+        HttpEntity<LinkedMultiValueMap<String, Object>> requestEntity = new HttpEntity<LinkedMultiValueMap<String, Object>>(
+                map, headers);
+
+        byte[] queueResult = new RestTemplate().postForObject(environment.getProperty("pdf.converter.url") + "forms/libreoffice/convert"
+                , requestEntity, byte[].class);
+
+        File resultPdf = new File(environment.getProperty("temp.temp") + uuid + ".pdf");
+        FileOutputStream output = new FileOutputStream(resultPdf);
+        IOUtils.write(queueResult, output);
+        output.close();
+        Files.deleteIfExists(file.getFile().toPath());
+
+        return resultPdf;
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/types/ParagraphStyle.java

@@ -4,7 +4,7 @@ package eu.eudat.logic.utilities.documents.types;
  * Created by ikalyvas on 2/26/2018.
  */
 public enum ParagraphStyle {
-    TEXT(0), HEADER1(1), HEADER2(2), HEADER3(3), HEADER4(4), TITLE(5), FOOTER(6), COMMENT(7), HEADER5(8), HEADER6(9);
+    TEXT(0), HEADER1(1), HEADER2(2), HEADER3(3), HEADER4(4), TITLE(5), FOOTER(6), COMMENT(7), HEADER5(8), HEADER6(9), HTML(10);
 
     private Integer value;
 
@@ -38,6 +38,8 @@ public enum ParagraphStyle {
                 return HEADER5;
             case 9:
                 return HEADER6;
+            case 10:
+                return HTML;
             default:
                 throw new RuntimeException("Unsupported ParagraphStyle Code");
         }

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/HtmlToWorldBuilder.java

@@ -0,0 +1,249 @@
+package eu.eudat.logic.utilities.documents.word;
+
+import org.apache.poi.xwpf.usermodel.*;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.select.NodeTraversor;
+import org.jsoup.select.NodeVisitor;
+import org.openxmlformats.schemas.wordprocessingml.x2006.main.*;
+
+import java.math.BigInteger;
+import java.util.*;
+
+public class HtmlToWorldBuilder implements NodeVisitor {
+
+    private final Map<String, Boolean> properties = new LinkedHashMap<>();
+    private XWPFParagraph paragraph;
+    private XWPFRun run;
+    private Boolean dumpRun;
+    private final float indentation;
+    private Boolean isIdentationUsed;
+    private XWPFNumbering numbering;
+    private Queue<BigInteger> abstractNumId;
+    private BigInteger numberingLevel;
+
+    public static HtmlToWorldBuilder convert(XWPFDocument document, Document htmlDocument, float indentation) {
+        XWPFParagraph paragraph = document.createParagraph();
+        HtmlToWorldBuilder htmlToWorldBuilder = new HtmlToWorldBuilder(paragraph, indentation);
+        NodeTraversor.traverse(htmlToWorldBuilder, htmlDocument);
+        return htmlToWorldBuilder;
+    }
+
+    public HtmlToWorldBuilder(XWPFParagraph paragraph, float indentation) {
+        this.paragraph = paragraph;
+        this.run = this.paragraph.createRun();
+        this.dumpRun = false;
+        this.indentation = indentation;
+        this.isIdentationUsed = false;
+        this.run.setFontSize(11);
+        this.abstractNumId = new ArrayDeque<>();
+        this.numberingLevel = BigInteger.valueOf(-1);
+        this.setDefaultIndentation();
+    }
+
+    @Override
+    public void head(Node node, int i) {
+        String name = node.nodeName();
+        if (name.equals("#text")) {
+            String text = ((TextNode)node).text();
+            this.run.setText(text);
+            this.dumpRun = true;
+        } else {
+            properties.put(name, true);
+        }
+        if (dumpRun) {
+            this.run = this.paragraph.createRun();
+            this.run.setFontSize(11);
+            this.dumpRun = false;
+        }
+        parseProperties(node);
+        properties.clear();
+    }
+
+    private void parseProperties(Node node) {
+        properties.entrySet().forEach(stringBooleanEntry -> {
+            switch (stringBooleanEntry.getKey()) {
+                case "i" :
+                case "em":
+                    this.run.setItalic(stringBooleanEntry.getValue());
+                    break;
+                case "b":
+                case "strong":
+                    this.run.setBold(stringBooleanEntry.getValue());
+                    break;
+                case "u":
+                case "ins":
+                    this.run.setUnderline(stringBooleanEntry.getValue() ? UnderlinePatterns.SINGLE : UnderlinePatterns.NONE);
+                    break;
+                case "small":
+                    this.run.setFontSize(stringBooleanEntry.getValue() ? 8 : 11);
+                    break;
+                case "del":
+                case "strike":
+                case "strikethrough":
+                case "s":
+                    this.run.setStrikeThrough(stringBooleanEntry.getValue());
+                    break;
+                case "mark":
+                    this.run.setTextHighlightColor(stringBooleanEntry.getValue() ? STHighlightColor.YELLOW.toString() : STHighlightColor.NONE.toString());
+                    break;
+                case "sub":
+                    this.run.setSubscript(stringBooleanEntry.getValue() ? VerticalAlign.SUBSCRIPT : VerticalAlign.BASELINE);
+                    break;
+                case "sup":
+                    this.run.setSubscript(stringBooleanEntry.getValue() ? VerticalAlign.SUPERSCRIPT : VerticalAlign.BASELINE);
+                    break;
+                case "div":
+                case "p":
+                    this.paragraph = this.paragraph.getDocument().createParagraph();
+                    this.run = this.paragraph.createRun();
+                    this.isIdentationUsed = false;
+                    this.setDefaultIndentation();
+                    if (stringBooleanEntry.getValue()) {
+                        if (node.hasAttr("align")) {
+                            String alignment = node.attr("align");
+                            this.paragraph.setAlignment(ParagraphAlignment.valueOf(alignment.toUpperCase(Locale.ROOT)));
+                        }
+                    }
+                    break;
+                case "blockquote":
+                    this.paragraph = this.paragraph.getDocument().createParagraph();
+                    this.run = this.paragraph.createRun();
+                    if (stringBooleanEntry.getValue()) {
+                        this.paragraph.setIndentationLeft(720);
+                    } else {
+                        this.isIdentationUsed = false;
+                        this.setDefaultIndentation();
+                    }
+                    break;
+                case "ul":
+                    if (stringBooleanEntry.getValue()) {
+                        createNumbering(STNumberFormat.BULLET);
+                    } else {
+                        this.paragraph = this.paragraph.getDocument().createParagraph();
+                        this.run = this.paragraph.createRun();
+                        this.isIdentationUsed = false;
+                        this.setDefaultIndentation();
+                        this.numberingLevel = this.numberingLevel.subtract(BigInteger.ONE);
+                        ((ArrayDeque)this.abstractNumId).removeLast();
+                    }
+                    break;
+                case "ol":
+                    if (stringBooleanEntry.getValue()) {
+                        createNumbering(STNumberFormat.DECIMAL);
+                    } else {
+                        this.paragraph = this.paragraph.getDocument().createParagraph();
+                        this.run = this.paragraph.createRun();
+                        this.isIdentationUsed = false;
+                        this.setDefaultIndentation();
+                        this.numberingLevel = this.numberingLevel.subtract(BigInteger.ONE);
+                        ((ArrayDeque)this.abstractNumId).removeLast();
+                    }
+                    break;
+                case "li":
+                    if (stringBooleanEntry.getValue()) {
+                        this.paragraph = this.paragraph.getDocument().createParagraph();
+                        this.paragraph.setIndentationLeft(Math.round(indentation * 720) * (numberingLevel.intValue() + 1));
+                        this.run = this.paragraph.createRun();
+                        this.paragraph.setNumID(((ArrayDeque<BigInteger>)abstractNumId).getLast());
+                    }
+                    break;
+                case "font":
+                    if (stringBooleanEntry.getValue()) {
+                        if (node.hasAttr("color")) {
+                            this.run.setColor(node.attr("color").substring(1));
+                        }
+                    } else {
+                        this.run.setColor("000000");
+                    }
+                    break;
+                case "a":
+                    if (stringBooleanEntry.getValue()) {
+                        if (node.hasAttr("href")) {
+                            this.run = createHyperLinkRun(node.attr("href"));
+                            this.run.setColor("0000FF");
+                            this.run.setUnderline(UnderlinePatterns.SINGLE);
+                        }
+                    } else {
+                        this.run = paragraph.createRun();
+                    }
+                    break;
+                case "br":
+                    if (stringBooleanEntry.getValue()) {
+                        this.run.addBreak();
+                    }
+                    break;
+            }
+        });
+    }
+
+    @Override
+    public void tail(Node node, int i) {
+        String name = node.nodeName();
+        properties.put(name, false);
+        parseProperties(node);
+        properties.clear();
+    }
+
+    //GK: This function creates one numbering.xml for the word document and adds a specific format.
+    //It imitates the numbering.xml that is usually generated by word editors like LibreOffice
+    private void createNumbering(STNumberFormat.Enum format) {
+        CTAbstractNum ctAbstractNum = CTAbstractNum.Factory.newInstance();
+        if (this.numbering == null) this.numbering = this.paragraph.getDocument().createNumbering();
+        BigInteger tempNumId = BigInteger.ONE;
+        boolean found = false;
+        while (!found) {
+            Object o = numbering.getAbstractNum(tempNumId);
+            found = (o == null);
+            if (!found) tempNumId = tempNumId.add(BigInteger.ONE);
+        }
+        ctAbstractNum.setAbstractNumId(tempNumId);
+        CTLvl ctLvl = ctAbstractNum.addNewLvl();
+        this.numberingLevel = numberingLevel.add(BigInteger.ONE);
+        ctLvl.setIlvl(numberingLevel);
+        ctLvl.addNewNumFmt().setVal(format);
+        ctLvl.addNewStart().setVal(BigInteger.ONE);
+        if (format == STNumberFormat.BULLET) {
+            ctLvl.addNewLvlJc().setVal(STJc.LEFT);
+            ctLvl.addNewLvlText().setVal("\u2022");
+            ctLvl.addNewRPr(); //Set the Symbol font
+            CTFonts f = ctLvl.getRPr().addNewRFonts();
+            f.setAscii("Symbol");
+            f.setHAnsi("Symbol");
+            f.setCs("Symbol");
+            f.setHint(STHint.DEFAULT);
+        } else {
+            ctLvl.addNewLvlText().setVal("%1.");
+        }
+        XWPFAbstractNum xwpfAbstractNum = new XWPFAbstractNum(ctAbstractNum);
+
+        this.abstractNumId.add(this.numbering.addAbstractNum(xwpfAbstractNum));
+        this.numbering.addNum(((ArrayDeque<BigInteger>)abstractNumId).getLast());
+    }
+
+    private XWPFHyperlinkRun createHyperLinkRun(String uri) {
+        String rId = this.paragraph.getDocument().getPackagePart().addExternalRelationship(uri, XWPFRelation.HYPERLINK.getRelation()).getId();
+
+        CTHyperlink cthyperLink=paragraph.getCTP().addNewHyperlink();
+        cthyperLink.setId(rId);
+        cthyperLink.addNewR();
+
+        return new XWPFHyperlinkRun(
+                cthyperLink,
+                cthyperLink.getRArray(0),
+                paragraph
+        );
+    }
+
+    private void setDefaultIndentation() {
+        if (!isIdentationUsed) {
+            this.paragraph.setIndentationLeft(Math.round(indentation * 720.0F));
+            this.isIdentationUsed = true;
+        }
+    }
+
+    public XWPFParagraph getParagraph() {
+        return paragraph;
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java

@@ -1,6 +1,5 @@
 package eu.eudat.logic.utilities.documents.word;
 
-import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.logic.services.forms.VisibilityRuleService;
@@ -15,6 +14,8 @@ import eu.eudat.models.data.user.components.datasetprofile.Section;
 import eu.eudat.models.data.user.composite.DatasetProfilePage;
 import eu.eudat.models.data.user.composite.PagedDatasetProfile;
 import org.apache.poi.xwpf.usermodel.*;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
 import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTAbstractNum;
 import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDecimalNumber;
 import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLvl;
@@ -24,7 +25,14 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.math.BigInteger;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
 import java.util.*;
+import java.util.stream.Collectors;
 
 public class WordBuilder {
     private static final Logger logger = LoggerFactory.getLogger(WordBuilder.class);
@@ -32,10 +40,12 @@ public class WordBuilder {
     private Map<ParagraphStyle, ApplierWithValue<XWPFDocument, String, XWPFParagraph>> options = new HashMap<>();
     private CTAbstractNum cTAbstractNum;
     private BigInteger numId;
+    private Integer indent;
 
     public WordBuilder() {
         this.cTAbstractNum = CTAbstractNum.Factory.newInstance();
         this.cTAbstractNum.setAbstractNumId(BigInteger.valueOf(1));
+        this.indent = 0;
         this.buildOptions();
     }
 
@@ -48,6 +58,11 @@ public class WordBuilder {
             run.setFontSize(11);
             return paragraph;
         });
+        this.options.put(ParagraphStyle.HTML, (mainDocumentPart, item) -> {
+            Document htmlDoc = Jsoup.parse(item.replaceAll("\n", "<br>"));
+            HtmlToWorldBuilder htmlToWorldBuilder = HtmlToWorldBuilder.convert(mainDocumentPart, htmlDoc, indent > 0 ? (indent/2.0F) * 0.8F : 0.8F);
+            return htmlToWorldBuilder.getParagraph();
+        });
         this.options.put(ParagraphStyle.TITLE, (mainDocumentPart, item) -> {
             XWPFParagraph paragraph = mainDocumentPart.createParagraph();
             paragraph.setStyle("Title");
@@ -124,8 +139,16 @@ public class WordBuilder {
 
     public XWPFDocument build(XWPFDocument document, PagedDatasetProfile pagedDatasetProfile, VisibilityRuleService visibilityRuleService) throws IOException {
         createPages(pagedDatasetProfile.getPages(), document, true, visibilityRuleService);
-        XWPFAbstractNum abstractNum = new XWPFAbstractNum(cTAbstractNum);
         XWPFNumbering numbering = document.createNumbering();
+        BigInteger tempNumId = BigInteger.ONE;
+        boolean found = false;
+        while (!found) {
+            Object o = numbering.getAbstractNum(tempNumId);
+            found = (o == null);
+            if (!found) tempNumId = tempNumId.add(BigInteger.ONE);
+        }
+        cTAbstractNum.setAbstractNumId(tempNumId);
+        XWPFAbstractNum abstractNum = new XWPFAbstractNum(cTAbstractNum);
         BigInteger abstractNumID = numbering.addAbstractNum(abstractNum);
         this.numId = numbering.addNum(abstractNumID);
         createPages(pagedDatasetProfile.getPages(), document, false, visibilityRuleService);
@@ -179,7 +202,8 @@ public class WordBuilder {
                 }
                 hasValue = createFields(compositeField.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService);
                 if (compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()) {
-                    for (FieldSet multiplicityFieldset : compositeField.getMultiplicityItems()) {
+                    List<FieldSet> list = compositeField.getMultiplicityItems().stream().sorted(Comparator.comparingInt(FieldSet::getOrdinal)).collect(Collectors.toList());
+                    for (FieldSet multiplicityFieldset : list) {
                        hasValue = createFields(multiplicityFieldset.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService);
                     }
                 }
@@ -199,16 +223,20 @@ public class WordBuilder {
     private Boolean createFields(List<Field> fields, XWPFDocument mainDocumentPart, Integer indent, Boolean createListing, VisibilityRuleService visibilityRuleService) {
         if (createListing) this.addListing(mainDocumentPart, indent, false, false);
         boolean hasValue = false;
-        for (Field field: fields) {
+        List<Field> tempFields = fields.stream().sorted(Comparator.comparingInt(Field::getOrdinal)).collect(Collectors.toList());
+        for (Field field: tempFields) {
             if (visibilityRuleService.isElementVisible(field.getId())) {
                 if (!createListing) {
                     try {
-                        XWPFParagraph paragraph = addParagraphContent(this.formatter(field), mainDocumentPart, ParagraphStyle.TEXT, numId);
+                        if (field.getValue() != null && !field.getValue().toString().isEmpty()) {
+                            this.indent = indent;
+                            XWPFParagraph paragraph = addParagraphContent(this.formatter(field), mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId);
                             if (paragraph != null) {
                                 CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl();
                                 number.setVal(BigInteger.valueOf(indent));
                                 hasValue = true;
                             }
+                        }
                     } catch (IOException e) {
                         logger.error(e.getMessage(), e);
                     }
@@ -221,11 +249,13 @@ public class WordBuilder {
     public XWPFParagraph addParagraphContent(String text, XWPFDocument mainDocumentPart, ParagraphStyle style, BigInteger numId) {
         if (text != null && !text.isEmpty()) {
             XWPFParagraph paragraph = this.options.get(style).apply(mainDocumentPart, text);
+            if (paragraph != null) {
                 if (numId != null) {
                     paragraph.setNumID(numId);
                 }
                 return paragraph;
             }
+        }
         return null;
     }
 
@@ -238,13 +268,16 @@ public class WordBuilder {
         }
         if (question) {
             cTLvl.addNewNumFmt().setVal(STNumberFormat.DECIMAL);
+            cTLvl.addNewLvlText().setVal("");
             cTLvl.setIlvl(BigInteger.valueOf(indent));
         } else if (!question && hasIndication) {
             cTLvl.addNewNumFmt().setVal(STNumberFormat.DECIMAL);
+            cTLvl.addNewLvlText().setVal("");
             cTLvl.setIlvl(BigInteger.valueOf(indent));
         }
         if (!question && !hasIndication) {
             cTLvl.addNewNumFmt().setVal(STNumberFormat.NONE);
+            cTLvl.addNewLvlText().setVal("");
             cTLvl.setIlvl(BigInteger.valueOf(indent));
         }
     }
@@ -275,44 +308,16 @@ public class WordBuilder {
                         try {
                             mapList = Arrays.asList(mapper.readValue(field.getValue().toString(), HashMap[].class));
                         }catch (Exception e) {
-                            logger.warn(e.getMessage(), e);
-                            logger.info("Moving to fallback parsing");
+                         //   logger.warn(e.getMessage(), e);
+                         //   logger.info("Moving to fallback parsing");
                             Map <String, Object> map = new HashMap<>();
                             map.put("label", field.getValue().toString());
                             mapList.add(map);
                         }
-                        /*try {
-                            if (field.getValue().toString().startsWith("[")) {
-                                JSONArray jsonarray = new JSONArray(field.getValue().toString());
-                                for (int i = 0; i < jsonarray.length(); i++) {
-                                    JSONObject jsonObject = jsonarray.getJSONObject(i);
-                                    String id = jsonObject.get("id").toString();
-                                    String label = jsonObject.getString("label");
-                                    if (id != null && label != null) {
-                                        map.put(id, label);
-                                    }
-                                }
-                            } else if (field.getValue().toString().startsWith("{")) {
-                                JSONObject jsonObject = new JSONObject(field.getValue().toString());
-                                String id = jsonObject.get("id").toString();
-                                String label = jsonObject.getString("label");
-                                if (id != null && label != null) {
-                                    map.put(id, label);
-                                }
-                            }
-                        } catch (Exception e) {
-                            Map<String, String> exMap = mapper.readValue(field.getValue().toString(), new TypeReference<Map<String, String>>() {
-                            });
-                            return exMap.get("label");
-                        }*/
                     }
                     StringBuilder sb = new StringBuilder();
                     int index = 0;
                     for (Map<String, Object> map: mapList) {
-                        /*if (!map.containsKey("label") && !map.containsKey("description")) {
-                            logger.error("Value is missing the \"label\" and the \"description\" attributes");
-                            map.put("label", "unknown Name");
-                        }*/
                         for (Map.Entry<String, Object> entry : map.entrySet()) {
                             if (entry.getValue() != null && (entry.getKey().equals("label") || entry.getKey().equals("description") || entry.getKey().equals("name"))) {
                                 sb.append(entry.getValue().toString());
@@ -325,17 +330,15 @@ public class WordBuilder {
                     return sb.toString();
                 } else if (comboboxType.equals("wordlist")) {
                     WordListData wordListData = (WordListData) field.getData();
-                    if (wordListData.getOptions().isEmpty() && field.getValue() != null) {
-                        logger.warn("World List has no values but the field has");
-                        logger.info("Return value as is");
-                        return field.getValue().toString();
-                    } else if (field.getValue() != null){
+                    if (field.getValue() != null){
                         ComboBoxData<WordListData>.Option selectedOption = null;
-                        for (ComboBoxData<WordListData>.Option option: wordListData.getOptions()) {
+                        if (!wordListData.getOptions().isEmpty()) {
+                            for (ComboBoxData<WordListData>.Option option : wordListData.getOptions()) {
                                 if (option.getValue().equals(field.getValue())) {
                                     selectedOption = option;
                                 }
                             }
+                        }
                         return selectedOption != null ? selectedOption.getLabel() : field.getValue().toString();
                     }
                     return "";
@@ -343,33 +346,39 @@ public class WordBuilder {
             }
             case "booleanDecision":
                 if (field.getValue() != null && field.getValue().equals("true")) return "Yes";
-                else return "No";
+                if (field.getValue() != null && field.getValue().equals("false")) return "No";
+                return null;
             case "radiobox":
                 return field.getValue() != null ? field.getValue().toString() : null;
             case "checkBox":
                 CheckBoxData data = (CheckBoxData) field.getData();
                 if (field.getValue() == null || field.getValue().equals("false")) return null;
                 return data.getLabel();
-            case "freetext":
             case "datepicker":
+            case "datePicker":{
+                Instant instant;
+                try {
+                    instant = Instant.parse((String) field.getValue());
+                } catch (DateTimeParseException ex) {
+                    instant = (Instant) DateTimeFormatter.ofPattern("yyyy-MM-dd").parse((String)field.getValue());
+                }
+                return field.getValue() != null ? DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()).format(instant) : "";
+            }
+            case "freetext":
             case "textarea":
+            case "richTextarea":
                 return field.getValue() != null ? field.getValue().toString(): "";
             case "datasetIdentifier":
             case "validation":
-                if (field.getValue() != null) {
+                if (field.getValue() != null && !field.getValue().toString().isEmpty()) {
                     Map<String, String> identifierData;
-                    try {
                     ObjectMapper mapper = new ObjectMapper();
+                    try {
                         identifierData = mapper.readValue(field.getValue().toString(), HashMap.class);
-                    } catch (JsonParseException ex) {
-                        identifierData = new HashMap<>();
-                        String parsedData = field.getValue().toString().substring(1, field.getValue().toString().length() - 1);
-                        StringTokenizer commaTokens = new StringTokenizer(parsedData, ", ");
-                        while (commaTokens.hasMoreTokens()) {
-                            String token = commaTokens.nextToken();
-                            StringTokenizer equalTokens = new StringTokenizer(token, "=");
-                            identifierData.put(equalTokens.nextToken(), equalTokens.nextToken());
-                        }
+                    } catch (Exception ex) {
+                      //  logger.warn(ex.getLocalizedMessage(), ex);
+                      //  logger.info("Reverting to custom parsing");
+                        identifierData = customParse(field.getValue().toString());
                     }
                     return "id: " + identifierData.get("identifier") + ", Validation Type: " + identifierData.get("type");
                 }
@@ -381,4 +390,17 @@ public class WordBuilder {
     private boolean hasVisibleFields(FieldSet compositeFields, VisibilityRuleService visibilityRuleService) {
         return compositeFields.getFields().stream().anyMatch(field -> visibilityRuleService.isElementVisible(field.getId()));
     }
+
+    private Map<String, String> customParse(String value) {
+        Map<String, String> result = new LinkedHashMap<>();
+        String parsedValue = value.replaceAll("[^a-zA-Z0-9\\s:=,]", "");
+        StringTokenizer commaTokens = new StringTokenizer(parsedValue, ", ");
+        String delimeter = parsedValue.contains("=") ? "=" : ":";
+        while (commaTokens.hasMoreTokens()) {
+            String token = commaTokens.nextToken();
+            StringTokenizer delimiterTokens = new StringTokenizer(token, delimeter);
+            result.put(delimiterTokens.nextToken(), delimiterTokens.nextToken());
+        }
+        return result;
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/XWPFHtmlDocument.java

@@ -0,0 +1,63 @@
+package eu.eudat.logic.utilities.documents.word;
+
+import org.apache.poi.ooxml.POIXMLDocumentPart;
+import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
+import org.apache.poi.openxml4j.opc.OPCPackage;
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.openxml4j.opc.PackagePartName;
+import org.apache.poi.openxml4j.opc.PackagingURIHelper;
+import org.apache.poi.xwpf.usermodel.XWPFDocument;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.UUID;
+
+public class XWPFHtmlDocument extends POIXMLDocumentPart {
+
+    private String html;
+    private String id;
+
+    public XWPFHtmlDocument(PackagePart pkg, String id) {
+        super(pkg);
+        this.html = "<!DOCTYPE html><html><head><meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\"><style></style><title>HTML import</title></head><body><p></p></body>";
+        this.id = id;
+    }
+
+    public String getHtml() {
+        return html;
+    }
+
+    public void setHtml(String html) {
+        this.html = this.html.replace("<p></p>", html);
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    @Override
+    protected void commit() throws IOException {
+        PackagePart packagePart = getPackagePart();
+        OutputStream outputStream = packagePart.getOutputStream();
+        Writer writer = new OutputStreamWriter(outputStream, "UTF-8");
+        writer.write(html);
+        writer.close();
+        outputStream.close();
+    }
+
+    public static XWPFHtmlDocument addHtmlDocument(XWPFDocument document) throws InvalidFormatException {
+        OPCPackage oPCPackage = document.getPackage();
+        String id = UUID.randomUUID().toString();
+        PackagePartName partName = PackagingURIHelper.createPartName("/word/" + id + ".html");
+        PackagePart part = oPCPackage.createPart(partName, "text/html");
+        XWPFHtmlDocument xWPFHtmlDocument = new XWPFHtmlDocument(part, id);
+        document.addRelation(xWPFHtmlDocument.getId(), new XWPFHtmlRelation(), xWPFHtmlDocument);
+        return xWPFHtmlDocument;
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/XWPFHtmlRelation.java

@@ -0,0 +1,11 @@
+package eu.eudat.logic.utilities.documents.word;
+
+import org.apache.poi.ooxml.POIXMLRelation;
+
+public class XWPFHtmlRelation extends POIXMLRelation {
+    public XWPFHtmlRelation() {
+        super("text/html",
+                "http://schemas.openxmlformats.org/officeDocument/2006/relationships/aFChunk",
+                "/word/htmlDoc#.html");
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/xml/ExportXmlBuilder.java

@@ -1,7 +1,10 @@
 package eu.eudat.logic.utilities.documents.xml;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.exc.MismatchedInputException;
 import eu.eudat.logic.services.forms.VisibilityRuleService;
 import eu.eudat.logic.utilities.builders.XmlBuilder;
+import eu.eudat.models.data.components.commons.datafield.ExternalDatasetsData;
 import eu.eudat.models.data.user.components.datasetprofile.Field;
 import eu.eudat.models.data.user.components.datasetprofile.FieldSet;
 import eu.eudat.models.data.user.components.datasetprofile.Section;
@@ -17,7 +20,11 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 
 public class ExportXmlBuilder {
@@ -92,19 +99,43 @@ public class ExportXmlBuilder {
             if (visibilityRuleService.isElementVisible(field.getId())) {
                 Element elementField = element.createElement("field");
                 elementField.setAttribute("id", field.getId());
+                if (field.getViewStyle().getRenderStyle().equals("externalDatasets")) {
+                    elementField.setAttribute("type", ((ExternalDatasetsData)field.getData()).getType());
+                }
                 if (field.getValue() != null) {
                     Element valueField = element.createElement("value");
+                    ObjectMapper mapper = new ObjectMapper();
                     try {
-                        JSONArray jsonArray = new JSONArray(field.getValue().toString());
+
+                        List<Map<String, Object>> jsonArray = mapper.readValue(field.getValue().toString(), List.class);
+//                        JSONArray jsonArray = new JSONArray(field.getValue().toString());
                         StringBuilder sb = new StringBuilder();
-                        for (int i = 0; i < jsonArray.length(); i++) {
+                        boolean firstTime = true;
+                        for (Map<String, Object> jsonElement: jsonArray) {
+                            if (!firstTime) {
+                                sb.append(", ");
+                            }
+                            sb.append(jsonElement.get("label") != null ? jsonElement.get("label") : jsonElement.get("name"));
+                            firstTime = false;
+
+                        }
+                        /*for (int i = 0; i < jsonArray.length(); i++) {
                             sb.append(jsonArray.getJSONObject(i).get("label").toString());
                             if (i != jsonArray.length() - 1) sb.append(", ");
-                        }
+                        }*/
                         valueField.setTextContent(sb.toString());
-                    } catch (JSONException ex) {
+                    } catch (IOException ex) {
+                        try {
+                            Map<String, Object> jsonElement = mapper.readValue(field.getValue().toString(), Map.class);
+                            valueField.setTextContent((jsonElement.get("label") != null ? jsonElement.get("label").toString() : jsonElement.get("name") != null ? jsonElement.get("name").toString() : ""));
+                        } catch (IOException e) {
+                            try {
+                                valueField.setTextContent(DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()).format(Instant.parse(field.getValue().toString())));
+                            } catch (Exception exc) {
                                 valueField.setTextContent(field.getValue().toString());
                             }
+                        }
+                    }
                     elementField.appendChild(valueField);
                 }
                 elementFields.appendChild(elementField);

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/xml/datasetProfileXml/ExportXmlBuilderDatasetProfile.java

@@ -260,6 +260,7 @@ public class ExportXmlBuilderDatasetProfile {
                     case CHECK_BOX:
                     case FREE_TEXT:
                     case TEXT_AREA:
+                    case RICH_TEXT_AREA:
                     case DATE_PICKER:
                     case DATASET_IDENTIFIER:
                     case CURRENCY:
@@ -290,6 +291,7 @@ public class ExportXmlBuilderDatasetProfile {
                         ExternalDatasetsData externalDatasetsData = (ExternalDatasetsData) field.getData();
                         dataOut.setAttribute("label", externalDatasetsData.getLabel());
                         dataOut.setAttribute("multiAutocomplete", externalDatasetsData.getMultiAutoComplete().toString());
+                        dataOut.setAttribute("type", externalDatasetsData.getType());
                         break;
                     case DATA_REPOSITORIES:
                         DataRepositoriesData dataRepositoriesData = (DataRepositoriesData) field.getData();

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/xml/datasetProfileXml/datasetProfileModel/DatasetProfile.java

@@ -52,7 +52,9 @@ public class DatasetProfile {
         List<eu.eudat.models.data.admin.components.datasetprofile.Section> sectionDatasetEntity = new LinkedList<>();
         for (Page xmlPage: page) {
             pagesDatasetEntity.add(xmlPage.toAdminCompositeModelPage());
-            sectionDatasetEntity.add(xmlPage.toAdminCompositeModelSection());
+            for (int i = 0; i < xmlPage.getSections().size(); i++) {
+                sectionDatasetEntity.add(xmlPage.toAdminCompositeModelSection(i));
+            }
         }
         newDatasetEntityProfile.setPages(pagesDatasetEntity);
         newDatasetEntityProfile.setSections(sectionDatasetEntity);

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/xml/datasetProfileXml/datasetProfileModel/Page.java

@@ -3,13 +3,14 @@ package eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileM
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
 
 @XmlRootElement(name = "page")
 public class Page {
     private String id;
     private int ordinal;
     private String title;
-    private Sections sections;
+    private List<Sections> sections;
 
     @XmlAttribute(name = "id")
     public String getId() {
@@ -39,11 +40,11 @@ public class Page {
     }
 
     @XmlElement(name = "sections")
-    public Sections getSections() {
+    public List<Sections> getSections() {
         return sections;
     }
 
-    public void setSections(Sections sections) {
+    public void setSections(List<Sections> sections) {
         this.sections = sections;
     }
 
@@ -55,7 +56,7 @@ public class Page {
         return pageEntity;
     }
 
-    public eu.eudat.models.data.admin.components.datasetprofile.Section toAdminCompositeModelSection(){
+    public eu.eudat.models.data.admin.components.datasetprofile.Section toAdminCompositeModelSection(int i){
      /*   eu.eudat.models.data.admin.components.datasetprofile.Section sectionEntity =new eu.eudat.models.data.admin.components.datasetprofile.Section();
 //        List<eu.eudat.models.data.admin.components.datasetprofile.Section> sectionsListEntity = new LinkedList<>();
 //        for (Section xmlsection:this.sections.section) {
@@ -68,6 +69,6 @@ public class Page {
         sectionEntity.setId(this.id);
         sectionEntity.setOrdinal(this.ordinal);
         sectionEntity.setTitle(this.title);*/
-        return sections.toAdminCompositeModelSection();
+        return sections.get(i).toAdminCompositeModelSection();
     }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/MultiDateDeserializer.java

@@ -0,0 +1,41 @@
+package eu.eudat.logic.utilities.json;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+public class MultiDateDeserializer extends StdDeserializer<Date> {
+
+    private static final List<String> DATE_FORMATS = Arrays.asList("yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss.S");
+
+
+    public MultiDateDeserializer() {
+        super(Date.class);
+    }
+
+    protected MultiDateDeserializer(Class<?> vc) {
+        super(vc);
+    }
+
+    @Override
+    public Date deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+        String text = p.getText();
+
+        for (String dateFormat: DATE_FORMATS) {
+            try {
+                return new SimpleDateFormat(dateFormat).parse(text);
+            } catch (ParseException ignored) {
+            }
+        }
+        throw new JsonParseException(p, "No supported Date format");
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/models/data/components/commons/ViewStyle.java

@@ -28,6 +28,7 @@ public class ViewStyle {
         CHECK_BOX("checkBox"),
         FREE_TEXT("freetext"),
         TEXT_AREA("textarea"),
+        RICH_TEXT_AREA("richTextarea"),
         DATE_PICKER("datePicker"),
         EXTERNAL_DATASETS("externalDatasets"),
         DATA_REPOSITORIES("dataRepositories"),

dmp-backend/web/src/main/java/eu/eudat/models/data/components/commons/datafield/AutoCompleteData.java

@@ -95,14 +95,14 @@ public class AutoCompleteData extends ComboBoxData<AutoCompleteData> {
             this.autoCompleteSingleDataList.add(new AutoCompleteSingleData());
             this.mapFromXml(item, this.autoCompleteSingleDataList.get(0));
         }
-        this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutocomplete"));
+        this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutoComplete"));
         return this;
     }
 
     private void mapFromXml(Element item, AutoCompleteSingleData singleData) {
         singleData.url = item.getAttribute("url");
         singleData.optionsRoot = item.getAttribute("optionsRoot");
-        this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutocomplete"));
+        this.multiAutoComplete = Boolean.parseBoolean(item.getAttribute("multiAutoComplete"));
         if (item.getAttribute("autoCompleteType") == null || item.getAttribute("autoCompleteType").equals("") ) {
             singleData.autocompleteType = AutocompleteType.UNCACHED.getValue();
         } else {
@@ -173,7 +173,7 @@ public class AutoCompleteData extends ComboBoxData<AutoCompleteData> {
         dataMap.put("label", item != null ? item.getAttribute("label") : "");
         //dataMap.put("url", item != null ? item.getAttribute("url") : "");
         dataMap.put("type", item != null ? item.getAttribute("type") : "autocomplete");
-        dataMap.put("multiAutoComplete", item != null ? Boolean.valueOf(item.getAttribute("multiAutocomplete")) : false);
+        dataMap.put("multiAutoComplete", item != null ? Boolean.valueOf(item.getAttribute("multiAutoComplete")) : false);
         List<Map<String, Object>> autoCompletes = new ArrayList<>();
         NodeList autoCompleteSingles = item.getChildNodes();
         for (int i = 0; i < autoCompleteSingles.getLength(); i++) {

dmp-backend/web/src/main/java/eu/eudat/models/data/components/commons/datafield/ExternalDatasetsData.java

@@ -8,6 +8,7 @@ import java.util.Map;
 
 public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
     private Boolean multiAutoComplete;
+    private String type;
 
     public Boolean getMultiAutoComplete() {
         return multiAutoComplete;
@@ -17,11 +18,20 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
         this.multiAutoComplete = multiAutoComplete;
     }
 
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
     @Override
     public ExternalDatasetsData fromData(Object data) {
         if (data != null) {
             this.setLabel((String) ((Map<String, Object>) data).get("label"));
             this.setMultiAutoComplete(((Map<String, Object>) data).get("multiAutoComplete") != null && !((Map<String, Object>) data).get("multiAutoComplete").toString().isEmpty()? Boolean.parseBoolean( ((Map<String, Object>) data).get("multiAutoComplete").toString()) : false);
+            this.setType(((Map<String, Object>) data).get("type") != null && !((Map<String, Object>) data).get("type").toString().isEmpty()? ((Map<String, Object>) data).get("type").toString() : "other");
         }
         return this;
     }
@@ -38,6 +48,9 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
         if (this.getMultiAutoComplete() != null) {
             root.setAttribute("multiAutoComplete", this.getMultiAutoComplete().toString());
         }
+        if (this.getType() != null) {
+            root.setAttribute("type", this.getType());
+        }
         return root;
     }
 
@@ -45,6 +58,7 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
     public ExternalDatasetsData fromXml(Element item) {
         this.setLabel(item != null ? item.getAttribute("label") : "");
         this.setMultiAutoComplete(Boolean.parseBoolean(item.getAttribute("multiAutoComplete")));
+        this.setType(item.getAttribute("type") != null ? item.getAttribute("type"): "other");
         return this;
     }
 
@@ -53,6 +67,7 @@ public class ExternalDatasetsData extends FieldData<ExternalDatasetsData> {
         HashMap dataMap = new HashMap();
         dataMap.put("label", item != null && item.getAttributes().getLength() > 0? item.getAttribute("label") : "");
         dataMap.put("multiAutoComplete", item != null && item.getAttributes().getLength() > 0? Boolean.parseBoolean(item.getAttribute("multiAutocomplete")) : false);
+        dataMap.put("type", item != null && item.getAttributes().getLength() > 0? item.getAttribute("type") : "other");
         return dataMap;
     }
 }

dmp-backend/web/src/main/java/eu/eudat/models/data/components/commons/datafield/RichTextAreaData.java

@@ -0,0 +1,43 @@
+package eu.eudat.models.data.components.commons.datafield;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class RichTextAreaData extends FieldData<RichTextAreaData> {
+    @Override
+    public RichTextAreaData fromData(Object data) {
+        if (data != null) {
+            this.setLabel(((Map<String, String>) data).get("label"));
+        }
+        return this;
+    }
+
+    @Override
+    public Object toData() {
+        return null;
+    }
+
+    @Override
+    public Element toXml(Document doc) {
+        Element root = doc.createElement("data");
+        root.setAttribute("label", this.getLabel());
+        return root;
+    }
+
+    @Override
+    public RichTextAreaData fromXml(Element item) {
+        this.setLabel(item.getAttribute("label"));
+        return this;
+    }
+
+    @Override
+    public Map<String, Object> toMap(Element item) {
+        HashMap dataMap = new HashMap();
+        dataMap.put("label", item != null ? item.getAttribute("label") : "");
+        return dataMap;
+    }
+}

dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java

@@ -7,7 +7,10 @@ import eu.eudat.models.data.funder.FunderDMPEditorModel;
 import eu.eudat.models.data.grant.GrantDMPEditorModel;
 import eu.eudat.models.data.project.ProjectDMPEditorModel;
 import eu.eudat.models.data.userinfo.UserInfo;
+import eu.eudat.models.data.userinfo.UserListingModel;
 import net.minidev.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.*;
 import java.util.stream.Collectors;
@@ -16,6 +19,8 @@ import java.util.stream.Collectors;
  * Created by ikalyvas on 2/5/2018.
  */
 public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataManagementPlanNewVersionModel> {
+    private static final Logger logger = LoggerFactory.getLogger(DataManagementPlanNewVersionModel.class);
+
     private UUID id;
     private String label;
     private UUID groupId;
@@ -26,7 +31,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
     private eu.eudat.models.data.grant.GrantDMPEditorModel grant;
     private List<Organisation> organisations;
     private List<Researcher> researchers;
-    private List<UserDMP> associatedUsers;
+    private List<UserListingModel> associatedUsers;
     private eu.eudat.models.data.userinfo.UserInfo creator;
     private Date created;
     private List<Dataset> datasets;
@@ -104,10 +109,10 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
         this.researchers = researchers;
     }
 
-    public List<UserDMP> getAssociatedUsers() {
+    public List<UserListingModel> getAssociatedUsers() {
         return associatedUsers;
     }
-    public void setAssociatedUsers(List<UserDMP> associatedUsers) {
+    public void setAssociatedUsers(List<UserListingModel> associatedUsers) {
         this.associatedUsers = associatedUsers;
     }
 
@@ -163,7 +168,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
     public DMP toDataModel() throws Exception {
         DMP entity = new DMP();
         entity.setId(this.id);
-        entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers)));
+       // entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers)));
         entity.setDescription(this.description);
         entity.setStatus((short) this.status);
         entity.setGroupId(this.groupId == null ? UUID.randomUUID() : this.groupId);
@@ -227,7 +232,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
                 funder.setModified(new Date());
 
                 entity.getGrant().setFunder(funder);
-            } else if (this.funder.getExistFunder() != null && this.funder.getLabel() == null){
+            } else if (this.funder.getExistFunder() != null && this.funder.getLabel() == null) {
                 entity.getGrant().setFunder(this.funder.getExistFunder().toDataModel());
                 entity.getGrant().getFunder().setType(Funder.FunderType.EXTERNAL.getValue());
             }

dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DmpImportModel.java

@@ -19,6 +19,10 @@ public class DmpImportModel {
     private List<UserInfoImportModels> associatedUsersImportModels;
     private List<DynamicFieldWithValueImportModels> dynamicFieldsImportModels;
     private List<DatasetImportModels> datasetImportModels;
+    private String language;
+    private Boolean visibility;
+    private String publicDate;
+    private String costs;
 
     @XmlElement(name = "description")
     public String getDescriptionImport() {
@@ -113,4 +117,37 @@ public class DmpImportModel {
     @XmlElement(name = "dataset")
     public List<DatasetImportModels> getDatasetImportModels() { return datasetImportModels; }
     public void setDatasetImportModels(List<DatasetImportModels> datasetImportModels) { this.datasetImportModels = datasetImportModels; }
+
+    @XmlElement(name = "language")
+    public String getLanguage() {
+        return language;
+    }
+
+    public void setLanguage(String language) {
+        this.language = language;
+    }
+    @XmlElement(name = "visibility")
+    public Boolean getVisibility() {
+        return visibility;
+    }
+
+    public void setVisibility(Boolean visibility) {
+        this.visibility = visibility;
+    }
+    @XmlElement(name = "publicDate")
+    public String getPublicDate() {
+        return publicDate;
+    }
+
+    public void setPublicDate(String publicDate) {
+        this.publicDate = publicDate;
+    }
+    @XmlElement(name = "costs")
+    public String getCosts() {
+        return costs;
+    }
+
+    public void setCosts(String costs) {
+        this.costs = costs;
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DataManagementPlanListingModel.java

@@ -25,7 +25,7 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
     private String label;
     private String grant;
   //  private String profile;
-   // private Date creationTime;
+    private Date creationTime;
     private Date modifiedTime;
     //private String organisations;
     private int version;
@@ -70,12 +70,12 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
         this.profile = profile;
     }*/
 
-    /*public Date getCreationTime() {
+    public Date getCreationTime() {
         return creationTime;
     }
     public void setCreationTime(Date creationTime) {
         this.creationTime = creationTime;
-    }*/
+    }
 
     public Date getModifiedTime() {
         return modifiedTime;
@@ -187,7 +187,7 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
         this.id = entity.getId().toString();
         this.label = entity.getLabel();
         this.groupId = entity.getGroupId();
-//        this.creationTime = entity.getCreated();
+        this.creationTime = entity.getCreated();
 //        this.associatedProfiles = entity.getAssociatedDmps().stream().map(item -> new AssociatedProfile().fromData(item)).collect(Collectors.toList());
         return this;
     }
@@ -196,7 +196,7 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
         this.id = entity.getId().toString();
         this.label = entity.getLabel();
         this.groupId = entity.getGroupId();
-//        this.creationTime = entity.getCreated();
+        this.creationTime = entity.getCreated();
         return this;
     }
 
@@ -205,8 +205,8 @@ public class DataManagementPlanListingModel implements DataModel<DMP, DataManage
         this.status = entity.getStatus();
         this.version = entity.getVersion();
         this.grant = entity.getGrant().getLabel();
-        /*if (entity.getProfile() != null) this.profile = entity.getProfile().getLabel();
-        this.creationTime = entity.getCreated();*/
+        /*if (entity.getProfile() != null) this.profile = entity.getProfile().getLabel(); */
+        this.creationTime = entity.getCreated();
         this.modifiedTime = entity.getModified();
 //        this.organisations = LabelBuilder.getLabel(entity.getOrganisations().stream().map(item -> new Organisation().fromDataModel(item)).collect(Collectors.toList()));
         this.datasets = entity.getDataset().stream().map(x-> new DatasetUrlListing().fromDataModel(x)).collect(Collectors.toList());

dmp-backend/web/src/main/java/eu/eudat/models/data/listingmodels/DatasetListingModel.java

@@ -199,11 +199,11 @@ public class DatasetListingModel implements DataModel<Dataset, DatasetListingMod
         this.profile = entity.getProfile() != null ? new DatasetProfileOverviewModel().fromDataModel(entity.getProfile()) : null;
         this.description = entity.getDescription();
         this.status = entity.getStatus();
-        this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
-        this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
-        this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
-        this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
-        this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
+        //this.grantAbbreviation = entity.getDmp() != null ? entity.getDmp().getGrant().getAbbreviation() : "";
+       // this.grantId = entity.getDmp() != null ? entity.getDmp().getGrant().getId().toString() : "";
+        //this.registries = LabelBuilder.getLabel(entity.getRegistries().stream().map(item -> new eu.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()));
+       // this.dataRepositories = LabelBuilder.getLabel(entity.getDatasetDataRepositories().stream().map(item -> new DataRepository().fromDataModel(item.getDataRepository())).collect(Collectors.toList()));
+        //this.services = LabelBuilder.getLabel(entity.getServices().stream().map(item -> new Service().fromDataModel(item.getService())).collect(Collectors.toList()));
         if (entity.getFinalizedAt() == null && entity.getStatus() == Dataset.Status.FINALISED.getValue()) {
             this.finalizedAt = entity.getDmp().getFinalizedAt();
         } else {

dmp-backend/web/src/main/java/eu/eudat/models/data/user/components/datasetprofile/Field.java

@@ -17,6 +17,7 @@ import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
@@ -201,13 +202,10 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
     @Override
     public void fromJsonObject(Map<String, Object> properties) {
         try {
-            JSONArray jsonArray = new JSONArray(properties.get(this.id).toString());
-            List<String> stringList = new LinkedList<>();
-            for (int i = 0; i < jsonArray.length(); i++) {
-                stringList.add(jsonArray.getJSONObject(i).toString());
-            }
+            ObjectMapper mapper = new ObjectMapper();
+            List<String> stringList = mapper.readValue(properties.get(this.id).toString(), LinkedList.class);
             this.value = stringList;
-        } catch (JSONException | NullPointerException e) {
+        } catch (JSONException | NullPointerException | IOException e) {
             this.value = (String) properties.get(this.id);
         }
         this.multiplicityItems = new LinkedList<>();
@@ -231,7 +229,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
     @Override
     public void toMap(Map<String, Object> fieldValues) {
         if (this.value != null) {
-            if (this.viewStyle != null && this.viewStyle.getRenderStyle().equals("datasetIdentifier")) {
+            if ((this.viewStyle != null && this.viewStyle.getRenderStyle().equals("datasetIdentifier") || this.value instanceof Collection)) {
                 ObjectMapper mapper = new ObjectMapper();
                 String valueString = null;
                 try {
@@ -240,7 +238,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
                 } catch (JsonProcessingException e) {
                     logger.error(e.getMessage(), e);
                 }
-            } else if (this.value instanceof Collection) {
+            } /*else if (this.value instanceof Collection) {
                 Collection valueCollection = (Collection) this.value;
                 StringBuilder valueBuilder = new StringBuilder();
                 valueBuilder.append("[");
@@ -252,7 +250,7 @@ public class Field implements Comparable, PropertiesModelBuilder, ViewStyleDefin
                }
                valueBuilder.append("]");
                fieldValues.put(this.id, valueBuilder.toString());
-            } else {
+            }*/ else {
                 fieldValues.put(this.id, this.value.toString());
             }
         } else {

dmp-backend/web/src/main/java/eu/eudat/models/data/user/components/datasetprofile/FieldSet.java

@@ -175,12 +175,12 @@ public class FieldSet implements Comparable, PropertiesModelBuilder, ViewStyleDe
 
     private FieldSet CloneForMultiplicity2(List<String> key, Map<String, Object> properties,String[] ids, int index){
         FieldSet newFieldSet = new FieldSet();
-        newFieldSet.id = ids[0]+"_"+ids[1]+"_"+ids[2];
+        newFieldSet.id = ids[0]+"_"+ids[1]+"_"+ids[2] + (ids.length > 4 ? "_" + ids[3] : "");
         newFieldSet.description = this.description;
         newFieldSet.extendedDescription = this.extendedDescription;
         newFieldSet.additionalInformation=this.additionalInformation;
         newFieldSet.title = this.title;
-        newFieldSet.ordinal = this.ordinal;
+        newFieldSet.ordinal = ids.length > 4 ? Integer.valueOf(ids[3]) : this.ordinal;
         newFieldSet.fields = new LinkedList();
 
         for (Field field: this.fields) {

dmp-backend/web/src/main/java/eu/eudat/models/rda/Dmp.java

@@ -9,6 +9,8 @@ import java.util.List;
 import java.util.Map;
 
 import com.fasterxml.jackson.annotation.*;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import eu.eudat.logic.utilities.json.MultiDateDeserializer;
 
 
 /**
@@ -71,6 +73,7 @@ public class Dmp implements Serializable
      * 
      */
     @JsonProperty("created")
+    @JsonDeserialize(using = MultiDateDeserializer.class)
     @JsonPropertyDescription("")
     private Date created;
     /**
@@ -147,6 +150,7 @@ public class Dmp implements Serializable
      * 
      */
     @JsonProperty("modified")
+    @JsonDeserialize(using = MultiDateDeserializer.class)
     @JsonPropertyDescription("Must be set each time DMP is modified. Indicates DMP version.")
     private Date modified;
     /**

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java

@@ -20,6 +20,10 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import javax.transaction.Transactional;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -152,8 +156,13 @@ public class DatasetRDAMapper {
 			templateIdsToValues.entrySet().forEach(entry -> {
 				boolean isFound = foundNodes.stream().anyMatch(node -> node.get("id").asText().equals(entry.getKey()));
 				if (!isFound && entry.getValue() != null && !entry.getValue().toString().isEmpty()) {
+					try {
+						Instant time = Instant.parse(entry.getValue().toString());
+						rda.setAdditionalProperty(entry.getKey(), DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()).format(time));
+					} catch (DateTimeParseException e) {
 						rda.setAdditionalProperty(entry.getKey(), entry.getValue());
 					}
+				}
 			});
 
 

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java

@@ -39,9 +39,9 @@ public class DmpRDAMapper {
 			throw new IllegalArgumentException("DMP is missing language and contact properties");
 		} else {
 			extraProperties = new org.json.JSONObject(dmp.getExtraProperties()).toMap();
-			if (extraProperties.get("language") == null) {
+			/*if (extraProperties.get("language") == null) {
 				throw new IllegalArgumentException("DMP must have it's language property defined");
-			}
+			}*/
 			if (extraProperties.get("contact") == null) {
 				throw new IllegalArgumentException("DMP must have it's contact property defined");
 			}
@@ -72,7 +72,7 @@ public class DmpRDAMapper {
 			} else {
 				rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN);
 			}
-			rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(extraProperties.get("language").toString()));
+			rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(extraProperties.get("language") != null ? extraProperties.get("language").toString() : "en"));
 			if (extraProperties.get("costs") != null) {
 				rda.setCost(new ArrayList<>());
 				((List) extraProperties.get("costs")).forEach(costl -> {
@@ -95,7 +95,7 @@ public class DmpRDAMapper {
 			rda.getContributor().addAll(dmp.getResearchers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList()));
 		}
 //		rda.getContributor().addAll(dmp.getUsers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList()));
-		rda.setDataset(dmp.getDataset().stream().map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor())).collect(Collectors.toList()));
+		rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor())).collect(Collectors.toList()));
 		rda.setProject(Collections.singletonList(ProjectRDAMapper.toRDA(dmp.getProject(), dmp.getGrant())));
 		rda.setAdditionalProperty("templates", dmp.getAssociatedDmps().stream().map(datasetProfile -> datasetProfile.getId().toString()).toArray());
 		return rda;
@@ -127,9 +127,11 @@ public class DmpRDAMapper {
 		entity.setDescription(rda.getDescription());
 		DatasetProfile defaultProfile = ((DatasetProfile)entity.getAssociatedDmps().toArray()[0]);
 		entity.setDataset(rda.getDataset().stream().map(rda1 -> datasetRDAMapper.toEntity(rda1, defaultProfile)).collect(Collectors.toSet()));
+		if (rda.getProject().size() > 0) {
 			Map<String, Object> result = ProjectRDAMapper.toEntity(rda.getProject().get(0), apiContext);
 			entity.setProject((Project) result.get("project"));
 			result.entrySet().stream().filter(entry -> entry.getKey().startsWith("grant")).forEach(entry -> entity.setGrant((Grant) entry.getValue()));
+		}
 
 		Map<String, Object> extraProperties = new HashMap<>();
 		extraProperties.put("language", LanguageRDAMapper.mapRDAIsoToLanguageIso(rda.getLanguage()));

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java

@@ -17,9 +17,11 @@ public class KeywordRDAMapper {
 		ObjectMapper mapper = new ObjectMapper();
 		try {
 			value = JavaToJson.objectStringToJson(value);
+			if (!value.isEmpty()) {
 				List<Tag> tags = Arrays.asList(mapper.readValue(value, Tag[].class));
 				List<String> keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList());
 				return keywordNames;
+			}
 		} catch (IOException e) {
 			logger.error(e.getMessage(), e);
 		}

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LanguageRDAMapper.java

@@ -1,28 +1,29 @@
 package eu.eudat.models.rda.mapper;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.models.rda.Language;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
-import java.util.stream.Collectors;
 
 public class LanguageRDAMapper {
 	private final static Map<String, Object> langMap = new HashMap<>();
 	private static final Logger logger = LoggerFactory.getLogger(LanguageRDAMapper.class);
 
 	static {
-		String json = null;
 		try {
-			json = new BufferedReader(new InputStreamReader(LanguageRDAMapper.class.getClassLoader().getResource("internal/rda-lang-map.json").openStream(), StandardCharsets.UTF_8))
-					.lines().collect(Collectors.joining("\n"));
-			langMap.putAll(new org.json.JSONObject(json).toMap());
+			ObjectMapper mapper = new ObjectMapper();
+			InputStreamReader isr = new InputStreamReader(LanguageRDAMapper.class.getClassLoader().getResource("internal/rda-lang-map.json").openStream(), StandardCharsets.UTF_8);
+			langMap.putAll(mapper.readValue(isr, LinkedHashMap.class));
+			isr.close();
+
 		} catch (IOException e) {
 			logger.error(e.getMessage(), e);
 		}