Merge remote-tracking branch 'origin/Development' into Development

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java

@@ -131,7 +131,7 @@ public class WordBuilder {
         this.options.put(ParagraphStyle.COMMENT, (mainDocumentPart, item) -> {
             XWPFParagraph paragraph = mainDocumentPart.createParagraph();
             XWPFRun run = paragraph.createRun();
-            run.setText(item);
+            run.setText("          " + item);
             run.setItalic(true);
             return paragraph;
         });
@@ -178,7 +178,7 @@ public class WordBuilder {
                     number.setVal(BigInteger.valueOf(indent));
                     paragraphPos = mainDocumentPart.getPosOfParagraph(paragraph);
                 }
-                createSections(section.getSections(), mainDocumentPart, ParagraphStyle.HEADER5, 1, createListing, visibilityRuleService, page, tempSectionString);
+                createSections(section.getSections(), mainDocumentPart, ParagraphStyle.HEADER4, 1, createListing, visibilityRuleService, page, tempSectionString);
                 hasValue = createCompositeFields(section.getCompositeFields(), mainDocumentPart, 2, createListing, visibilityRuleService, page, tempSectionString);
 
                 if (!hasValue && paragraphPos > -1) {
@@ -193,17 +193,25 @@ public class WordBuilder {
         boolean hasValue = false;
         for (FieldSet compositeField: compositeFields) {
             if (visibilityRuleService.isElementVisible(compositeField.getId()) && hasVisibleFields(compositeField, visibilityRuleService)) {
+                char c = 'a';
                 int paragraphPos = -1;
                 if (compositeField.getTitle() != null && !compositeField.getTitle().isEmpty() && !createListing) {
                     XWPFParagraph paragraph = addParagraphContent(page + "." + section + "." + (compositeField.getOrdinal() +1) + " " + compositeField.getTitle(), mainDocumentPart, ParagraphStyle.HEADER6, numId);
                     CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl();
                     number.setVal(BigInteger.valueOf(indent));
                     paragraphPos = mainDocumentPart.getPosOfParagraph(paragraph);
+                    if(compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()){
+                        addParagraphContent(c + ".\n", mainDocumentPart, ParagraphStyle.HEADER6, numId);
+                    }
                 }
                 hasValue = createFields(compositeField.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService);
                 if (compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()) {
                     List<FieldSet> list = compositeField.getMultiplicityItems().stream().sorted(Comparator.comparingInt(FieldSet::getOrdinal)).collect(Collectors.toList());
                     for (FieldSet multiplicityFieldset : list) {
+                        if(!createListing){
+                            c++;
+                            addParagraphContent(c + ".\n", mainDocumentPart, ParagraphStyle.HEADER6, numId);
+                        }
                         hasValue = createFields(multiplicityFieldset.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService);
                     }
                 }
@@ -224,13 +232,31 @@ public class WordBuilder {
         if (createListing) this.addListing(mainDocumentPart, indent, false, false);
         boolean hasValue = false;
         List<Field> tempFields = fields.stream().sorted(Comparator.comparingInt(Field::getOrdinal)).collect(Collectors.toList());
+        List<Field> formats = tempFields.stream().filter(f -> {
+            try {
+                String fTemp = this.formatter(f);
+                return fTemp != null && !fTemp.isEmpty();
+            } catch (IOException e) {
+                logger.error(e.getMessage(), e);
+            }
+            return false;
+        }).collect(Collectors.toList());
         for (Field field: tempFields) {
             if (visibilityRuleService.isElementVisible(field.getId())) {
                 if (!createListing) {
                     try {
                         if (field.getValue() != null && !field.getValue().toString().isEmpty()) {
                             this.indent = indent;
-                            XWPFParagraph paragraph = addParagraphContent(this.formatter(field), mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId);
+                            String format = this.formatter(field);
+                            if(format != null){
+                                if(format.charAt(0) == '['){
+                                    format = format.substring(1, format.length() - 1).replaceAll(",", ", ");
+                                }
+                                if(formats.size() > 1){
+                                    format = "\t• " + format;
+                                }
+                            }
+                            XWPFParagraph paragraph = addParagraphContent(format, mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId);
                             if (paragraph != null) {
                                 CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl();
                                 number.setVal(BigInteger.valueOf(indent));

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java

@@ -4,10 +4,10 @@ public class JavaToJson {
 
 	public static String objectStringToJson(String object) {
 		String result = object.replaceAll("=", "\":\"")
-				.replaceAll("\\{", "{\"")
+				//.replaceAll("\\{", "{\"")
 				.replaceAll(", ", "\", \"")
-				.replaceAll("}", "\"}" ).
+				//.replaceAll("}", "\"}" ).
-				replaceAll("}\", \"\\{", "}, {");
+				.replaceAll("}\", \"\\{", "}, {");
 		return result;
 	}
 }

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java

@@ -1,10 +1,19 @@
 package eu.eudat.models.rda.mapper;
 
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import eu.eudat.logic.utilities.json.JavaToJson;
 import eu.eudat.models.rda.Cost;
+import eu.eudat.models.rda.PidSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class CostRDAMapper {
+	private static final Logger logger = LoggerFactory.getLogger(DatasetRDAMapper.class);
 	
 	public static Cost toRDA(Map<String, Object> cost) {
 		Cost rda = new Cost();
@@ -19,4 +28,69 @@ public class CostRDAMapper {
 		return rda;
 	}
 
+	public static List<Cost> toRDAList(List<JsonNode> nodes) throws JsonProcessingException {
+		Map<String, Cost> rdaMap = new HashMap<>();
+		for(JsonNode node: nodes){
+			String rdaProperty = node.get("rdaProperty").asText();
+			String rdaValue = node.get("value").asText();
+			if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){
+				continue;
+			}
+			String key = node.get("numbering").asText();
+			if(!key.contains("mult")){
+				key = "0";
+			}
+			else{
+				key = "" + key.charAt(4);
+			}
+			Cost rda;
+			if(rdaMap.containsKey(key)){
+				rda = rdaMap.get(key);
+			}
+			else{
+				rda = new Cost();
+				rdaMap.put(key, rda);
+			}
+			if(rdaProperty.contains("value")){
+				rda.setValue(Double.valueOf(rdaValue));
+			}
+			else if(rdaProperty.contains("currency_code")){
+				String json = JavaToJson.objectStringToJson(rdaValue);
+				HashMap<String,String> result =
+						new ObjectMapper().readValue(json, HashMap.class);
+				rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value")));
+			}
+			else if(rdaProperty.contains("title")){
+				Iterator<JsonNode> iter = node.get("value").elements();
+				StringBuilder title = new StringBuilder();
+				while(iter.hasNext()){
+					String next = iter.next().asText();
+					if(!next.equals("Other")) {
+						title.append(next).append(", ");
+					}
+				}
+				if(title.length() > 2){
+					rda.setTitle(title.substring(0, title.length() - 2));
+				}
+				else{
+					String t = rda.getTitle();
+					if(t == null){ // only other as title
+						rda.setTitle(rdaValue);
+					}
+					else{ // option + other
+						rda.setTitle(t + ", " + rdaValue);
+					}
+				}
+			}
+			else if(rdaProperty.contains("description")){
+				rda.setDescription(rdaValue);
+			}
+		}
+		List<Cost> rdaList = rdaMap.values().stream()
+				.filter(cost -> cost.getTitle() != null)
+				.collect(Collectors.toList());
+		return rdaList;
+
+	}
+
 }

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java

@@ -9,10 +9,7 @@ import eu.eudat.logic.services.ApiContext;
 import eu.eudat.logic.utilities.json.JsonSearcher;
 import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel;
 import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
-import eu.eudat.models.rda.Contributor;
+import eu.eudat.models.rda.*;
-import eu.eudat.models.rda.Dataset;
-import eu.eudat.models.rda.DatasetId;
-import eu.eudat.models.rda.Language;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -20,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import javax.transaction.Transactional;
+import java.net.URI;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
@@ -43,7 +41,7 @@ public class DatasetRDAMapper {
 	}
 
 	@Transactional
-	public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, List<Contributor> contributors) {
+	public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, eu.eudat.models.rda.Dmp dmp) {
 		Dataset rda = new Dataset();
 //		rda.setDatasetId(DatasetIdRDAMapper.toRDA(dataset.getId()));
 		if (dataset.getLabel() == null) {
@@ -128,7 +126,11 @@ public class DatasetRDAMapper {
 			}
 			List<JsonNode> personalDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.personal_data");
 			if (!personalDataNodes.isEmpty()) {
+				try{
 					rda.setPersonalData(personalDataNodes.stream().map(personalDataNode -> Dataset.PersonalData.fromValue(personalDataNode.get("value").asText())).findFirst().get());
+				}catch(IllegalArgumentException e){
+					rda.setPersonalData(Dataset.PersonalData.UNKNOWN);
+				}
 			} else {
 				rda.setPersonalData(Dataset.PersonalData.UNKNOWN);
 			}
@@ -140,7 +142,11 @@ public class DatasetRDAMapper {
 			}
 			List<JsonNode> sensitiveDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.sensitive_data");
 			if (!sensitiveDataNodes.isEmpty()) {
+				try{
 					rda.setSensitiveData(sensitiveDataNodes.stream().map(sensitiveDataNode -> Dataset.SensitiveData.fromValue(sensitiveDataNode.get("value").asText())).findFirst().get());
+				}catch(IllegalArgumentException e){
+					rda.setSensitiveData(Dataset.SensitiveData.UNKNOWN);
+				}
 			} else {
 				rda.setSensitiveData(Dataset.SensitiveData.UNKNOWN);
 			}
@@ -156,15 +162,58 @@ public class DatasetRDAMapper {
 			}
 			List<JsonNode> contributorNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.contributor");
 			if (!contributorNodes.isEmpty()) {
-				contributors.addAll(contributorNodes.stream().map(contributorNode -> {
+				dmp.getContributor().addAll(contributorNodes.stream().map(contributorNode -> {
 					JsonNode value = contributorNode.get("value");
 					if (value.isArray()) {
 						return StreamSupport.stream(value.spliterator(), false).map(node -> ContributorRDAMapper.toRDA(node.asText())).collect(Collectors.toList());
 					} else {
-						return Collections.singletonList(new Contributor());
+						return Collections.singletonList(new Contributor()); // return null kalutera
 					}
 				}).flatMap(Collection::stream).collect(Collectors.toList()));
 			}
+			List<JsonNode> costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.cost");
+			if (!costNodes.isEmpty()) {
+				dmp.getCost().addAll(CostRDAMapper.toRDAList(costNodes));
+			}
+			List<JsonNode> ethicsNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.ethical_issues");
+			if (!ethicsNodes.isEmpty()) {
+				for(JsonNode node: ethicsNodes){
+					String rdaProperty = node.get("rdaProperty").asText();
+					String rdaValue = node.get("value").asText();
+					if(rdaValue == null || rdaValue.isEmpty()){
+						continue;
+					}
+					if(rdaProperty.contains("exist")){
+						try {
+							Dmp.EthicalIssuesExist exists = dmp.getEthicalIssuesExist();
+							if(exists == null
+							   || ((exists == Dmp.EthicalIssuesExist.NO || exists == Dmp.EthicalIssuesExist.UNKNOWN) && rdaValue.equals("yes"))
+							   || (exists == Dmp.EthicalIssuesExist.YES && !(rdaValue.equals("no") || rdaValue.equals("unknown")))
+							   || (exists == Dmp.EthicalIssuesExist.UNKNOWN && rdaValue.equals("no"))){
+								dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(rdaValue));
+							}
+						}catch(IllegalArgumentException e){
+							logger.warn(e.getLocalizedMessage() + ". Setting ethical_issues_exist to unknown");
+							dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN);
+						}
+					}
+					else if(rdaProperty.contains("description")){
+						if(dmp.getEthicalIssuesDescription() == null){
+							dmp.setEthicalIssuesDescription(rdaValue);
+						}
+						else{
+							dmp.setEthicalIssuesDescription(dmp.getEthicalIssuesDescription() + ", " + rdaValue);
+						}
+					}
+					else if(rdaProperty.contains("report")){
+						try {
+							dmp.setEthicalIssuesReport(URI.create(rdaValue));
+						} catch (IllegalArgumentException e) {
+							logger.warn(e.getLocalizedMessage() + ". Skipping url parsing");
+						}
+					}
+				}
+			}
 			List<JsonNode> foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes,
 					keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).collect(Collectors.toList());
 			templateIdsToValues.entrySet().forEach(entry -> {

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java

@@ -1,7 +1,11 @@
 package eu.eudat.models.rda.mapper;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.eudat.logic.utilities.helpers.MyStringUtils;
+import eu.eudat.logic.utilities.json.JavaToJson;
 import eu.eudat.logic.utilities.json.JsonSearcher;
 import eu.eudat.models.rda.Distribution;
 import eu.eudat.models.rda.License;
@@ -21,7 +25,8 @@ public class DistributionRDAMapper {
 		for (JsonNode node: nodes) {
 			String rdaProperty = node.get("rdaProperty").asText();
 			String rdaValue = node.get("value").asText();
-			if(rdaValue == null || rdaValue.isEmpty()){
+			//if(rdaValue == null || rdaValue.isEmpty()){
+			if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){
 				continue;
 			}
 			String key = node.get("numbering").asText();
@@ -78,7 +83,25 @@ public class DistributionRDAMapper {
 							rda.setLicense(license != null? Collections.singletonList(license): new ArrayList<>());
 							break;
 						case FORMAT:
+							if(node.get("value").isArray()){
+								Iterator<JsonNode> iter = node.get("value").elements();
+								List<String> formats = new ArrayList<>();
+								while(iter.hasNext()) {
+									String format = JavaToJson.objectStringToJson(iter.next().asText());
+									try {
+										Map<String, String> result = new ObjectMapper().readValue(format, HashMap.class);
+										format = result.get("label");
+										formats.add(format);
+									}
+									catch(JsonProcessingException e){
+										logger.warn(e.getMessage());
+									}
+								}
+								rda.setFormat(formats);
+							}
+							else{
 								rda.setFormat(new ArrayList<>(Arrays.asList(rdaValue.replace(" ", "").split(","))));
+							}
 							rda.setAdditionalProperty(ImportPropertyName.FORMAT.getName(), node.get("id").asText());
 							break;
 						case TITLE:

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java

@@ -68,7 +68,7 @@ public class DmpRDAMapper {
 
 		if (!extraProperties.isEmpty()) {
 			if (extraProperties.get("ethicalIssues") != null) {
-				rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(extraProperties.get("ethicalIsses").toString()));
+				rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(extraProperties.get("ethicalIssues").toString()));
 			} else {
 				rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN);
 			}
@@ -95,7 +95,7 @@ public class DmpRDAMapper {
 			rda.getContributor().addAll(dmp.getResearchers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList()));
 		}
 //		rda.getContributor().addAll(dmp.getUsers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList()));
-		rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor())).collect(Collectors.toList()));
+		rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda)).collect(Collectors.toList()));
 		rda.setProject(Collections.singletonList(ProjectRDAMapper.toRDA(dmp.getProject(), dmp.getGrant())));
 		rda.setAdditionalProperty("templates", dmp.getAssociatedDmps().stream().map(datasetProfile -> datasetProfile.getId().toString()).toArray());
 		return rda;

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java

@@ -22,7 +22,7 @@ public class HostRDAMapper {
 			String rdaProperty = node.get("rdaProperty").asText();
 			if (rdaProperty.contains("host")) {
 				int firstDiff = MyStringUtils.getFirstDifference(numbering, node.get("numbering").asText());
-				if (firstDiff == -1 || firstDiff > 2) {
+				if (firstDiff == -1 || firstDiff >= 2) {
 					String rdaValue = node.get("value").asText();
 					if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){
 						continue;
@@ -67,7 +67,14 @@ public class HostRDAMapper {
 										while(iter.hasNext()) {
 											pList.add(iter.next().asText());
 										}
-										List<PidSystem> pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList());
+										List<PidSystem> pidList;
+										if(pList.size() == 0){
+											pidList = Arrays.stream(rdaValue.replaceAll("[\\[\"\\]]","").split(","))
+													.map(PidSystem::fromValue).collect(Collectors.toList());
+										}
+										else{
+											pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList());
+										}
 										rda.setPidSystem(pidList);
 										rda.setAdditionalProperty(ImportPropertyName.PID_SYSTEM.getName(), node.get("id").asText());
 									}

dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java

@@ -15,22 +15,18 @@ public class KeywordRDAMapper {
 	private static final Logger logger = LoggerFactory.getLogger(KeywordRDAMapper.class);
 
 	public static List<String> toRDA(String value) {
-		ObjectMapper mapper = new ObjectMapper();
-			value = JavaToJson.objectStringToJson(value);
 		if (!value.isEmpty()) {
 			try {
-					List<Tag> tags = Arrays.asList(mapper.readValue(value, Tag[].class));
+				ObjectMapper mapper = new ObjectMapper();
-					List<String> keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList());
+				String valueJson = JavaToJson.objectStringToJson(value);
-					return keywordNames;
+				List<Tag> tags = Arrays.asList(mapper.readValue(valueJson, Tag[].class));
+				return tags.stream().map(Tag::getName).collect(Collectors.toList());
 			} catch (JsonProcessingException e) {
 				logger.warn(e.getMessage() + ". Attempting to parse it as a String list.");
-					if(!value.isEmpty()) {
 				return new ArrayList<>(Arrays.asList(value.replace(" ", "").split(",")));
 			}
 		}
 
-			}
-
 		return new ArrayList<>();
 	}
 }

dmp-backend/web/src/main/resources/RDACommonStandards.txt

@@ -34,12 +34,49 @@ dataset.security_and_privacy.description
 dataset.security_and_privacy.title
 dataset.sensitive_data
 dataset.technical_resource.description
-dataset.technical_resource.technical_resource
+dataset.technical_resource.name
-dataset.technical_resource.technical_resource.description
+dataset.title
-dataset.technical_resource.technical_resource.name
 dataset.type
 dataset.issued
 dataset.dataset_id
 dataset.dataset_id.identifier
 dataset.dataset_id.type
+dataset.description
+dmp.contact
+dmp.contact.contact_id.identifier
+dmp.contact.contact_id.type
+dmp.contact.mbox
+dmp.contact.name
 dmp.contributor
+dmp.contributor.contributor_id.identifier
+dmp.contributor.contributor_id.type
+dmp.contributor.mbox
+dmp.contributor.name
+dmp.contributor.role
+dmp.cost
+dmp.cost.currency_code
+dmp.cost.description
+dmp.cost.title
+dmp.cost.value
+dmp.created
+dmp.description
+dmp.dmp_id
+dmp.dmp_id.identifier
+dmp.dmp_id.type
+dmp.ethical_issues_description
+dmp.ethical_issues_exist
+dmp.ethical_issues_report
+dmp.language
+dmp.modified
+dmp.project
+dmp.project.description
+dmp.project.end
+dmp.project.funding
+dmp.project.funding.funder_id.identifier
+dmp.project.funding.funder_id.type
+dmp.project.funding.funding_status
+dmp.project.funding.grant_id.identifier
+dmp.project.funding.grant_id.type
+dmp.project.start
+dmp.project.title
+dmp.title

dmp-backend/web/src/main/resources/config/application-devel.properties

@@ -19,7 +19,7 @@ elasticsearch.index=dmps
 http-logger.server-address = http://localhost:31311
 
 ####################PDF OVERRIDES CONFIGURATIONS##########
-pdf.converter.url=http://localhost:88/
+pdf.converter.url=http://localhost:3000/
 
 ####################CONFIGURATION FILES OVERRIDES CONFIGURATIONS##########
 configuration.externalUrls=externalUrls/ExternalUrls.xml