From e675ad7c9d02cd7062fb3b7729d23e4a89cfb4a4 Mon Sep 17 00:00:00 2001 From: "DESKTOP-4ES9U2E\\aldom" Date: Wed, 29 Dec 2021 15:24:46 +0200 Subject: [PATCH 1/3] madmp changes --- .../utilities/documents/word/WordBuilder.java | 10 +- .../logic/utilities/json/JavaToJson.java | 6 +- .../java/eu/eudat/models/rda/Contact.java | 4 +- .../java/eu/eudat/models/rda/ContactId.java | 4 +- .../java/eu/eudat/models/rda/Contributor.java | 4 +- .../eu/eudat/models/rda/ContributorId.java | 4 +- .../main/java/eu/eudat/models/rda/Cost.java | 4 +- .../java/eu/eudat/models/rda/DatasetId.java | 4 +- .../main/java/eu/eudat/models/rda/DmpId.java | 4 +- .../java/eu/eudat/models/rda/FunderId.java | 4 +- .../java/eu/eudat/models/rda/Funding.java | 4 +- .../main/java/eu/eudat/models/rda/Host.java | 4 +- .../java/eu/eudat/models/rda/License.java | 4 +- .../eudat/models/rda/MetadataStandardId.java | 4 +- .../java/eu/eudat/models/rda/Project.java | 4 +- .../models/rda/mapper/CostRDAMapper.java | 76 +++++++++- .../models/rda/mapper/DatasetIdRDAMapper.java | 28 ++-- .../models/rda/mapper/DatasetRDAMapper.java | 43 ++++-- .../rda/mapper/DistributionRDAMapper.java | 65 +++++++-- .../eudat/models/rda/mapper/DmpRDAMapper.java | 2 +- .../models/rda/mapper/HostRDAMapper.java | 138 ++++++++++-------- .../models/rda/mapper/KeywordRDAMapper.java | 24 +-- .../models/rda/mapper/LicenseRDAMapper.java | 24 ++- .../mapper/SecurityAndPrivacyRDAMapper.java | 11 +- .../mapper/TechnicalResourceRDAMapper.java | 11 +- .../src/main/resources/RDACommonStandards.txt | 45 +++++- 26 files changed, 383 insertions(+), 152 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java index 97eb201db..78c38db09 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java @@ -131,7 +131,7 @@ public class WordBuilder { this.options.put(ParagraphStyle.COMMENT, (mainDocumentPart, item) -> { XWPFParagraph paragraph = mainDocumentPart.createParagraph(); XWPFRun run = paragraph.createRun(); - run.setText(item); + run.setText(" " + item); run.setItalic(true); return paragraph; }); @@ -178,7 +178,7 @@ public class WordBuilder { number.setVal(BigInteger.valueOf(indent)); paragraphPos = mainDocumentPart.getPosOfParagraph(paragraph); } - createSections(section.getSections(), mainDocumentPart, ParagraphStyle.HEADER5, 1, createListing, visibilityRuleService, page, tempSectionString); + createSections(section.getSections(), mainDocumentPart, ParagraphStyle.HEADER4, 1, createListing, visibilityRuleService, page, tempSectionString); hasValue = createCompositeFields(section.getCompositeFields(), mainDocumentPart, 2, createListing, visibilityRuleService, page, tempSectionString); if (!hasValue && paragraphPos > -1) { @@ -230,7 +230,11 @@ public class WordBuilder { try { if (field.getValue() != null && !field.getValue().toString().isEmpty()) { this.indent = indent; - XWPFParagraph paragraph = addParagraphContent(this.formatter(field), mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId); + String format = this.formatter(field); + if(format != null && format.charAt(0) == '['){ + format = format.substring(1, format.length() - 1).replaceAll(",", ", "); + } + XWPFParagraph paragraph = addParagraphContent(format, mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId); if (paragraph != null) { CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl(); number.setVal(BigInteger.valueOf(indent)); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java index 8338c1526..aa365fe0e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java @@ -4,10 +4,10 @@ public class JavaToJson { public static String objectStringToJson(String object) { String result = object.replaceAll("=", "\":\"") - .replaceAll("\\{", "{\"") + //.replaceAll("\\{", "{\"") .replaceAll(", ", "\", \"") - .replaceAll("}", "\"}" ). - replaceAll("}\", \"\\{", "}, {"); + //.replaceAll("}", "\"}" ). + .replaceAll("}\", \"\\{", "}, {"); return result; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java index 606a3ae2c..bbd5e40f9 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java @@ -133,12 +133,12 @@ public class Contact implements Serializable this.name = name; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java index 60454ea6c..11bd8b54e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java @@ -100,12 +100,12 @@ public class ContactId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java index 21dfe68c8..c53c2551d 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java @@ -168,12 +168,12 @@ public class Contributor implements Serializable this.role = role; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java index 4045f5e0e..c92a5227a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java @@ -101,12 +101,12 @@ public class ContributorId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java index db32ff641..ee5b3289f 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java @@ -162,12 +162,12 @@ public class Cost implements Serializable this.value = value; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java index 6c1219c1c..02108950b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java @@ -109,12 +109,12 @@ public class DatasetId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java index 7240efaa4..d277fbdd9 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java @@ -101,12 +101,12 @@ public class DmpId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java index 12b8bb8f7..4033f415c 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java @@ -101,12 +101,12 @@ public class FunderId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java index 0cc378e7c..0b8aa8c53 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java @@ -133,12 +133,12 @@ public class Funding implements Serializable this.grantId = grantId; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java index bfa6ec615..2227460df 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java @@ -391,12 +391,12 @@ public class Host implements Serializable this.url = url; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java index 1527f5503..e622e14cc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java @@ -100,12 +100,12 @@ public class License implements Serializable this.startDate = startDate; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java index 06b54a1f1..a693f34a5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java @@ -101,12 +101,12 @@ public class MetadataStandardId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java index 09c231ca3..4854fa609 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java @@ -199,12 +199,12 @@ public class Project implements Serializable this.title = title; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java index 3d93d0768..942f48d4a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java @@ -1,10 +1,19 @@ package eu.eudat.models.rda.mapper; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.models.rda.Cost; +import eu.eudat.models.rda.PidSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CostRDAMapper { + private static final Logger logger = LoggerFactory.getLogger(DatasetRDAMapper.class); public static Cost toRDA(Map cost) { Cost rda = new Cost(); @@ -19,4 +28,69 @@ public class CostRDAMapper { return rda; } + public static List toRDAList(List nodes) throws JsonProcessingException { + Map rdaMap = new HashMap<>(); + for(JsonNode node: nodes){ + String rdaProperty = node.get("rdaProperty").asText(); + String rdaValue = node.get("value").asText(); + if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){ + continue; + } + String key = node.get("numbering").asText(); + if(!key.contains("mult")){ + key = "0"; + } + else{ + key = "" + key.charAt(4); + } + Cost rda; + if(rdaMap.containsKey(key)){ + rda = rdaMap.get(key); + } + else{ + rda = new Cost(); + rdaMap.put(key, rda); + } + if(rdaProperty.contains("value")){ + rda.setValue(Double.valueOf(rdaValue)); + } + else if(rdaProperty.contains("currency_code")){ + String json = JavaToJson.objectStringToJson(rdaValue); + HashMap result = + new ObjectMapper().readValue(json, HashMap.class); + rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value"))); + } + else if(rdaProperty.contains("title")){ + Iterator iter = node.get("value").elements(); + StringBuilder title = new StringBuilder(); + while(iter.hasNext()){ + String next = iter.next().asText(); + if(!next.equals("Other")) { + title.append(next).append(", "); + } + } + if(title.length() > 2){ + rda.setTitle(title.substring(0, title.length() - 2)); + } + else{ + String t = rda.getTitle(); + if(t == null){ // only other as title + rda.setTitle(rdaValue); + } + else{ // option + other + rda.setTitle(t + ", " + rdaValue); + } + } + } + else if(rdaProperty.contains("description")){ + rda.setDescription(rdaValue); + } + } + List rdaList = rdaMap.values().stream() + .filter(cost -> cost.getTitle() != null) + .collect(Collectors.toList()); + return rdaList; + + } + } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java index a97c58e6a..6d87a10e4 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java @@ -29,19 +29,23 @@ public class DatasetIdRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); - - ObjectMapper mapper = new ObjectMapper(); - try { - Map values = mapper.readValue(rdaValue, HashMap.class); - if (!values.isEmpty()) { - values.entrySet().forEach(entry -> finalRDAMap(data, entry.getKey(), (String) entry.getValue())); - } else { - finalRDAMap(data, rdaProperty, rdaValue); - } - } catch (IOException e) { - finalRDAMap(data, rdaProperty, rdaValue); - logger.error(e.getMessage(), e); + if(rdaValue.isEmpty()){ + continue; } + finalRDAMap(data, rdaProperty, rdaValue); + +// ObjectMapper mapper = new ObjectMapper(); +// try { +// Map values = mapper.readValue(rdaValue, HashMap.class); +// if (!values.isEmpty()) { +// values.entrySet().forEach(entry -> finalRDAMap(data, entry.getKey(), (String) entry.getValue())); +// } else { +// finalRDAMap(data, rdaProperty, rdaValue); +// } +// } catch (IOException e) { +// finalRDAMap(data, rdaProperty, rdaValue); +// logger.error(e.getMessage(), e); +// } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java index 67858a14b..fda0ea8ec 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java @@ -9,10 +9,7 @@ import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel; import eu.eudat.models.data.datasetwizard.DatasetWizardModel; -import eu.eudat.models.rda.Contributor; -import eu.eudat.models.rda.Dataset; -import eu.eudat.models.rda.DatasetId; -import eu.eudat.models.rda.Language; +import eu.eudat.models.rda.*; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +40,7 @@ public class DatasetRDAMapper { } @Transactional - public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, List contributors) { + public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, List contributors, List costs) { Dataset rda = new Dataset(); // rda.setDatasetId(DatasetIdRDAMapper.toRDA(dataset.getId())); if (dataset.getLabel() == null) { @@ -68,13 +65,13 @@ public class DatasetRDAMapper { rda.setDatasetId(new DatasetId(dataset.getId().toString(), DatasetId.Type.OTHER)); } List typeNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.type"); - if (!typeNodes.isEmpty()) { + if (!typeNodes.isEmpty() && !typeNodes.get(0).get("value").asText().isEmpty()) { rda.setType(typeNodes.get(0).get("value").asText()); } else { rda.setType("DMP Dataset"); } List languageNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.language"); - if (!languageNodes.isEmpty()) { + if (!languageNodes.isEmpty() && !languageNodes.get(0).get("value").asText().isEmpty()) { rda.setLanguage(Language.fromValue(languageNodes.get(0).get("value").asText())); } else { rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(dataset.getProfile().getLanguage())); @@ -83,21 +80,33 @@ public class DatasetRDAMapper { if (!metadataNodes.isEmpty()) { rda.setMetadata(MetadataRDAMapper.toRDAList(metadataNodes)); } + else{ + rda.setMetadata(new ArrayList<>()); + } List qaNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.data_quality_assurance"); if (!qaNodes.isEmpty()) { + List qaList = qaNodes.stream().map(qaNode -> qaNode.get("value").asText()).filter(qaNode -> !qaNode.isEmpty()).collect(Collectors.toList()); + rda.setDataQualityAssurance(qaList); + /*rda.setDataQualityAssurance(qaNodes.stream().map(qaNode -> qaNode.get("value").asText()).collect(Collectors.toList())); for (int i = 0; i < qaNodes.size(); i++) { rda.setAdditionalProperty("qaId" + (i + 1), qaNodes.get(i).get("id").asText()); }*/ - rda.setDataQualityAssurance(Collections.singletonList(qaNodes.get(0).get("value").asText())); + //rda.setDataQualityAssurance(Collections.singletonList(qaNodes.get(0).get("value").asText())); + } + else{ + rda.setDataQualityAssurance(new ArrayList<>()); } List preservationNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.preservation_statement"); - if (!preservationNodes.isEmpty()) { + if (!preservationNodes.isEmpty() && !preservationNodes.get(0).get("value").asText().isEmpty()) { rda.setPreservationStatement(preservationNodes.get(0).get("value").asText()); } List distributionNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.distribution"); if (!distributionNodes.isEmpty()) { - rda.setDistribution(Collections.singletonList(DistributionRDAMapper.toRDA(distributionNodes))); + rda.setDistribution(DistributionRDAMapper.toRDAList(distributionNodes)); + } + else{ + rda.setDistribution(new ArrayList<>()); } List keywordNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.keyword"); if (!keywordNodes.isEmpty()) { @@ -126,6 +135,9 @@ public class DatasetRDAMapper { if (!securityAndPrivacyNodes.isEmpty()) { rda.setSecurityAndPrivacy(SecurityAndPrivacyRDAMapper.toRDAList(securityAndPrivacyNodes)); } + else{ + rda.setSecurityAndPrivacy(new ArrayList<>()); + } List sensitiveDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.sensitive_data"); if (!sensitiveDataNodes.isEmpty()) { rda.setSensitiveData(sensitiveDataNodes.stream().map(sensitiveDataNode -> Dataset.SensitiveData.fromValue(sensitiveDataNode.get("value").asText())).findFirst().get()); @@ -136,8 +148,11 @@ public class DatasetRDAMapper { if (!technicalResourceNodes.isEmpty()) { rda.setTechnicalResource(TechnicalResourceRDAMapper.toRDAList(technicalResourceNodes)); } + else{ + rda.setTechnicalResource(new ArrayList<>()); + } List issuedNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.issued"); - if (!issuedNodes.isEmpty()) { + if (!issuedNodes.isEmpty() && !issuedNodes.get(0).get("value").asText().isEmpty()) { rda.setIssued(issuedNodes.get(0).get("value").asText()); } List contributorNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.contributor"); @@ -147,10 +162,14 @@ public class DatasetRDAMapper { if (value.isArray()) { return StreamSupport.stream(value.spliterator(), false).map(node -> ContributorRDAMapper.toRDA(node.asText())).collect(Collectors.toList()); } else { - return Collections.singletonList(new Contributor()); + return Collections.singletonList(new Contributor()); // return null kalutera } }).flatMap(Collection::stream).collect(Collectors.toList())); } + List costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.cost"); + if (!costNodes.isEmpty()) { + costs.addAll(CostRDAMapper.toRDAList(costNodes)); + } List foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes, keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).collect(Collectors.toList()); templateIdsToValues.entrySet().forEach(entry -> { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java index c26f8df6a..11785d491 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java @@ -1,7 +1,10 @@ package eu.eudat.models.rda.mapper; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.logic.utilities.helpers.MyStringUtils; +import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.rda.Distribution; import org.slf4j.Logger; @@ -14,16 +17,34 @@ import java.util.stream.Collectors; public class DistributionRDAMapper { private static final Logger logger = LoggerFactory.getLogger(DistributionRDAMapper.class); - public static List toRDAList(List nodes) { + public static List toRDAList(List nodes) throws JsonProcessingException { Map rdaMap = new HashMap<>(); for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); - Distribution rda = getRelative(rdaMap, node.get("numbering").asText()); - if (!rdaMap.containsValue(rda)) { - rdaMap.put(node.get("numbering").asText(), rda); + if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){ // node.get("value").isNull() + continue; } + String key = node.get("numbering").asText(); + if(!key.contains("mult")){ + key = "0"; + } + else{ + key = "" + key.charAt(4); + } + Distribution rda; + if(rdaMap.containsKey(key)){ + rda = rdaMap.get(key); + } + else { + rda = new Distribution(); + rdaMap.put(key, rda); + } +// Distribution rda = getRelative(rdaMap, node.get("numbering").asText()); +// if (!rdaMap.containsValue(rda)) { +// rdaMap.put(node.get("numbering").asText(), rda); +// } for (ExportPropertyName exportPropertyName : ExportPropertyName.values()) { if (rdaProperty.contains(exportPropertyName.getName())) { switch (exportPropertyName) { @@ -40,8 +61,10 @@ public class DistributionRDAMapper { rda.setAdditionalProperty(ImportPropertyName.DOWNLOAD_URL.getName(), node.get("id").asText()); break; case DESCRIPTION: - rda.setDescription(rdaValue); - rda.setAdditionalProperty(ImportPropertyName.DESCRIPTION.getName(), node.get("id").asText()); + if(!rdaProperty.contains("host")) { + rda.setDescription(rdaValue); + rda.setAdditionalProperty(ImportPropertyName.DESCRIPTION.getName(), node.get("id").asText()); + } break; case DATA_ACCESS: rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue)); @@ -56,12 +79,29 @@ public class DistributionRDAMapper { rda.setLicense(Collections.singletonList(LicenseRDAMapper.toRDA(licenseNodes))); break; case FORMAT: - rda.setFormat(Collections.singletonList(rdaValue)); + //rda.setFormat(Collections.singletonList(rdaValue)); + if(node.get("value").isArray()){ + Iterator iter = node.get("value").elements(); + List formats = new ArrayList<>(); + while(iter.hasNext()) { + String format = JavaToJson.objectStringToJson(iter.next().asText()); + Map result = + new ObjectMapper().readValue(format, HashMap.class); + format = result.get("label"); + formats.add(format); + } + rda.setFormat(formats); + } + else{ + rda.setFormat(new ArrayList<>(Arrays.asList(rdaValue.replace(" ", "").split(",")))); + } rda.setAdditionalProperty(ImportPropertyName.FORMAT.getName(), node.get("id").asText()); break; case TITLE: - rda.setTitle(rdaValue); - rda.setAdditionalProperty(ImportPropertyName.TITLE.getName(), node.get("id").asText()); + if(!rdaProperty.contains("host")) { + rda.setTitle(rdaValue); + rda.setAdditionalProperty(ImportPropertyName.TITLE.getName(), node.get("id").asText()); + } break; case HOST: rda.setHost(HostRDAMapper.toRDA(nodes, node.get("numbering").asText())); @@ -71,7 +111,12 @@ public class DistributionRDAMapper { } } - return new ArrayList<>(rdaMap.values()); + List rdaList = rdaMap.values().stream() + .filter(distro -> distro.getTitle() != null) + .collect(Collectors.toList()); + + return rdaList; + //return new ArrayList<>(rdaMap.values()); } public static Map toProperties(List rdas) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java index 700f0bfda..d2abe28d6 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java @@ -95,7 +95,7 @@ public class DmpRDAMapper { rda.getContributor().addAll(dmp.getResearchers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); } // rda.getContributor().addAll(dmp.getUsers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); - rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor())).collect(Collectors.toList())); + rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor(), rda.getCost())).collect(Collectors.toList())); rda.setProject(Collections.singletonList(ProjectRDAMapper.toRDA(dmp.getProject(), dmp.getGrant()))); rda.setAdditionalProperty("templates", dmp.getAssociatedDmps().stream().map(datasetProfile -> datasetProfile.getId().toString()).toArray()); return rda; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java index d96aa3ca3..c52338e2a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java @@ -8,10 +8,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; public class HostRDAMapper { private static final Logger logger = LoggerFactory.getLogger(HostRDAMapper.class); @@ -19,11 +17,14 @@ public class HostRDAMapper { public static Host toRDA(List nodes, String numbering) { Host rda = new Host(); for (JsonNode node: nodes) { - String rdaProperty = node.get("rdaProperties").asText(); + String rdaProperty = node.get("rdaProperty").asText(); if (rdaProperty.contains("host")) { int firstDiff = MyStringUtils.getFirstDifference(numbering, node.get("numbering").asText()); - if (firstDiff == -1 || firstDiff > 2) { + if (firstDiff == -1 || firstDiff >= 2) { String rdaValue = node.get("value").asText(); + if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){ + continue; + } for (ExportPropertyName propertyName: ExportPropertyName.values()) { if (rdaProperty.contains(propertyName.getName())) { switch (propertyName) { @@ -51,8 +52,27 @@ public class HostRDAMapper { rda.setAdditionalProperty(ImportPropertyName.GEO_LOCATION.getName(), node.get("id").asText()); break; case PID_SYSTEM: - rda.setPidSystem(Collections.singletonList(PidSystem.fromValue(rdaValue))); - rda.setAdditionalProperty(ImportPropertyName.PID_SYSTEM.getName(), node.get("id").asText()); + try{ + Iterator iter = node.get("value").elements(); + List pList = new ArrayList<>(); + while(iter.hasNext()) { + pList.add(iter.next().asText()); + } + List pidList; + if(pList.size() == 0){ + pidList = Arrays.stream(rdaValue.replaceAll("[\\[\"\\]]","").split(",")) + .map(PidSystem::fromValue).collect(Collectors.toList()); + } + else{ + pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); + } + rda.setPidSystem(pidList); + rda.setAdditionalProperty(ImportPropertyName.PID_SYSTEM.getName(), node.get("id").asText()); + } + catch (IllegalArgumentException e){ + rda.setPidSystem(new ArrayList<>()); + break; + } break; case STORAGE_TYPE: rda.setStorageType(rdaValue); @@ -67,8 +87,12 @@ public class HostRDAMapper { rda.setAdditionalProperty(ImportPropertyName.TITLE.getName(), node.get("id").asText()); break; case URL: - rda.setUrl(URI.create(rdaValue)); - rda.setAdditionalProperty(ImportPropertyName.URL.getName(), node.get("id").asText()); + try { + rda.setUrl(URI.create(rdaValue)); + rda.setAdditionalProperty(ImportPropertyName.URL.getName(), node.get("id").asText()); + } catch (IllegalArgumentException e) { + logger.warn(e.getLocalizedMessage() + ". Skipping url parsing"); + } break; } } @@ -76,13 +100,9 @@ public class HostRDAMapper { } } } - - if (rda.getTitle() == null) { - throw new IllegalArgumentException("Host Title is missing"); - } - - if (rda.getUrl() == null) { - throw new IllegalArgumentException("Host Url is missing"); + + if(rda.getTitle() == null || rda.getUrl() == null){ + return null; } return rda; @@ -90,49 +110,49 @@ public class HostRDAMapper { public static Map toProperties(Host rda) { Map properties = new HashMap<>(); - rda.getAdditionalProperties().entrySet().forEach(entry -> { - try { - ImportPropertyName importPropertyName = ImportPropertyName.fromString(entry.getKey()); - switch (importPropertyName) { - case AVAILABILITY: - properties.put(entry.getValue().toString(), rda.getAvailability()); - break; - case TITLE: - properties.put(entry.getValue().toString(), rda.getTitle()); - break; - case DESCRIPTION: - properties.put(entry.getValue().toString(), rda.getDescription()); - break; - case BACKUP_FREQUENCY: - properties.put(entry.getValue().toString(), rda.getBackupFrequency()); - break; - case BACKUP_TYPE: - properties.put(entry.getValue().toString(), rda.getBackupType()); - break; - case CERTIFIED_WITH: - properties.put(entry.getValue().toString(), rda.getCertifiedWith().value()); - break; - case GEO_LOCATION: - properties.put(entry.getValue().toString(), rda.getGeoLocation().value()); - break; - case PID_SYSTEM: - properties.put(entry.getValue().toString(), rda.getPidSystem().get(0).value()); - break; - case STORAGE_TYPE: - properties.put(entry.getValue().toString(), rda.getStorageType()); - break; - case SUPPORT_VERSIONING: - properties.put(entry.getValue().toString(), rda.getSupportVersioning().value()); - break; - case URL: - properties.put(entry.getValue().toString(), rda.getUrl().toString()); - break; - } - } catch (Exception e) { - logger.error(e.getMessage(), e); + rda.getAdditionalProperties().entrySet().forEach(entry -> { + try { + ImportPropertyName importPropertyName = ImportPropertyName.fromString(entry.getKey()); + switch (importPropertyName) { + case AVAILABILITY: + properties.put(entry.getValue().toString(), rda.getAvailability()); + break; + case TITLE: + properties.put(entry.getValue().toString(), rda.getTitle()); + break; + case DESCRIPTION: + properties.put(entry.getValue().toString(), rda.getDescription()); + break; + case BACKUP_FREQUENCY: + properties.put(entry.getValue().toString(), rda.getBackupFrequency()); + break; + case BACKUP_TYPE: + properties.put(entry.getValue().toString(), rda.getBackupType()); + break; + case CERTIFIED_WITH: + properties.put(entry.getValue().toString(), rda.getCertifiedWith().value()); + break; + case GEO_LOCATION: + properties.put(entry.getValue().toString(), rda.getGeoLocation().value()); + break; + case PID_SYSTEM: + properties.put(entry.getValue().toString(), rda.getPidSystem().get(0).value()); + break; + case STORAGE_TYPE: + properties.put(entry.getValue().toString(), rda.getStorageType()); + break; + case SUPPORT_VERSIONING: + properties.put(entry.getValue().toString(), rda.getSupportVersioning().value()); + break; + case URL: + properties.put(entry.getValue().toString(), rda.getUrl().toString()); + break; } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } - }); + }); return properties; } @@ -193,4 +213,4 @@ public class HostRDAMapper { throw new Exception("No name available"); } } -} +} \ No newline at end of file diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java index 2e8a93d37..681373be7 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java @@ -14,18 +14,20 @@ public class KeywordRDAMapper { private static final Logger logger = LoggerFactory.getLogger(KeywordRDAMapper.class); public static List toRDA(String value) { - ObjectMapper mapper = new ObjectMapper(); - try { - value = JavaToJson.objectStringToJson(value); - if (!value.isEmpty()) { - List tags = Arrays.asList(mapper.readValue(value, Tag[].class)); - List keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList()); - return keywordNames; - } - } catch (IOException e) { - logger.error(e.getMessage(), e); +// ObjectMapper mapper = new ObjectMapper(); +// try { +// value = JavaToJson.objectStringToJson(value); +// if (!value.isEmpty()) { +// List tags = Arrays.asList(mapper.readValue(value, Tag[].class)); +// List keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList()); +// return keywordNames; +// } +// } catch (IOException e) { +// logger.error(e.getMessage(), e); +// } + if(!value.isEmpty()) { + return new ArrayList<>(Arrays.asList(value.replace(" ", "").split(","))); } - return new ArrayList<>(); } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java index 108f5eecb..d005882cc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java @@ -16,12 +16,16 @@ public class LicenseRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String value = node.get("value").asText(); - + if(value.equals(null)){ + continue; + } for (LicenceProperties licenceProperties: LicenceProperties.values()) { if (rdaProperty.contains(licenceProperties.getName())) { switch (licenceProperties) { case LICENSE_REF: - rda.setLicenseRef(URI.create(value)); + if(value.contains(":")){ // valid uri + rda.setLicenseRef(URI.create(value)); + } break; case START_DATE: rda.setStartDate(value); @@ -39,12 +43,16 @@ public class LicenseRDAMapper { }*/ } - if (rda.getLicenseRef() == null) { - throw new IllegalArgumentException("Licence Reference is missing"); - } - - if (rda.getStartDate() == null) { - throw new IllegalArgumentException("License Start Date is missing"); +// if (rda.getLicenseRef() == null) { +// throw new IllegalArgumentException("Licence Reference is missing"); +// } +// +// if (rda.getStartDate() == null) { +// throw new IllegalArgumentException("License Start Date is missing"); +// } + + if(rda.getLicenseRef() == null || rda.getStartDate() == null){ + return null; } return rda; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java index 6006910a1..f87b7ee71 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java @@ -2,11 +2,13 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; +import eu.eudat.models.rda.Distribution; import eu.eudat.models.rda.SecurityAndPrivacy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; public class SecurityAndPrivacyRDAMapper { private static final Logger logger = LoggerFactory.getLogger(SecurityAndPrivacyRDAMapper.class); @@ -17,6 +19,9 @@ public class SecurityAndPrivacyRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); + if(rdaValue == null || rdaValue.isEmpty()){ + continue; + } SecurityAndPrivacy rda = getRelative(rdaMap, node.get("numbering").asText()); if (!rdaMap.containsValue(rda)) { @@ -38,7 +43,11 @@ public class SecurityAndPrivacyRDAMapper { } } - return new ArrayList<>(rdaMap.values()); + List rdaList = rdaMap.values().stream() + .filter(sap -> sap.getTitle() != null) + .collect(Collectors.toList()); + return rdaList; + //return new ArrayList<>(rdaMap.values()); } public static Map toProperties(List rdas) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java index 5214a7518..9b029bedf 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java @@ -2,11 +2,13 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; +import eu.eudat.models.rda.SecurityAndPrivacy; import eu.eudat.models.rda.TechnicalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; public class TechnicalResourceRDAMapper { private static final Logger logger = LoggerFactory.getLogger(TechnicalResourceRDAMapper.class); @@ -17,6 +19,9 @@ public class TechnicalResourceRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); + if(rdaValue == null || rdaValue.isEmpty()){ + continue; + } TechnicalResource rda = getRelative(rdaMap, node.get("numbering").asText()); if (!rdaMap.containsValue(rda)) { @@ -38,7 +43,11 @@ public class TechnicalResourceRDAMapper { } } - return new ArrayList<>(rdaMap.values()); + List rdaList = rdaMap.values().stream() + .filter(tr -> tr.getName() != null) + .collect(Collectors.toList()); + return rdaList; + //return new ArrayList<>(rdaMap.values()); } public static Map toProperties(List rdas) { diff --git a/dmp-backend/web/src/main/resources/RDACommonStandards.txt b/dmp-backend/web/src/main/resources/RDACommonStandards.txt index d8693586b..03c8c0dcb 100644 --- a/dmp-backend/web/src/main/resources/RDACommonStandards.txt +++ b/dmp-backend/web/src/main/resources/RDACommonStandards.txt @@ -34,12 +34,49 @@ dataset.security_and_privacy.description dataset.security_and_privacy.title dataset.sensitive_data dataset.technical_resource.description -dataset.technical_resource.technical_resource -dataset.technical_resource.technical_resource.description -dataset.technical_resource.technical_resource.name +dataset.technical_resource.name +dataset.title dataset.type dataset.issued dataset.dataset_id dataset.dataset_id.identifier dataset.dataset_id.type -dmp.contributor \ No newline at end of file +dataset.description +dmp.contact +dmp.contact.contact_id.identifier +dmp.contact.contact_id.type +dmp.contact.mbox +dmp.contact.name +dmp.contributor +dmp.contributor.contributor_id.identifier +dmp.contributor.contributor_id.type +dmp.contributor.mbox +dmp.contributor.name +dmp.contributor.role +dmp.cost +dmp.cost.currency_code +dmp.cost.description +dmp.cost.title +dmp.cost.value +dmp.created +dmp.description +dmp.dmp_id +dmp.dmp_id.identifier +dmp.dmp_id.type +dmp.ethical_issues_description +dmp.ethical_issues_exist +dmp.ethical_issues_report +dmp.language +dmp.modified +dmp.project +dmp.project.description +dmp.project.end +dmp.project.funding +dmp.project.funding.funder_id.identifier +dmp.project.funding.funder_id.type +dmp.project.funding.funding_status +dmp.project.funding.grant_id.identifier +dmp.project.funding.grant_id.type +dmp.project.start +dmp.project.title +dmp.title \ No newline at end of file From 9d5d685ad689d37663dbe25c53512dc5dd021ccf Mon Sep 17 00:00:00 2001 From: "DESKTOP-4ES9U2E\\aldom" Date: Wed, 12 Jan 2022 14:10:00 +0200 Subject: [PATCH 2/3] ethical_issues added to DatasetRDAMapper, minor fixes to dmp docx export --- .../utilities/documents/word/WordBuilder.java | 28 ++++++++- .../models/rda/mapper/DatasetRDAMapper.java | 58 +++++++++++++++++-- .../eudat/models/rda/mapper/DmpRDAMapper.java | 4 +- .../models/rda/mapper/HostRDAMapper.java | 17 +++--- .../models/rda/mapper/KeywordRDAMapper.java | 24 ++++---- 5 files changed, 98 insertions(+), 33 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java index 78c38db09..ad3dc1d27 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java @@ -193,18 +193,26 @@ public class WordBuilder { boolean hasValue = false; for (FieldSet compositeField: compositeFields) { if (visibilityRuleService.isElementVisible(compositeField.getId()) && hasVisibleFields(compositeField, visibilityRuleService)) { + char c = 'a'; int paragraphPos = -1; if (compositeField.getTitle() != null && !compositeField.getTitle().isEmpty() && !createListing) { XWPFParagraph paragraph = addParagraphContent(page + "." + section + "." + (compositeField.getOrdinal() +1) + " " + compositeField.getTitle(), mainDocumentPart, ParagraphStyle.HEADER6, numId); CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl(); number.setVal(BigInteger.valueOf(indent)); paragraphPos = mainDocumentPart.getPosOfParagraph(paragraph); + if(compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()){ + addParagraphContent(c + ".\n", mainDocumentPart, ParagraphStyle.HEADER6, numId); + } } hasValue = createFields(compositeField.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService); if (compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()) { List
list = compositeField.getMultiplicityItems().stream().sorted(Comparator.comparingInt(FieldSet::getOrdinal)).collect(Collectors.toList()); for (FieldSet multiplicityFieldset : list) { - hasValue = createFields(multiplicityFieldset.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService); + if(!createListing){ + c++; + addParagraphContent(c + ".\n", mainDocumentPart, ParagraphStyle.HEADER6, numId); + } + hasValue = createFields(multiplicityFieldset.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService); } } if (hasValue && compositeField.getHasCommentField() && compositeField.getCommentFieldValue() != null && !compositeField.getCommentFieldValue().isEmpty() && !createListing) { @@ -224,6 +232,15 @@ public class WordBuilder { if (createListing) this.addListing(mainDocumentPart, indent, false, false); boolean hasValue = false; List tempFields = fields.stream().sorted(Comparator.comparingInt(Field::getOrdinal)).collect(Collectors.toList()); + List formats = tempFields.stream().filter(f -> { + try { + String fTemp = this.formatter(f); + return fTemp != null && !fTemp.isEmpty(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return false; + }).collect(Collectors.toList()); for (Field field: tempFields) { if (visibilityRuleService.isElementVisible(field.getId())) { if (!createListing) { @@ -231,8 +248,13 @@ public class WordBuilder { if (field.getValue() != null && !field.getValue().toString().isEmpty()) { this.indent = indent; String format = this.formatter(field); - if(format != null && format.charAt(0) == '['){ - format = format.substring(1, format.length() - 1).replaceAll(",", ", "); + if(format != null){ + if(format.charAt(0) == '['){ + format = format.substring(1, format.length() - 1).replaceAll(",", ", "); + } + if(formats.size() > 1){ + format = "\t• " + format; + } } XWPFParagraph paragraph = addParagraphContent(format, mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId); if (paragraph != null) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java index cd0496a38..11644f0a4 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java @@ -17,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.transaction.Transactional; +import java.net.URI; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -40,7 +41,7 @@ public class DatasetRDAMapper { } @Transactional - public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, List contributors, List costs) { + public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, eu.eudat.models.rda.Dmp dmp) { Dataset rda = new Dataset(); // rda.setDatasetId(DatasetIdRDAMapper.toRDA(dataset.getId())); if (dataset.getLabel() == null) { @@ -125,7 +126,11 @@ public class DatasetRDAMapper { } List personalDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.personal_data"); if (!personalDataNodes.isEmpty()) { - rda.setPersonalData(personalDataNodes.stream().map(personalDataNode -> Dataset.PersonalData.fromValue(personalDataNode.get("value").asText())).findFirst().get()); + try{ + rda.setPersonalData(personalDataNodes.stream().map(personalDataNode -> Dataset.PersonalData.fromValue(personalDataNode.get("value").asText())).findFirst().get()); + }catch(IllegalArgumentException e){ + rda.setPersonalData(Dataset.PersonalData.UNKNOWN); + } } else { rda.setPersonalData(Dataset.PersonalData.UNKNOWN); } @@ -137,7 +142,11 @@ public class DatasetRDAMapper { } List sensitiveDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.sensitive_data"); if (!sensitiveDataNodes.isEmpty()) { - rda.setSensitiveData(sensitiveDataNodes.stream().map(sensitiveDataNode -> Dataset.SensitiveData.fromValue(sensitiveDataNode.get("value").asText())).findFirst().get()); + try{ + rda.setSensitiveData(sensitiveDataNodes.stream().map(sensitiveDataNode -> Dataset.SensitiveData.fromValue(sensitiveDataNode.get("value").asText())).findFirst().get()); + }catch(IllegalArgumentException e){ + rda.setSensitiveData(Dataset.SensitiveData.UNKNOWN); + } } else { rda.setSensitiveData(Dataset.SensitiveData.UNKNOWN); } @@ -153,7 +162,7 @@ public class DatasetRDAMapper { } List contributorNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.contributor"); if (!contributorNodes.isEmpty()) { - contributors.addAll(contributorNodes.stream().map(contributorNode -> { + dmp.getContributor().addAll(contributorNodes.stream().map(contributorNode -> { JsonNode value = contributorNode.get("value"); if (value.isArray()) { return StreamSupport.stream(value.spliterator(), false).map(node -> ContributorRDAMapper.toRDA(node.asText())).collect(Collectors.toList()); @@ -164,7 +173,46 @@ public class DatasetRDAMapper { } List costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.cost"); if (!costNodes.isEmpty()) { - costs.addAll(CostRDAMapper.toRDAList(costNodes)); + dmp.getCost().addAll(CostRDAMapper.toRDAList(costNodes)); + } + List ethicsNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.ethical_issues"); + if (!ethicsNodes.isEmpty()) { + for(JsonNode node: ethicsNodes){ + String rdaProperty = node.get("rdaProperty").asText(); + String rdaValue = node.get("value").asText(); + if(rdaValue == null || rdaValue.isEmpty()){ + continue; + } + if(rdaProperty.contains("exist")){ + try { + Dmp.EthicalIssuesExist exists = dmp.getEthicalIssuesExist(); + if(exists == null + || ((exists == Dmp.EthicalIssuesExist.NO || exists == Dmp.EthicalIssuesExist.UNKNOWN) && rdaValue.equals("yes")) + || (exists == Dmp.EthicalIssuesExist.YES && !(rdaValue.equals("no") || rdaValue.equals("unknown"))) + || (exists == Dmp.EthicalIssuesExist.UNKNOWN && rdaValue.equals("no"))){ + dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(rdaValue)); + } + }catch(IllegalArgumentException e){ + logger.warn(e.getLocalizedMessage() + ". Setting ethical_issues_exist to unknown"); + dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN); + } + } + else if(rdaProperty.contains("description")){ + if(dmp.getEthicalIssuesDescription() == null){ + dmp.setEthicalIssuesDescription(rdaValue); + } + else{ + dmp.setEthicalIssuesDescription(dmp.getEthicalIssuesDescription() + ", " + rdaValue); + } + } + else if(rdaProperty.contains("report")){ + try { + dmp.setEthicalIssuesReport(URI.create(rdaValue)); + } catch (IllegalArgumentException e) { + logger.warn(e.getLocalizedMessage() + ". Skipping url parsing"); + } + } + } } List foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes, keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).collect(Collectors.toList()); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java index d2abe28d6..539367e10 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java @@ -68,7 +68,7 @@ public class DmpRDAMapper { if (!extraProperties.isEmpty()) { if (extraProperties.get("ethicalIssues") != null) { - rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(extraProperties.get("ethicalIsses").toString())); + rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(extraProperties.get("ethicalIssues").toString())); } else { rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN); } @@ -95,7 +95,7 @@ public class DmpRDAMapper { rda.getContributor().addAll(dmp.getResearchers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); } // rda.getContributor().addAll(dmp.getUsers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); - rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor(), rda.getCost())).collect(Collectors.toList())); + rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda)).collect(Collectors.toList())); rda.setProject(Collections.singletonList(ProjectRDAMapper.toRDA(dmp.getProject(), dmp.getGrant()))); rda.setAdditionalProperty("templates", dmp.getAssociatedDmps().stream().map(datasetProfile -> datasetProfile.getId().toString()).toArray()); return rda; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java index 88ccfe827..7e14486fc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java @@ -67,15 +67,14 @@ public class HostRDAMapper { while(iter.hasNext()) { pList.add(iter.next().asText()); } -// List pidList; -// if(pList.size() == 0){ -// pidList = Arrays.stream(rdaValue.replaceAll("[\\[\"\\]]","").split(",")) -// .map(PidSystem::fromValue).collect(Collectors.toList()); -// } -// else{ -// pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); -// } - List pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); + List pidList; + if(pList.size() == 0){ + pidList = Arrays.stream(rdaValue.replaceAll("[\\[\"\\]]","").split(",")) + .map(PidSystem::fromValue).collect(Collectors.toList()); + } + else{ + pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); + } rda.setPidSystem(pidList); rda.setAdditionalProperty(ImportPropertyName.PID_SYSTEM.getName(), node.get("id").asText()); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java index f39f92b48..dcdd0deac 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java @@ -15,21 +15,17 @@ public class KeywordRDAMapper { private static final Logger logger = LoggerFactory.getLogger(KeywordRDAMapper.class); public static List toRDA(String value) { - ObjectMapper mapper = new ObjectMapper(); - value = JavaToJson.objectStringToJson(value); - if (!value.isEmpty()) { - try { - List tags = Arrays.asList(mapper.readValue(value, Tag[].class)); - List keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList()); - return keywordNames; - } catch (JsonProcessingException e) { - logger.warn(e.getMessage() + ". Attempting to parse it as a String list."); - if(!value.isEmpty()) { - return new ArrayList<>(Arrays.asList(value.replace(" ", "").split(","))); - } - } - + if (!value.isEmpty()) { + try { + ObjectMapper mapper = new ObjectMapper(); + String valueJson = JavaToJson.objectStringToJson(value); + List tags = Arrays.asList(mapper.readValue(valueJson, Tag[].class)); + return tags.stream().map(Tag::getName).collect(Collectors.toList()); + } catch (JsonProcessingException e) { + logger.warn(e.getMessage() + ". Attempting to parse it as a String list."); + return new ArrayList<>(Arrays.asList(value.replace(" ", "").split(","))); } + } return new ArrayList<>(); } From 641088e289bed61a164c2bfe48fcbf1e550f3c83 Mon Sep 17 00:00:00 2001 From: Aldo Mihasi Date: Wed, 12 Jan 2022 16:31:07 +0200 Subject: [PATCH 3/3] Remove local configs, commited by mistake --- .../main/resources/config/application-devel.properties | 10 +++++----- .../src/main/resources/config/application.properties | 4 ++-- dmp-frontend/src/assets/config/config.json | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dmp-backend/web/src/main/resources/config/application-devel.properties b/dmp-backend/web/src/main/resources/config/application-devel.properties index 7a16d94fc..d7df68f89 100644 --- a/dmp-backend/web/src/main/resources/config/application-devel.properties +++ b/dmp-backend/web/src/main/resources/config/application-devel.properties @@ -1,9 +1,9 @@ dmp.domain = http://localhost:4200 ####################PERSISTENCE OVERRIDES CONFIGURATIONS########## -database.url=jdbc:postgresql://dbserver02.local.cite.gr:5432/dmptool -database.username=dmtadm -database.password=t00L4DM@18! +database.url= +database.username= +database.password= spring.datasource.maxIdle=10 spring.datasource.minIdle=5 spring.datasource.maxActive=10 @@ -38,7 +38,7 @@ facebook.login.clientSecret= facebook.login.namespace= #############GOOGLE LOGIN CONFIGURATIONS######### -google.login.clientId=524432312250-sc9qsmtmbvlv05r44onl6l93ia3k9deo.apps.googleusercontent.com +google.login.clientId= #############LINKEDIN LOGIN CONFIGURATIONS######### linkedin.login.clientId= @@ -86,7 +86,7 @@ zenodo.login.redirect_uri=http://localhost:4200/login/external/zenodo #############CONTACT EMAIL CONFIGURATIONS######### contact_email.mail= -language.path=C:\\Users\\aldom\\OneDrive\\Desktop\\CITE\\dmp-frontend\\src\\assets\\i18n\\ +language.path=dmp-frontend/src/assets/i18n/ #############LOGGING######### logging.config=classpath:logging/logback-${spring.profiles.active}.xml diff --git a/dmp-backend/web/src/main/resources/config/application.properties b/dmp-backend/web/src/main/resources/config/application.properties index 40f9ccd2b..9ae11ab94 100644 --- a/dmp-backend/web/src/main/resources/config/application.properties +++ b/dmp-backend/web/src/main/resources/config/application.properties @@ -98,7 +98,7 @@ database.lock-fail-interval=120000 ##########################MISC########################################## #############USER GUIDE######### -userguide.path=C:\\Users\\aldom\\OneDrive\\Desktop\\CITE\\user-guide +userguide.path=user-guide/ #############NOTIFICATION######### notification.rateInterval=30000 @@ -112,7 +112,7 @@ notification.modifiedFinalised.subject=[OpenDMP] The {name} has been modified an logging.config=classpath:logging/logback-${spring.profiles.active}.xml #############TEMP######### -temp.temp=C:\\Users\\aldom\\OneDrive\\Desktop\\CITE-TMP\\ +temp.temp=tmp/ #############ZENODO######### zenodo.affiliation=ARGOS diff --git a/dmp-frontend/src/assets/config/config.json b/dmp-frontend/src/assets/config/config.json index 35b504a22..ea878f347 100644 --- a/dmp-frontend/src/assets/config/config.json +++ b/dmp-frontend/src/assets/config/config.json @@ -10,7 +10,7 @@ "loginProviders": { "enabled": [1, 2, 3, 4, 5, 6, 7, 8], "facebookConfiguration": { "clientId": "" }, - "googleConfiguration": { "clientId": "524432312250-sc9qsmtmbvlv05r44onl6l93ia3k9deo.apps.googleusercontent.com" }, + "googleConfiguration": { "clientId": "" }, "linkedInConfiguration": { "clientId": "", "oauthUrl": "https://www.linkedin.com/oauth/v2/authorization",