From e675ad7c9d02cd7062fb3b7729d23e4a89cfb4a4 Mon Sep 17 00:00:00 2001 From: "DESKTOP-4ES9U2E\\aldom" Date: Wed, 29 Dec 2021 15:24:46 +0200 Subject: [PATCH 01/11] madmp changes --- .../utilities/documents/word/WordBuilder.java | 10 +- .../logic/utilities/json/JavaToJson.java | 6 +- .../java/eu/eudat/models/rda/Contact.java | 4 +- .../java/eu/eudat/models/rda/ContactId.java | 4 +- .../java/eu/eudat/models/rda/Contributor.java | 4 +- .../eu/eudat/models/rda/ContributorId.java | 4 +- .../main/java/eu/eudat/models/rda/Cost.java | 4 +- .../java/eu/eudat/models/rda/DatasetId.java | 4 +- .../main/java/eu/eudat/models/rda/DmpId.java | 4 +- .../java/eu/eudat/models/rda/FunderId.java | 4 +- .../java/eu/eudat/models/rda/Funding.java | 4 +- .../main/java/eu/eudat/models/rda/Host.java | 4 +- .../java/eu/eudat/models/rda/License.java | 4 +- .../eudat/models/rda/MetadataStandardId.java | 4 +- .../java/eu/eudat/models/rda/Project.java | 4 +- .../models/rda/mapper/CostRDAMapper.java | 76 +++++++++- .../models/rda/mapper/DatasetIdRDAMapper.java | 28 ++-- .../models/rda/mapper/DatasetRDAMapper.java | 43 ++++-- .../rda/mapper/DistributionRDAMapper.java | 65 +++++++-- .../eudat/models/rda/mapper/DmpRDAMapper.java | 2 +- .../models/rda/mapper/HostRDAMapper.java | 138 ++++++++++-------- .../models/rda/mapper/KeywordRDAMapper.java | 24 +-- .../models/rda/mapper/LicenseRDAMapper.java | 24 ++- .../mapper/SecurityAndPrivacyRDAMapper.java | 11 +- .../mapper/TechnicalResourceRDAMapper.java | 11 +- .../src/main/resources/RDACommonStandards.txt | 45 +++++- 26 files changed, 383 insertions(+), 152 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java index 97eb201db..78c38db09 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java @@ -131,7 +131,7 @@ public class WordBuilder { this.options.put(ParagraphStyle.COMMENT, (mainDocumentPart, item) -> { XWPFParagraph paragraph = mainDocumentPart.createParagraph(); XWPFRun run = paragraph.createRun(); - run.setText(item); + run.setText(" " + item); run.setItalic(true); return paragraph; }); @@ -178,7 +178,7 @@ public class WordBuilder { number.setVal(BigInteger.valueOf(indent)); paragraphPos = mainDocumentPart.getPosOfParagraph(paragraph); } - createSections(section.getSections(), mainDocumentPart, ParagraphStyle.HEADER5, 1, createListing, visibilityRuleService, page, tempSectionString); + createSections(section.getSections(), mainDocumentPart, ParagraphStyle.HEADER4, 1, createListing, visibilityRuleService, page, tempSectionString); hasValue = createCompositeFields(section.getCompositeFields(), mainDocumentPart, 2, createListing, visibilityRuleService, page, tempSectionString); if (!hasValue && paragraphPos > -1) { @@ -230,7 +230,11 @@ public class WordBuilder { try { if (field.getValue() != null && !field.getValue().toString().isEmpty()) { this.indent = indent; - XWPFParagraph paragraph = addParagraphContent(this.formatter(field), mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId); + String format = this.formatter(field); + if(format != null && format.charAt(0) == '['){ + format = format.substring(1, format.length() - 1).replaceAll(",", ", "); + } + XWPFParagraph paragraph = addParagraphContent(format, mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId); if (paragraph != null) { CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl(); number.setVal(BigInteger.valueOf(indent)); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java index 8338c1526..aa365fe0e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/json/JavaToJson.java @@ -4,10 +4,10 @@ public class JavaToJson { public static String objectStringToJson(String object) { String result = object.replaceAll("=", "\":\"") - .replaceAll("\\{", "{\"") + //.replaceAll("\\{", "{\"") .replaceAll(", ", "\", \"") - .replaceAll("}", "\"}" ). - replaceAll("}\", \"\\{", "}, {"); + //.replaceAll("}", "\"}" ). + .replaceAll("}\", \"\\{", "}, {"); return result; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java index 606a3ae2c..bbd5e40f9 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contact.java @@ -133,12 +133,12 @@ public class Contact implements Serializable this.name = name; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java index 60454ea6c..11bd8b54e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContactId.java @@ -100,12 +100,12 @@ public class ContactId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java index 21dfe68c8..c53c2551d 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Contributor.java @@ -168,12 +168,12 @@ public class Contributor implements Serializable this.role = role; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java index 4045f5e0e..c92a5227a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/ContributorId.java @@ -101,12 +101,12 @@ public class ContributorId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java index db32ff641..ee5b3289f 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Cost.java @@ -162,12 +162,12 @@ public class Cost implements Serializable this.value = value; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java index 6c1219c1c..02108950b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DatasetId.java @@ -109,12 +109,12 @@ public class DatasetId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java index 7240efaa4..d277fbdd9 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/DmpId.java @@ -101,12 +101,12 @@ public class DmpId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java index 12b8bb8f7..4033f415c 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/FunderId.java @@ -101,12 +101,12 @@ public class FunderId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java index 0cc378e7c..0b8aa8c53 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Funding.java @@ -133,12 +133,12 @@ public class Funding implements Serializable this.grantId = grantId; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java index bfa6ec615..2227460df 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Host.java @@ -391,12 +391,12 @@ public class Host implements Serializable this.url = url; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java index 1527f5503..e622e14cc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/License.java @@ -100,12 +100,12 @@ public class License implements Serializable this.startDate = startDate; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java index 06b54a1f1..a693f34a5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/MetadataStandardId.java @@ -101,12 +101,12 @@ public class MetadataStandardId implements Serializable this.type = type; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java index 09c231ca3..4854fa609 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/Project.java @@ -199,12 +199,12 @@ public class Project implements Serializable this.title = title; } - @JsonAnyGetter + @JsonProperty("additional_properties") public Map getAdditionalProperties() { return this.additionalProperties; } - @JsonAnySetter + @JsonProperty("additional_properties") public void setAdditionalProperty(String name, Object value) { this.additionalProperties.put(name, value); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java index 3d93d0768..942f48d4a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java @@ -1,10 +1,19 @@ package eu.eudat.models.rda.mapper; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.models.rda.Cost; +import eu.eudat.models.rda.PidSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CostRDAMapper { + private static final Logger logger = LoggerFactory.getLogger(DatasetRDAMapper.class); public static Cost toRDA(Map cost) { Cost rda = new Cost(); @@ -19,4 +28,69 @@ public class CostRDAMapper { return rda; } + public static List toRDAList(List nodes) throws JsonProcessingException { + Map rdaMap = new HashMap<>(); + for(JsonNode node: nodes){ + String rdaProperty = node.get("rdaProperty").asText(); + String rdaValue = node.get("value").asText(); + if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){ + continue; + } + String key = node.get("numbering").asText(); + if(!key.contains("mult")){ + key = "0"; + } + else{ + key = "" + key.charAt(4); + } + Cost rda; + if(rdaMap.containsKey(key)){ + rda = rdaMap.get(key); + } + else{ + rda = new Cost(); + rdaMap.put(key, rda); + } + if(rdaProperty.contains("value")){ + rda.setValue(Double.valueOf(rdaValue)); + } + else if(rdaProperty.contains("currency_code")){ + String json = JavaToJson.objectStringToJson(rdaValue); + HashMap result = + new ObjectMapper().readValue(json, HashMap.class); + rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value"))); + } + else if(rdaProperty.contains("title")){ + Iterator iter = node.get("value").elements(); + StringBuilder title = new StringBuilder(); + while(iter.hasNext()){ + String next = iter.next().asText(); + if(!next.equals("Other")) { + title.append(next).append(", "); + } + } + if(title.length() > 2){ + rda.setTitle(title.substring(0, title.length() - 2)); + } + else{ + String t = rda.getTitle(); + if(t == null){ // only other as title + rda.setTitle(rdaValue); + } + else{ // option + other + rda.setTitle(t + ", " + rdaValue); + } + } + } + else if(rdaProperty.contains("description")){ + rda.setDescription(rdaValue); + } + } + List rdaList = rdaMap.values().stream() + .filter(cost -> cost.getTitle() != null) + .collect(Collectors.toList()); + return rdaList; + + } + } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java index a97c58e6a..6d87a10e4 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java @@ -29,19 +29,23 @@ public class DatasetIdRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); - - ObjectMapper mapper = new ObjectMapper(); - try { - Map values = mapper.readValue(rdaValue, HashMap.class); - if (!values.isEmpty()) { - values.entrySet().forEach(entry -> finalRDAMap(data, entry.getKey(), (String) entry.getValue())); - } else { - finalRDAMap(data, rdaProperty, rdaValue); - } - } catch (IOException e) { - finalRDAMap(data, rdaProperty, rdaValue); - logger.error(e.getMessage(), e); + if(rdaValue.isEmpty()){ + continue; } + finalRDAMap(data, rdaProperty, rdaValue); + +// ObjectMapper mapper = new ObjectMapper(); +// try { +// Map values = mapper.readValue(rdaValue, HashMap.class); +// if (!values.isEmpty()) { +// values.entrySet().forEach(entry -> finalRDAMap(data, entry.getKey(), (String) entry.getValue())); +// } else { +// finalRDAMap(data, rdaProperty, rdaValue); +// } +// } catch (IOException e) { +// finalRDAMap(data, rdaProperty, rdaValue); +// logger.error(e.getMessage(), e); +// } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java index 67858a14b..fda0ea8ec 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java @@ -9,10 +9,7 @@ import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel; import eu.eudat.models.data.datasetwizard.DatasetWizardModel; -import eu.eudat.models.rda.Contributor; -import eu.eudat.models.rda.Dataset; -import eu.eudat.models.rda.DatasetId; -import eu.eudat.models.rda.Language; +import eu.eudat.models.rda.*; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +40,7 @@ public class DatasetRDAMapper { } @Transactional - public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, List contributors) { + public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, List contributors, List costs) { Dataset rda = new Dataset(); // rda.setDatasetId(DatasetIdRDAMapper.toRDA(dataset.getId())); if (dataset.getLabel() == null) { @@ -68,13 +65,13 @@ public class DatasetRDAMapper { rda.setDatasetId(new DatasetId(dataset.getId().toString(), DatasetId.Type.OTHER)); } List typeNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.type"); - if (!typeNodes.isEmpty()) { + if (!typeNodes.isEmpty() && !typeNodes.get(0).get("value").asText().isEmpty()) { rda.setType(typeNodes.get(0).get("value").asText()); } else { rda.setType("DMP Dataset"); } List languageNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.language"); - if (!languageNodes.isEmpty()) { + if (!languageNodes.isEmpty() && !languageNodes.get(0).get("value").asText().isEmpty()) { rda.setLanguage(Language.fromValue(languageNodes.get(0).get("value").asText())); } else { rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(dataset.getProfile().getLanguage())); @@ -83,21 +80,33 @@ public class DatasetRDAMapper { if (!metadataNodes.isEmpty()) { rda.setMetadata(MetadataRDAMapper.toRDAList(metadataNodes)); } + else{ + rda.setMetadata(new ArrayList<>()); + } List qaNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.data_quality_assurance"); if (!qaNodes.isEmpty()) { + List qaList = qaNodes.stream().map(qaNode -> qaNode.get("value").asText()).filter(qaNode -> !qaNode.isEmpty()).collect(Collectors.toList()); + rda.setDataQualityAssurance(qaList); + /*rda.setDataQualityAssurance(qaNodes.stream().map(qaNode -> qaNode.get("value").asText()).collect(Collectors.toList())); for (int i = 0; i < qaNodes.size(); i++) { rda.setAdditionalProperty("qaId" + (i + 1), qaNodes.get(i).get("id").asText()); }*/ - rda.setDataQualityAssurance(Collections.singletonList(qaNodes.get(0).get("value").asText())); + //rda.setDataQualityAssurance(Collections.singletonList(qaNodes.get(0).get("value").asText())); + } + else{ + rda.setDataQualityAssurance(new ArrayList<>()); } List preservationNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.preservation_statement"); - if (!preservationNodes.isEmpty()) { + if (!preservationNodes.isEmpty() && !preservationNodes.get(0).get("value").asText().isEmpty()) { rda.setPreservationStatement(preservationNodes.get(0).get("value").asText()); } List distributionNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.distribution"); if (!distributionNodes.isEmpty()) { - rda.setDistribution(Collections.singletonList(DistributionRDAMapper.toRDA(distributionNodes))); + rda.setDistribution(DistributionRDAMapper.toRDAList(distributionNodes)); + } + else{ + rda.setDistribution(new ArrayList<>()); } List keywordNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.keyword"); if (!keywordNodes.isEmpty()) { @@ -126,6 +135,9 @@ public class DatasetRDAMapper { if (!securityAndPrivacyNodes.isEmpty()) { rda.setSecurityAndPrivacy(SecurityAndPrivacyRDAMapper.toRDAList(securityAndPrivacyNodes)); } + else{ + rda.setSecurityAndPrivacy(new ArrayList<>()); + } List sensitiveDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.sensitive_data"); if (!sensitiveDataNodes.isEmpty()) { rda.setSensitiveData(sensitiveDataNodes.stream().map(sensitiveDataNode -> Dataset.SensitiveData.fromValue(sensitiveDataNode.get("value").asText())).findFirst().get()); @@ -136,8 +148,11 @@ public class DatasetRDAMapper { if (!technicalResourceNodes.isEmpty()) { rda.setTechnicalResource(TechnicalResourceRDAMapper.toRDAList(technicalResourceNodes)); } + else{ + rda.setTechnicalResource(new ArrayList<>()); + } List issuedNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.issued"); - if (!issuedNodes.isEmpty()) { + if (!issuedNodes.isEmpty() && !issuedNodes.get(0).get("value").asText().isEmpty()) { rda.setIssued(issuedNodes.get(0).get("value").asText()); } List contributorNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.contributor"); @@ -147,10 +162,14 @@ public class DatasetRDAMapper { if (value.isArray()) { return StreamSupport.stream(value.spliterator(), false).map(node -> ContributorRDAMapper.toRDA(node.asText())).collect(Collectors.toList()); } else { - return Collections.singletonList(new Contributor()); + return Collections.singletonList(new Contributor()); // return null kalutera } }).flatMap(Collection::stream).collect(Collectors.toList())); } + List costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.cost"); + if (!costNodes.isEmpty()) { + costs.addAll(CostRDAMapper.toRDAList(costNodes)); + } List foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes, keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).collect(Collectors.toList()); templateIdsToValues.entrySet().forEach(entry -> { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java index c26f8df6a..11785d491 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java @@ -1,7 +1,10 @@ package eu.eudat.models.rda.mapper; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.logic.utilities.helpers.MyStringUtils; +import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.rda.Distribution; import org.slf4j.Logger; @@ -14,16 +17,34 @@ import java.util.stream.Collectors; public class DistributionRDAMapper { private static final Logger logger = LoggerFactory.getLogger(DistributionRDAMapper.class); - public static List toRDAList(List nodes) { + public static List toRDAList(List nodes) throws JsonProcessingException { Map rdaMap = new HashMap<>(); for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); - Distribution rda = getRelative(rdaMap, node.get("numbering").asText()); - if (!rdaMap.containsValue(rda)) { - rdaMap.put(node.get("numbering").asText(), rda); + if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){ // node.get("value").isNull() + continue; } + String key = node.get("numbering").asText(); + if(!key.contains("mult")){ + key = "0"; + } + else{ + key = "" + key.charAt(4); + } + Distribution rda; + if(rdaMap.containsKey(key)){ + rda = rdaMap.get(key); + } + else { + rda = new Distribution(); + rdaMap.put(key, rda); + } +// Distribution rda = getRelative(rdaMap, node.get("numbering").asText()); +// if (!rdaMap.containsValue(rda)) { +// rdaMap.put(node.get("numbering").asText(), rda); +// } for (ExportPropertyName exportPropertyName : ExportPropertyName.values()) { if (rdaProperty.contains(exportPropertyName.getName())) { switch (exportPropertyName) { @@ -40,8 +61,10 @@ public class DistributionRDAMapper { rda.setAdditionalProperty(ImportPropertyName.DOWNLOAD_URL.getName(), node.get("id").asText()); break; case DESCRIPTION: - rda.setDescription(rdaValue); - rda.setAdditionalProperty(ImportPropertyName.DESCRIPTION.getName(), node.get("id").asText()); + if(!rdaProperty.contains("host")) { + rda.setDescription(rdaValue); + rda.setAdditionalProperty(ImportPropertyName.DESCRIPTION.getName(), node.get("id").asText()); + } break; case DATA_ACCESS: rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue)); @@ -56,12 +79,29 @@ public class DistributionRDAMapper { rda.setLicense(Collections.singletonList(LicenseRDAMapper.toRDA(licenseNodes))); break; case FORMAT: - rda.setFormat(Collections.singletonList(rdaValue)); + //rda.setFormat(Collections.singletonList(rdaValue)); + if(node.get("value").isArray()){ + Iterator iter = node.get("value").elements(); + List formats = new ArrayList<>(); + while(iter.hasNext()) { + String format = JavaToJson.objectStringToJson(iter.next().asText()); + Map result = + new ObjectMapper().readValue(format, HashMap.class); + format = result.get("label"); + formats.add(format); + } + rda.setFormat(formats); + } + else{ + rda.setFormat(new ArrayList<>(Arrays.asList(rdaValue.replace(" ", "").split(",")))); + } rda.setAdditionalProperty(ImportPropertyName.FORMAT.getName(), node.get("id").asText()); break; case TITLE: - rda.setTitle(rdaValue); - rda.setAdditionalProperty(ImportPropertyName.TITLE.getName(), node.get("id").asText()); + if(!rdaProperty.contains("host")) { + rda.setTitle(rdaValue); + rda.setAdditionalProperty(ImportPropertyName.TITLE.getName(), node.get("id").asText()); + } break; case HOST: rda.setHost(HostRDAMapper.toRDA(nodes, node.get("numbering").asText())); @@ -71,7 +111,12 @@ public class DistributionRDAMapper { } } - return new ArrayList<>(rdaMap.values()); + List rdaList = rdaMap.values().stream() + .filter(distro -> distro.getTitle() != null) + .collect(Collectors.toList()); + + return rdaList; + //return new ArrayList<>(rdaMap.values()); } public static Map toProperties(List rdas) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java index 700f0bfda..d2abe28d6 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java @@ -95,7 +95,7 @@ public class DmpRDAMapper { rda.getContributor().addAll(dmp.getResearchers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); } // rda.getContributor().addAll(dmp.getUsers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); - rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor())).collect(Collectors.toList())); + rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor(), rda.getCost())).collect(Collectors.toList())); rda.setProject(Collections.singletonList(ProjectRDAMapper.toRDA(dmp.getProject(), dmp.getGrant()))); rda.setAdditionalProperty("templates", dmp.getAssociatedDmps().stream().map(datasetProfile -> datasetProfile.getId().toString()).toArray()); return rda; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java index d96aa3ca3..c52338e2a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java @@ -8,10 +8,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; public class HostRDAMapper { private static final Logger logger = LoggerFactory.getLogger(HostRDAMapper.class); @@ -19,11 +17,14 @@ public class HostRDAMapper { public static Host toRDA(List nodes, String numbering) { Host rda = new Host(); for (JsonNode node: nodes) { - String rdaProperty = node.get("rdaProperties").asText(); + String rdaProperty = node.get("rdaProperty").asText(); if (rdaProperty.contains("host")) { int firstDiff = MyStringUtils.getFirstDifference(numbering, node.get("numbering").asText()); - if (firstDiff == -1 || firstDiff > 2) { + if (firstDiff == -1 || firstDiff >= 2) { String rdaValue = node.get("value").asText(); + if(rdaValue == null || (rdaValue.isEmpty() && !node.get("value").isArray())){ + continue; + } for (ExportPropertyName propertyName: ExportPropertyName.values()) { if (rdaProperty.contains(propertyName.getName())) { switch (propertyName) { @@ -51,8 +52,27 @@ public class HostRDAMapper { rda.setAdditionalProperty(ImportPropertyName.GEO_LOCATION.getName(), node.get("id").asText()); break; case PID_SYSTEM: - rda.setPidSystem(Collections.singletonList(PidSystem.fromValue(rdaValue))); - rda.setAdditionalProperty(ImportPropertyName.PID_SYSTEM.getName(), node.get("id").asText()); + try{ + Iterator iter = node.get("value").elements(); + List pList = new ArrayList<>(); + while(iter.hasNext()) { + pList.add(iter.next().asText()); + } + List pidList; + if(pList.size() == 0){ + pidList = Arrays.stream(rdaValue.replaceAll("[\\[\"\\]]","").split(",")) + .map(PidSystem::fromValue).collect(Collectors.toList()); + } + else{ + pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); + } + rda.setPidSystem(pidList); + rda.setAdditionalProperty(ImportPropertyName.PID_SYSTEM.getName(), node.get("id").asText()); + } + catch (IllegalArgumentException e){ + rda.setPidSystem(new ArrayList<>()); + break; + } break; case STORAGE_TYPE: rda.setStorageType(rdaValue); @@ -67,8 +87,12 @@ public class HostRDAMapper { rda.setAdditionalProperty(ImportPropertyName.TITLE.getName(), node.get("id").asText()); break; case URL: - rda.setUrl(URI.create(rdaValue)); - rda.setAdditionalProperty(ImportPropertyName.URL.getName(), node.get("id").asText()); + try { + rda.setUrl(URI.create(rdaValue)); + rda.setAdditionalProperty(ImportPropertyName.URL.getName(), node.get("id").asText()); + } catch (IllegalArgumentException e) { + logger.warn(e.getLocalizedMessage() + ". Skipping url parsing"); + } break; } } @@ -76,13 +100,9 @@ public class HostRDAMapper { } } } - - if (rda.getTitle() == null) { - throw new IllegalArgumentException("Host Title is missing"); - } - - if (rda.getUrl() == null) { - throw new IllegalArgumentException("Host Url is missing"); + + if(rda.getTitle() == null || rda.getUrl() == null){ + return null; } return rda; @@ -90,49 +110,49 @@ public class HostRDAMapper { public static Map toProperties(Host rda) { Map properties = new HashMap<>(); - rda.getAdditionalProperties().entrySet().forEach(entry -> { - try { - ImportPropertyName importPropertyName = ImportPropertyName.fromString(entry.getKey()); - switch (importPropertyName) { - case AVAILABILITY: - properties.put(entry.getValue().toString(), rda.getAvailability()); - break; - case TITLE: - properties.put(entry.getValue().toString(), rda.getTitle()); - break; - case DESCRIPTION: - properties.put(entry.getValue().toString(), rda.getDescription()); - break; - case BACKUP_FREQUENCY: - properties.put(entry.getValue().toString(), rda.getBackupFrequency()); - break; - case BACKUP_TYPE: - properties.put(entry.getValue().toString(), rda.getBackupType()); - break; - case CERTIFIED_WITH: - properties.put(entry.getValue().toString(), rda.getCertifiedWith().value()); - break; - case GEO_LOCATION: - properties.put(entry.getValue().toString(), rda.getGeoLocation().value()); - break; - case PID_SYSTEM: - properties.put(entry.getValue().toString(), rda.getPidSystem().get(0).value()); - break; - case STORAGE_TYPE: - properties.put(entry.getValue().toString(), rda.getStorageType()); - break; - case SUPPORT_VERSIONING: - properties.put(entry.getValue().toString(), rda.getSupportVersioning().value()); - break; - case URL: - properties.put(entry.getValue().toString(), rda.getUrl().toString()); - break; - } - } catch (Exception e) { - logger.error(e.getMessage(), e); + rda.getAdditionalProperties().entrySet().forEach(entry -> { + try { + ImportPropertyName importPropertyName = ImportPropertyName.fromString(entry.getKey()); + switch (importPropertyName) { + case AVAILABILITY: + properties.put(entry.getValue().toString(), rda.getAvailability()); + break; + case TITLE: + properties.put(entry.getValue().toString(), rda.getTitle()); + break; + case DESCRIPTION: + properties.put(entry.getValue().toString(), rda.getDescription()); + break; + case BACKUP_FREQUENCY: + properties.put(entry.getValue().toString(), rda.getBackupFrequency()); + break; + case BACKUP_TYPE: + properties.put(entry.getValue().toString(), rda.getBackupType()); + break; + case CERTIFIED_WITH: + properties.put(entry.getValue().toString(), rda.getCertifiedWith().value()); + break; + case GEO_LOCATION: + properties.put(entry.getValue().toString(), rda.getGeoLocation().value()); + break; + case PID_SYSTEM: + properties.put(entry.getValue().toString(), rda.getPidSystem().get(0).value()); + break; + case STORAGE_TYPE: + properties.put(entry.getValue().toString(), rda.getStorageType()); + break; + case SUPPORT_VERSIONING: + properties.put(entry.getValue().toString(), rda.getSupportVersioning().value()); + break; + case URL: + properties.put(entry.getValue().toString(), rda.getUrl().toString()); + break; } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } - }); + }); return properties; } @@ -193,4 +213,4 @@ public class HostRDAMapper { throw new Exception("No name available"); } } -} +} \ No newline at end of file diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java index 2e8a93d37..681373be7 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java @@ -14,18 +14,20 @@ public class KeywordRDAMapper { private static final Logger logger = LoggerFactory.getLogger(KeywordRDAMapper.class); public static List toRDA(String value) { - ObjectMapper mapper = new ObjectMapper(); - try { - value = JavaToJson.objectStringToJson(value); - if (!value.isEmpty()) { - List tags = Arrays.asList(mapper.readValue(value, Tag[].class)); - List keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList()); - return keywordNames; - } - } catch (IOException e) { - logger.error(e.getMessage(), e); +// ObjectMapper mapper = new ObjectMapper(); +// try { +// value = JavaToJson.objectStringToJson(value); +// if (!value.isEmpty()) { +// List tags = Arrays.asList(mapper.readValue(value, Tag[].class)); +// List keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList()); +// return keywordNames; +// } +// } catch (IOException e) { +// logger.error(e.getMessage(), e); +// } + if(!value.isEmpty()) { + return new ArrayList<>(Arrays.asList(value.replace(" ", "").split(","))); } - return new ArrayList<>(); } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java index 108f5eecb..d005882cc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java @@ -16,12 +16,16 @@ public class LicenseRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String value = node.get("value").asText(); - + if(value.equals(null)){ + continue; + } for (LicenceProperties licenceProperties: LicenceProperties.values()) { if (rdaProperty.contains(licenceProperties.getName())) { switch (licenceProperties) { case LICENSE_REF: - rda.setLicenseRef(URI.create(value)); + if(value.contains(":")){ // valid uri + rda.setLicenseRef(URI.create(value)); + } break; case START_DATE: rda.setStartDate(value); @@ -39,12 +43,16 @@ public class LicenseRDAMapper { }*/ } - if (rda.getLicenseRef() == null) { - throw new IllegalArgumentException("Licence Reference is missing"); - } - - if (rda.getStartDate() == null) { - throw new IllegalArgumentException("License Start Date is missing"); +// if (rda.getLicenseRef() == null) { +// throw new IllegalArgumentException("Licence Reference is missing"); +// } +// +// if (rda.getStartDate() == null) { +// throw new IllegalArgumentException("License Start Date is missing"); +// } + + if(rda.getLicenseRef() == null || rda.getStartDate() == null){ + return null; } return rda; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java index 6006910a1..f87b7ee71 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java @@ -2,11 +2,13 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; +import eu.eudat.models.rda.Distribution; import eu.eudat.models.rda.SecurityAndPrivacy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; public class SecurityAndPrivacyRDAMapper { private static final Logger logger = LoggerFactory.getLogger(SecurityAndPrivacyRDAMapper.class); @@ -17,6 +19,9 @@ public class SecurityAndPrivacyRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); + if(rdaValue == null || rdaValue.isEmpty()){ + continue; + } SecurityAndPrivacy rda = getRelative(rdaMap, node.get("numbering").asText()); if (!rdaMap.containsValue(rda)) { @@ -38,7 +43,11 @@ public class SecurityAndPrivacyRDAMapper { } } - return new ArrayList<>(rdaMap.values()); + List rdaList = rdaMap.values().stream() + .filter(sap -> sap.getTitle() != null) + .collect(Collectors.toList()); + return rdaList; + //return new ArrayList<>(rdaMap.values()); } public static Map toProperties(List rdas) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java index 5214a7518..9b029bedf 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java @@ -2,11 +2,13 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; +import eu.eudat.models.rda.SecurityAndPrivacy; import eu.eudat.models.rda.TechnicalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; public class TechnicalResourceRDAMapper { private static final Logger logger = LoggerFactory.getLogger(TechnicalResourceRDAMapper.class); @@ -17,6 +19,9 @@ public class TechnicalResourceRDAMapper { for (JsonNode node: nodes) { String rdaProperty = node.get("rdaProperty").asText(); String rdaValue = node.get("value").asText(); + if(rdaValue == null || rdaValue.isEmpty()){ + continue; + } TechnicalResource rda = getRelative(rdaMap, node.get("numbering").asText()); if (!rdaMap.containsValue(rda)) { @@ -38,7 +43,11 @@ public class TechnicalResourceRDAMapper { } } - return new ArrayList<>(rdaMap.values()); + List rdaList = rdaMap.values().stream() + .filter(tr -> tr.getName() != null) + .collect(Collectors.toList()); + return rdaList; + //return new ArrayList<>(rdaMap.values()); } public static Map toProperties(List rdas) { diff --git a/dmp-backend/web/src/main/resources/RDACommonStandards.txt b/dmp-backend/web/src/main/resources/RDACommonStandards.txt index d8693586b..03c8c0dcb 100644 --- a/dmp-backend/web/src/main/resources/RDACommonStandards.txt +++ b/dmp-backend/web/src/main/resources/RDACommonStandards.txt @@ -34,12 +34,49 @@ dataset.security_and_privacy.description dataset.security_and_privacy.title dataset.sensitive_data dataset.technical_resource.description -dataset.technical_resource.technical_resource -dataset.technical_resource.technical_resource.description -dataset.technical_resource.technical_resource.name +dataset.technical_resource.name +dataset.title dataset.type dataset.issued dataset.dataset_id dataset.dataset_id.identifier dataset.dataset_id.type -dmp.contributor \ No newline at end of file +dataset.description +dmp.contact +dmp.contact.contact_id.identifier +dmp.contact.contact_id.type +dmp.contact.mbox +dmp.contact.name +dmp.contributor +dmp.contributor.contributor_id.identifier +dmp.contributor.contributor_id.type +dmp.contributor.mbox +dmp.contributor.name +dmp.contributor.role +dmp.cost +dmp.cost.currency_code +dmp.cost.description +dmp.cost.title +dmp.cost.value +dmp.created +dmp.description +dmp.dmp_id +dmp.dmp_id.identifier +dmp.dmp_id.type +dmp.ethical_issues_description +dmp.ethical_issues_exist +dmp.ethical_issues_report +dmp.language +dmp.modified +dmp.project +dmp.project.description +dmp.project.end +dmp.project.funding +dmp.project.funding.funder_id.identifier +dmp.project.funding.funder_id.type +dmp.project.funding.funding_status +dmp.project.funding.grant_id.identifier +dmp.project.funding.grant_id.type +dmp.project.start +dmp.project.title +dmp.title \ No newline at end of file From 1d68b2a3796b5a853f61e7f2b87238f5b0345947 Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Fri, 7 Jan 2022 18:06:19 +0200 Subject: [PATCH 02/11] Minor improvement for dataset prefilling --- .../models/data/datasetwizard/DatasetWizardModel.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java index 5606cf096..0758daaad 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java @@ -327,10 +327,11 @@ public class DatasetWizardModel implements DataModel method.getName().equals(methodName)).collect(Collectors.toList()).get(0); setterMethod.invoke(datasetWizardModel, sourceValue); - }catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { - e.printStackTrace(); + }catch (InvocationTargetException | IllegalAccessException e) { + throw e; } } else { List nodes = JsonSearcher.findNodes(parentNode, "rdaProperty", prefillingMapping.getMaDmpTarget()); From db01f2d1199b75ecbf1ed6d0639edaf755e098cf Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Wed, 12 Jan 2022 11:43:32 +0200 Subject: [PATCH 03/11] Add fixedMappings for prefilling --- .../logic/managers/PrefillingManager.java | 2 +- .../entities/DefaultPrefillingMapping.java | 38 ++++++++++++++ .../entities/PrefillingFixedMapping.java | 38 ++++++++++++++ .../proxy/config/entities/PrefillingGet.java | 20 +++++--- .../config/entities/PrefillingMapping.java | 35 ++----------- .../datasetwizard/DatasetWizardModel.java | 49 ++++++++++++------- .../resources/externalUrls/ExternalUrls.xml | 4 ++ 7 files changed, 130 insertions(+), 56 deletions(-) create mode 100644 dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/DefaultPrefillingMapping.java create mode 100644 dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingFixedMapping.java diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java index 1b8203e92..de03af282 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java @@ -52,7 +52,7 @@ public class PrefillingManager { PrefillingGet prefillingGet = prefillingConfig.getPrefillingGet(); Map prefillingEntity = getSingle(prefillingGet.getUrl(), prefillId); DatasetProfile datasetProfile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(profileId); - return DatasetWizardModel.fromPrefilledEntity(prefillingEntity, prefillingGet.getMappings(), datasetProfile, datasetManager); + return DatasetWizardModel.fromPrefilledEntity(prefillingEntity, prefillingGet, datasetProfile, datasetManager); } private Map getSingle(String url, String id) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/DefaultPrefillingMapping.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/DefaultPrefillingMapping.java new file mode 100644 index 000000000..f348087c6 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/DefaultPrefillingMapping.java @@ -0,0 +1,38 @@ +package eu.eudat.logic.proxy.config.entities; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "mapping") +public class DefaultPrefillingMapping implements PrefillingMapping{ + private String source; + private String target; + private String maDmpTarget; + + public String getSource() { + return source; + } + + @XmlAttribute(name = "source") + public void setSource(String source) { + this.source = source; + } + + public String getTarget() { + return target; + } + + @XmlAttribute(name = "target") + public void setTarget(String target) { + this.target = target; + } + + public String getMaDmpTarget() { + return maDmpTarget; + } + + @XmlAttribute(name = "maDmpTarget") + public void setMaDmpTarget(String maDmpTarget) { + this.maDmpTarget = maDmpTarget; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingFixedMapping.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingFixedMapping.java new file mode 100644 index 000000000..d9b3309b6 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingFixedMapping.java @@ -0,0 +1,38 @@ +package eu.eudat.logic.proxy.config.entities; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "fixedMapping") +public class PrefillingFixedMapping implements PrefillingMapping{ + private String target; + private String maDmpTarget; + private String value; + + public String getTarget() { + return target; + } + + @XmlAttribute(name = "target") + public void setTarget(String target) { + this.target = target; + } + + public String getMaDmpTarget() { + return maDmpTarget; + } + + @XmlAttribute(name = "maDmpTarget") + public void setMaDmpTarget(String maDmpTarget) { + this.maDmpTarget = maDmpTarget; + } + + public String getValue() { + return value; + } + + @XmlAttribute(name = "value") + public void setValue(String value) { + this.value = value; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingGet.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingGet.java index c74b5f9f4..ab73b6d5f 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingGet.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingGet.java @@ -1,15 +1,13 @@ package eu.eudat.logic.proxy.config.entities; -import eu.eudat.logic.proxy.config.FetchStrategy; -import eu.eudat.logic.proxy.config.UrlConfiguration; - import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import java.util.List; public class PrefillingGet{ private String url; - private List mappings; + private List mappings; + private List fixedMappings; public String getUrl() { return url; @@ -20,13 +18,23 @@ public class PrefillingGet{ this.url = url; } - public List getMappings() { + public List getMappings() { return mappings; } @XmlElement(name = "mapping") @XmlElementWrapper - public void setMappings(List mappings) { + public void setMappings(List mappings) { this.mappings = mappings; } + + public List getFixedMappings() { + return fixedMappings; + } + + @XmlElement(name = "fixedMapping") + @XmlElementWrapper + public void setFixedMappings(List fixedMappings) { + this.fixedMappings = fixedMappings; + } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingMapping.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingMapping.java index a6d33296c..ce3422123 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingMapping.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/entities/PrefillingMapping.java @@ -1,38 +1,13 @@ package eu.eudat.logic.proxy.config.entities; import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlRootElement; -@XmlRootElement(name = "mapping") -public class PrefillingMapping { - private String source; - private String target; - private String maDmpTarget; +public interface PrefillingMapping { + String getTarget(); - public String getSource() { - return source; - } + void setTarget(String target); - @XmlAttribute(name = "source") - public void setSource(String source) { - this.source = source; - } + String getMaDmpTarget(); - public String getTarget() { - return target; - } - - @XmlAttribute(name = "target") - public void setTarget(String target) { - this.target = target; - } - - public String getMaDmpTarget() { - return maDmpTarget; - } - - @XmlAttribute(name = "maDmpTarget") - public void setMaDmpTarget(String maDmpTarget) { - this.maDmpTarget = maDmpTarget; - } + void setMaDmpTarget(String maDmpTarget); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java index 0758daaad..1ad2d666f 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java @@ -1,10 +1,14 @@ package eu.eudat.models.data.datasetwizard; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.entities.*; import eu.eudat.elastic.entities.Tag; import eu.eudat.logic.managers.DatasetManager; +import eu.eudat.logic.proxy.config.entities.PrefillingFixedMapping; +import eu.eudat.logic.proxy.config.entities.PrefillingGet; +import eu.eudat.logic.proxy.config.entities.DefaultPrefillingMapping; import eu.eudat.logic.proxy.config.entities.PrefillingMapping; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.DataModel; @@ -20,7 +24,6 @@ import net.minidev.json.JSONValue; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.*; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -305,7 +308,7 @@ public class DatasetWizardModel implements DataModel prefilledEntity, List mappings, + public static DatasetWizardModel fromPrefilledEntity(Map prefilledEntity, PrefillingGet prefillingGet, DatasetProfile profile, DatasetManager datasetManager) throws Exception { DatasetWizardModel datasetWizardModel = new DatasetWizardModel(); datasetWizardModel.setProfile(new DatasetProfileOverviewModel().fromDataModel(profile)); @@ -314,7 +317,7 @@ public class DatasetWizardModel implements DataModel properties = new HashMap<>(); ObjectMapper objectMapper = new ObjectMapper(); JsonNode parentNode = objectMapper.readTree(objectMapper.writeValueAsString(datasetManager.getPagedProfile(datasetWizardModel, dataset))); - for (PrefillingMapping prefillingMapping: mappings) { + for (DefaultPrefillingMapping prefillingMapping: prefillingGet.getMappings()) { List sourceKeys = Arrays.asList(prefillingMapping.getSource().split("\\.")); Object sourceValue = null; for (String sourceKey: sourceKeys) { @@ -324,28 +327,36 @@ public class DatasetWizardModel implements DataModel method.getName().equals(methodName)).collect(Collectors.toList()).get(0); - setterMethod.invoke(datasetWizardModel, sourceValue); - }catch (InvocationTargetException | IllegalAccessException e) { - throw e; - } - } else { - List nodes = JsonSearcher.findNodes(parentNode, "rdaProperty", prefillingMapping.getMaDmpTarget()); - for (JsonNode node: nodes) { - String id = node.get(0) != null ? node.get(0).get("id").asText() : node.get("id").asText(); - properties.put(id, sourceValue); - } - } + setValue(prefillingMapping, objectMapper.writeValueAsString(sourceValue), datasetWizardModel, parentNode, properties); + } + for (PrefillingFixedMapping fixedMapping: prefillingGet.getFixedMappings()) { + setValue(fixedMapping, fixedMapping.getValue(), datasetWizardModel, parentNode, properties); } dataset.setProperties(objectMapper.writeValueAsString(properties)); datasetWizardModel.setDatasetProfileDefinition(datasetManager.getPagedProfile(datasetWizardModel, dataset)); return datasetWizardModel; } + private static void setValue(PrefillingMapping prefillingMapping, String value, DatasetWizardModel datasetWizardModel, JsonNode parentNode, Map properties) throws InvocationTargetException, IllegalAccessException, JsonProcessingException { + if (prefillingMapping.getTarget() != null) { + try { + String methodName = "set" + prefillingMapping.getTarget().substring(0, 1).toUpperCase(Locale.ROOT) + prefillingMapping.getTarget().substring(1); + Method setterMethod = Arrays.stream(DatasetWizardModel.class.getDeclaredMethods()) + .filter(method -> method.getName().equals(methodName)).collect(Collectors.toList()).get(0); + Class[] params = setterMethod.getParameterTypes(); + setterMethod.invoke(datasetWizardModel, new ObjectMapper().readValue(value, params[0])); + }catch (InvocationTargetException | IllegalAccessException | JsonProcessingException e) { + throw e; + } + } else { + List nodes = JsonSearcher.findNodes(parentNode, "rdaProperty", prefillingMapping.getMaDmpTarget()); + for (JsonNode node: nodes) { + String id = node.get(0) != null ? node.get(0).get("id").asText() : node.get("id").asText(); + properties.put(id, value); + } + } + } + @Override public String getHint() { return "datasetWizardModel"; diff --git a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml index e272cae9a..96eaffd72 100644 --- a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml +++ b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml @@ -1034,6 +1034,10 @@ + + + + From 91f98a87594d3e52ef37757e5b09bdb90787d17d Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Wed, 12 Jan 2022 11:45:57 +0200 Subject: [PATCH 04/11] Rename sample tag --- .../web/src/main/resources/externalUrls/ExternalUrls.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml index 96eaffd72..770948fb9 100644 --- a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml +++ b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml @@ -1035,7 +1035,7 @@ - + From 9d5d685ad689d37663dbe25c53512dc5dd021ccf Mon Sep 17 00:00:00 2001 From: "DESKTOP-4ES9U2E\\aldom" Date: Wed, 12 Jan 2022 14:10:00 +0200 Subject: [PATCH 05/11] ethical_issues added to DatasetRDAMapper, minor fixes to dmp docx export --- .../utilities/documents/word/WordBuilder.java | 28 ++++++++- .../models/rda/mapper/DatasetRDAMapper.java | 58 +++++++++++++++++-- .../eudat/models/rda/mapper/DmpRDAMapper.java | 4 +- .../models/rda/mapper/HostRDAMapper.java | 17 +++--- .../models/rda/mapper/KeywordRDAMapper.java | 24 ++++---- 5 files changed, 98 insertions(+), 33 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java index 78c38db09..ad3dc1d27 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/documents/word/WordBuilder.java @@ -193,18 +193,26 @@ public class WordBuilder { boolean hasValue = false; for (FieldSet compositeField: compositeFields) { if (visibilityRuleService.isElementVisible(compositeField.getId()) && hasVisibleFields(compositeField, visibilityRuleService)) { + char c = 'a'; int paragraphPos = -1; if (compositeField.getTitle() != null && !compositeField.getTitle().isEmpty() && !createListing) { XWPFParagraph paragraph = addParagraphContent(page + "." + section + "." + (compositeField.getOrdinal() +1) + " " + compositeField.getTitle(), mainDocumentPart, ParagraphStyle.HEADER6, numId); CTDecimalNumber number = paragraph.getCTP().getPPr().getNumPr().addNewIlvl(); number.setVal(BigInteger.valueOf(indent)); paragraphPos = mainDocumentPart.getPosOfParagraph(paragraph); + if(compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()){ + addParagraphContent(c + ".\n", mainDocumentPart, ParagraphStyle.HEADER6, numId); + } } hasValue = createFields(compositeField.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService); if (compositeField.getMultiplicityItems() != null && !compositeField.getMultiplicityItems().isEmpty()) { List
list = compositeField.getMultiplicityItems().stream().sorted(Comparator.comparingInt(FieldSet::getOrdinal)).collect(Collectors.toList()); for (FieldSet multiplicityFieldset : list) { - hasValue = createFields(multiplicityFieldset.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService); + if(!createListing){ + c++; + addParagraphContent(c + ".\n", mainDocumentPart, ParagraphStyle.HEADER6, numId); + } + hasValue = createFields(multiplicityFieldset.getFields(), mainDocumentPart, 3, createListing, visibilityRuleService); } } if (hasValue && compositeField.getHasCommentField() && compositeField.getCommentFieldValue() != null && !compositeField.getCommentFieldValue().isEmpty() && !createListing) { @@ -224,6 +232,15 @@ public class WordBuilder { if (createListing) this.addListing(mainDocumentPart, indent, false, false); boolean hasValue = false; List tempFields = fields.stream().sorted(Comparator.comparingInt(Field::getOrdinal)).collect(Collectors.toList()); + List formats = tempFields.stream().filter(f -> { + try { + String fTemp = this.formatter(f); + return fTemp != null && !fTemp.isEmpty(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return false; + }).collect(Collectors.toList()); for (Field field: tempFields) { if (visibilityRuleService.isElementVisible(field.getId())) { if (!createListing) { @@ -231,8 +248,13 @@ public class WordBuilder { if (field.getValue() != null && !field.getValue().toString().isEmpty()) { this.indent = indent; String format = this.formatter(field); - if(format != null && format.charAt(0) == '['){ - format = format.substring(1, format.length() - 1).replaceAll(",", ", "); + if(format != null){ + if(format.charAt(0) == '['){ + format = format.substring(1, format.length() - 1).replaceAll(",", ", "); + } + if(formats.size() > 1){ + format = "\t• " + format; + } } XWPFParagraph paragraph = addParagraphContent(format, mainDocumentPart, field.getViewStyle().getRenderStyle().equals("richTextarea") ? ParagraphStyle.HTML : ParagraphStyle.TEXT, numId); if (paragraph != null) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java index cd0496a38..11644f0a4 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java @@ -17,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.transaction.Transactional; +import java.net.URI; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -40,7 +41,7 @@ public class DatasetRDAMapper { } @Transactional - public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, List contributors, List costs) { + public Dataset toRDA(eu.eudat.data.entities.Dataset dataset, eu.eudat.models.rda.Dmp dmp) { Dataset rda = new Dataset(); // rda.setDatasetId(DatasetIdRDAMapper.toRDA(dataset.getId())); if (dataset.getLabel() == null) { @@ -125,7 +126,11 @@ public class DatasetRDAMapper { } List personalDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.personal_data"); if (!personalDataNodes.isEmpty()) { - rda.setPersonalData(personalDataNodes.stream().map(personalDataNode -> Dataset.PersonalData.fromValue(personalDataNode.get("value").asText())).findFirst().get()); + try{ + rda.setPersonalData(personalDataNodes.stream().map(personalDataNode -> Dataset.PersonalData.fromValue(personalDataNode.get("value").asText())).findFirst().get()); + }catch(IllegalArgumentException e){ + rda.setPersonalData(Dataset.PersonalData.UNKNOWN); + } } else { rda.setPersonalData(Dataset.PersonalData.UNKNOWN); } @@ -137,7 +142,11 @@ public class DatasetRDAMapper { } List sensitiveDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.sensitive_data"); if (!sensitiveDataNodes.isEmpty()) { - rda.setSensitiveData(sensitiveDataNodes.stream().map(sensitiveDataNode -> Dataset.SensitiveData.fromValue(sensitiveDataNode.get("value").asText())).findFirst().get()); + try{ + rda.setSensitiveData(sensitiveDataNodes.stream().map(sensitiveDataNode -> Dataset.SensitiveData.fromValue(sensitiveDataNode.get("value").asText())).findFirst().get()); + }catch(IllegalArgumentException e){ + rda.setSensitiveData(Dataset.SensitiveData.UNKNOWN); + } } else { rda.setSensitiveData(Dataset.SensitiveData.UNKNOWN); } @@ -153,7 +162,7 @@ public class DatasetRDAMapper { } List contributorNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.contributor"); if (!contributorNodes.isEmpty()) { - contributors.addAll(contributorNodes.stream().map(contributorNode -> { + dmp.getContributor().addAll(contributorNodes.stream().map(contributorNode -> { JsonNode value = contributorNode.get("value"); if (value.isArray()) { return StreamSupport.stream(value.spliterator(), false).map(node -> ContributorRDAMapper.toRDA(node.asText())).collect(Collectors.toList()); @@ -164,7 +173,46 @@ public class DatasetRDAMapper { } List costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.cost"); if (!costNodes.isEmpty()) { - costs.addAll(CostRDAMapper.toRDAList(costNodes)); + dmp.getCost().addAll(CostRDAMapper.toRDAList(costNodes)); + } + List ethicsNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dmp.ethical_issues"); + if (!ethicsNodes.isEmpty()) { + for(JsonNode node: ethicsNodes){ + String rdaProperty = node.get("rdaProperty").asText(); + String rdaValue = node.get("value").asText(); + if(rdaValue == null || rdaValue.isEmpty()){ + continue; + } + if(rdaProperty.contains("exist")){ + try { + Dmp.EthicalIssuesExist exists = dmp.getEthicalIssuesExist(); + if(exists == null + || ((exists == Dmp.EthicalIssuesExist.NO || exists == Dmp.EthicalIssuesExist.UNKNOWN) && rdaValue.equals("yes")) + || (exists == Dmp.EthicalIssuesExist.YES && !(rdaValue.equals("no") || rdaValue.equals("unknown"))) + || (exists == Dmp.EthicalIssuesExist.UNKNOWN && rdaValue.equals("no"))){ + dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(rdaValue)); + } + }catch(IllegalArgumentException e){ + logger.warn(e.getLocalizedMessage() + ". Setting ethical_issues_exist to unknown"); + dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN); + } + } + else if(rdaProperty.contains("description")){ + if(dmp.getEthicalIssuesDescription() == null){ + dmp.setEthicalIssuesDescription(rdaValue); + } + else{ + dmp.setEthicalIssuesDescription(dmp.getEthicalIssuesDescription() + ", " + rdaValue); + } + } + else if(rdaProperty.contains("report")){ + try { + dmp.setEthicalIssuesReport(URI.create(rdaValue)); + } catch (IllegalArgumentException e) { + logger.warn(e.getLocalizedMessage() + ". Skipping url parsing"); + } + } + } } List foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes, keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).collect(Collectors.toList()); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java index d2abe28d6..539367e10 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java @@ -68,7 +68,7 @@ public class DmpRDAMapper { if (!extraProperties.isEmpty()) { if (extraProperties.get("ethicalIssues") != null) { - rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(extraProperties.get("ethicalIsses").toString())); + rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.fromValue(extraProperties.get("ethicalIssues").toString())); } else { rda.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN); } @@ -95,7 +95,7 @@ public class DmpRDAMapper { rda.getContributor().addAll(dmp.getResearchers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); } // rda.getContributor().addAll(dmp.getUsers().stream().map(ContributorRDAMapper::toRDA).collect(Collectors.toList())); - rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda.getContributor(), rda.getCost())).collect(Collectors.toList())); + rda.setDataset(dmp.getDataset().stream().filter(dataset -> dataset.getStatus() != eu.eudat.elastic.entities.Dmp.DMPStatus.DELETED.getValue()).map(dataset -> datasetRDAMapper.toRDA(dataset, rda)).collect(Collectors.toList())); rda.setProject(Collections.singletonList(ProjectRDAMapper.toRDA(dmp.getProject(), dmp.getGrant()))); rda.setAdditionalProperty("templates", dmp.getAssociatedDmps().stream().map(datasetProfile -> datasetProfile.getId().toString()).toArray()); return rda; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java index 88ccfe827..7e14486fc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java @@ -67,15 +67,14 @@ public class HostRDAMapper { while(iter.hasNext()) { pList.add(iter.next().asText()); } -// List pidList; -// if(pList.size() == 0){ -// pidList = Arrays.stream(rdaValue.replaceAll("[\\[\"\\]]","").split(",")) -// .map(PidSystem::fromValue).collect(Collectors.toList()); -// } -// else{ -// pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); -// } - List pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); + List pidList; + if(pList.size() == 0){ + pidList = Arrays.stream(rdaValue.replaceAll("[\\[\"\\]]","").split(",")) + .map(PidSystem::fromValue).collect(Collectors.toList()); + } + else{ + pidList = pList.stream().map(PidSystem::fromValue).collect(Collectors.toList()); + } rda.setPidSystem(pidList); rda.setAdditionalProperty(ImportPropertyName.PID_SYSTEM.getName(), node.get("id").asText()); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java index f39f92b48..dcdd0deac 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/KeywordRDAMapper.java @@ -15,21 +15,17 @@ public class KeywordRDAMapper { private static final Logger logger = LoggerFactory.getLogger(KeywordRDAMapper.class); public static List toRDA(String value) { - ObjectMapper mapper = new ObjectMapper(); - value = JavaToJson.objectStringToJson(value); - if (!value.isEmpty()) { - try { - List tags = Arrays.asList(mapper.readValue(value, Tag[].class)); - List keywordNames = tags.stream().map(Tag::getName).collect(Collectors.toList()); - return keywordNames; - } catch (JsonProcessingException e) { - logger.warn(e.getMessage() + ". Attempting to parse it as a String list."); - if(!value.isEmpty()) { - return new ArrayList<>(Arrays.asList(value.replace(" ", "").split(","))); - } - } - + if (!value.isEmpty()) { + try { + ObjectMapper mapper = new ObjectMapper(); + String valueJson = JavaToJson.objectStringToJson(value); + List tags = Arrays.asList(mapper.readValue(valueJson, Tag[].class)); + return tags.stream().map(Tag::getName).collect(Collectors.toList()); + } catch (JsonProcessingException e) { + logger.warn(e.getMessage() + ". Attempting to parse it as a String list."); + return new ArrayList<>(Arrays.asList(value.replace(" ", "").split(","))); } + } return new ArrayList<>(); } From 641088e289bed61a164c2bfe48fcbf1e550f3c83 Mon Sep 17 00:00:00 2001 From: Aldo Mihasi Date: Wed, 12 Jan 2022 16:31:07 +0200 Subject: [PATCH 06/11] Remove local configs, commited by mistake --- .../main/resources/config/application-devel.properties | 10 +++++----- .../src/main/resources/config/application.properties | 4 ++-- dmp-frontend/src/assets/config/config.json | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dmp-backend/web/src/main/resources/config/application-devel.properties b/dmp-backend/web/src/main/resources/config/application-devel.properties index 7a16d94fc..d7df68f89 100644 --- a/dmp-backend/web/src/main/resources/config/application-devel.properties +++ b/dmp-backend/web/src/main/resources/config/application-devel.properties @@ -1,9 +1,9 @@ dmp.domain = http://localhost:4200 ####################PERSISTENCE OVERRIDES CONFIGURATIONS########## -database.url=jdbc:postgresql://dbserver02.local.cite.gr:5432/dmptool -database.username=dmtadm -database.password=t00L4DM@18! +database.url= +database.username= +database.password= spring.datasource.maxIdle=10 spring.datasource.minIdle=5 spring.datasource.maxActive=10 @@ -38,7 +38,7 @@ facebook.login.clientSecret= facebook.login.namespace= #############GOOGLE LOGIN CONFIGURATIONS######### -google.login.clientId=524432312250-sc9qsmtmbvlv05r44onl6l93ia3k9deo.apps.googleusercontent.com +google.login.clientId= #############LINKEDIN LOGIN CONFIGURATIONS######### linkedin.login.clientId= @@ -86,7 +86,7 @@ zenodo.login.redirect_uri=http://localhost:4200/login/external/zenodo #############CONTACT EMAIL CONFIGURATIONS######### contact_email.mail= -language.path=C:\\Users\\aldom\\OneDrive\\Desktop\\CITE\\dmp-frontend\\src\\assets\\i18n\\ +language.path=dmp-frontend/src/assets/i18n/ #############LOGGING######### logging.config=classpath:logging/logback-${spring.profiles.active}.xml diff --git a/dmp-backend/web/src/main/resources/config/application.properties b/dmp-backend/web/src/main/resources/config/application.properties index 40f9ccd2b..9ae11ab94 100644 --- a/dmp-backend/web/src/main/resources/config/application.properties +++ b/dmp-backend/web/src/main/resources/config/application.properties @@ -98,7 +98,7 @@ database.lock-fail-interval=120000 ##########################MISC########################################## #############USER GUIDE######### -userguide.path=C:\\Users\\aldom\\OneDrive\\Desktop\\CITE\\user-guide +userguide.path=user-guide/ #############NOTIFICATION######### notification.rateInterval=30000 @@ -112,7 +112,7 @@ notification.modifiedFinalised.subject=[OpenDMP] The {name} has been modified an logging.config=classpath:logging/logback-${spring.profiles.active}.xml #############TEMP######### -temp.temp=C:\\Users\\aldom\\OneDrive\\Desktop\\CITE-TMP\\ +temp.temp=tmp/ #############ZENODO######### zenodo.affiliation=ARGOS diff --git a/dmp-frontend/src/assets/config/config.json b/dmp-frontend/src/assets/config/config.json index 35b504a22..ea878f347 100644 --- a/dmp-frontend/src/assets/config/config.json +++ b/dmp-frontend/src/assets/config/config.json @@ -10,7 +10,7 @@ "loginProviders": { "enabled": [1, 2, 3, 4, 5, 6, 7, 8], "facebookConfiguration": { "clientId": "" }, - "googleConfiguration": { "clientId": "524432312250-sc9qsmtmbvlv05r44onl6l93ia3k9deo.apps.googleusercontent.com" }, + "googleConfiguration": { "clientId": "" }, "linkedInConfiguration": { "clientId": "", "oauthUrl": "https://www.linkedin.com/oauth/v2/authorization", From 5a70ce7602022b27bfb54c9e20205a705658e3c8 Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Thu, 13 Jan 2022 13:26:30 +0200 Subject: [PATCH 07/11] Fix issue of not keeping dataset template users after version update --- .../java/eu/eudat/logic/managers/DatasetProfileManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java index 70edaa08e..dd3fa9a59 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java @@ -291,6 +291,8 @@ public class DatasetProfileManager { modelDefinition.setGroupId(oldDatasetProfile.getGroupId()); modelDefinition.setLanguage(oldDatasetProfile.getLanguage()); apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(modelDefinition); + eu.eudat.data.entities.DatasetProfile datasetProfile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(modelDefinition); + this.storeDatasetProfileUsers(datasetProfile, profile); return modelDefinition; } else { throw new DatasetProfileNewVersionException("Version to update not the latest."); From 6460d4087a6a20dff58544d9b65d689c9d08b71a Mon Sep 17 00:00:00 2001 From: "k.triantafyllou" Date: Fri, 14 Jan 2022 17:50:02 +0200 Subject: [PATCH 08/11] Change gitlab links to gitea --- .../ui/faq/faq-content/faq-content.component.html | 12 ++++++------ .../sidebar-footer/terms/terms.component.html | 4 ++-- dmp-frontend/src/assets/splash/about/faqs.html | 10 +++++----- .../src/assets/splash/resources/co-branding.html | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dmp-frontend/src/app/ui/faq/faq-content/faq-content.component.html b/dmp-frontend/src/app/ui/faq/faq-content/faq-content.component.html index 1ebb60fa3..7b30a95f3 100644 --- a/dmp-frontend/src/app/ui/faq/faq-content/faq-content.component.html +++ b/dmp-frontend/src/app/ui/faq/faq-content/faq-content.component.html @@ -16,7 +16,7 @@ familiarize users with basic RDM concepts and guide them throughout the process of writing DMPs. It also utilises the OpenAIRE pool of services and inferred sources to make DMPs more dynamic in use and easier to be completed and published. Argos is - based on the OpenDMP open source software and is available through the OpenAIRE Service catalogue and the . But, you can always find Argos at argos.openaire.eu . To access Argos software, please visit - https://gitlab.eudat.eu/dmp/OpenAIRE-EUDAT-DMP-service-pilot/tree/master + https://code-repo.d4science.org/MaDgiK-CITE/argos/src/branch/master .


@@ -462,7 +462,7 @@

Yes, it is. The OpenDMP software that Argos has deployed upon is open source code available under Apache 2.0 license. You may find more information about the software - here.


@@ -472,8 +472,8 @@

Of course! Please feel free to suggest new features and to actively contribute to Argos development via pull requests in GitLab. + href="https://code-repo.d4science.org/MaDgiK-CITE/argos/src/branch/master" + target="_blank">Gitea.


diff --git a/dmp-frontend/src/app/ui/sidebar/sidebar-footer/terms/terms.component.html b/dmp-frontend/src/app/ui/sidebar/sidebar-footer/terms/terms.component.html index e8f39d395..63ae0b084 100644 --- a/dmp-frontend/src/app/ui/sidebar/sidebar-footer/terms/terms.component.html +++ b/dmp-frontend/src/app/ui/sidebar/sidebar-footer/terms/terms.component.html @@ -9,7 +9,7 @@

The OpenDMP service was developed to provide a more flexible, collaborative environment with machine actionable solutions in writing, sharing and publishing Data Management Plans (DMPs). It is a product of cooperation between OpenAIRE AMKE and EUDAT CDI and is offered both as a software “OpenDMP '' and as an online service under the name “ARGOS”.

    -
  1. OpenDMP software is offered under the Free Open Source Software license  Apache 2.0, for further development and use by institutions and interested parties.
    2. +
  2. OpenDMP software is offered under the Free Open Source Software license  Apache 2.0, for further development and use by institutions and interested parties.
  3. ARGOS service is offered by OpenAIRE as part of its mission to support Open Science in the European Research Area, focusing on information linking and contextualisation that enriches its Research Graph. Use of ARGOS denotes agreement with the following terms:
    1. ARGOS is a software interface and a database with no storage capacity to store or preserve research data. The DMPs created are hosted in the OpenAIRE production environment for the sole purpose of exposing the DMP records once finalised (“published”). If assigned a DOI, the DMP records are linked to and preserved in Zenodo, the OpenAIRE’s repository service. The ARGOS service is made available for use free-of-charge for research, educational and informational purposes.
      2. @@ -31,4 +31,4 @@

      For any questions or comments you may have about the current Terms of Service, please contact us: argos@openaire.eu 

      - \ No newline at end of file + diff --git a/dmp-frontend/src/assets/splash/about/faqs.html b/dmp-frontend/src/assets/splash/about/faqs.html index 032cd893d..ace506821 100644 --- a/dmp-frontend/src/assets/splash/about/faqs.html +++ b/dmp-frontend/src/assets/splash/about/faqs.html @@ -105,7 +105,7 @@ writing DMPs. It also utilises the OpenAIRE pool of services and inferred sources to make DMPs more dynamic in use and easier to be completed and published. Argos is based on the OpenDMP open source software and is available through the OpenAIRE Service catalogue and the EOSC Catalogue. But, you can always find Argos at argos.openaire.eu . To access Argos software, please visit - https://gitlab.eudat.eu/dmp/OpenAIRE-EUDAT-DMP-service-pilot/tree/master + https://code-repo.d4science.org/MaDgiK-CITE/argos/src/branch/master . @@ -866,7 +866,7 @@
      Yes, it is. The OpenDMP software that Argos has deployed upon is open source code available under Apache 2.0 license. You may find more information about the software - here. + here.
      @@ -880,7 +880,7 @@
      Of course! Please feel free to suggest new features and to actively contribute to - Argos development via pull requests in GitLab. + Argos development via pull requests in Gitea.
      diff --git a/dmp-frontend/src/assets/splash/resources/co-branding.html b/dmp-frontend/src/assets/splash/resources/co-branding.html index f1a3966fa..3306d668a 100644 --- a/dmp-frontend/src/assets/splash/resources/co-branding.html +++ b/dmp-frontend/src/assets/splash/resources/co-branding.html @@ -78,15 +78,15 @@

      Open to all to suggest new features and to actively contribute to Argos development via pull requests of code in - - GitLab + + Gitea !
      Note that this page is under development.

      - +
      From 61de8c4df0583d32368afdf985b1a74c8282de69 Mon Sep 17 00:00:00 2001 From: Aldo Mihasi Date: Tue, 18 Jan 2022 11:38:24 +0200 Subject: [PATCH 09/11] dmp import fixes --- .../models/rda/mapper/DatasetRDAMapper.java | 97 ++++++++++++------- .../rda/mapper/DistributionRDAMapper.java | 19 +++- .../models/rda/mapper/HostRDAMapper.java | 10 +- 3 files changed, 89 insertions(+), 37 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java index 11644f0a4..6cbdcc98a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java @@ -1,11 +1,13 @@ package eu.eudat.models.rda.mapper; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.entities.DatasetProfile; import eu.eudat.elastic.entities.Tag; import eu.eudat.logic.managers.DatasetManager; import eu.eudat.logic.services.ApiContext; +import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel; import eu.eudat.models.data.datasetwizard.DatasetWizardModel; @@ -89,10 +91,27 @@ public class DatasetRDAMapper { for (int i = 0; i < qaNodes.size(); i++) { rda.setAdditionalProperty("qaId" + (i + 1), qaNodes.get(i).get("id").asText()); }*/ - List qaList = qaNodes.stream() - .map(qaNode -> qaNode.get("value").asText()) - .filter(qaNode -> !qaNode.isEmpty()) - .collect(Collectors.toList()); + List qaList = new ArrayList<>(); + String qa; + for(JsonNode node: qaNodes){ + if(node.get("value").isArray()){ + Iterator iter = node.get("value").elements(); + while(iter.hasNext()) { + qa = iter.next().asText(); + qaList.add(qa); + } + } + } + String data_quality; + for(JsonNode dqa: qaNodes){ + data_quality = dqa.get("value").asText(); + if(!data_quality.isEmpty()){ + qaList.add(data_quality); + rda.setAdditionalProperty("otherDQAID", dqa.get("id")); + rda.setAdditionalProperty("otherDQA", data_quality); + break; + } + } rda.setDataQualityAssurance(qaList); }else{ rda.setDataQualityAssurance(new ArrayList<>()); @@ -197,21 +216,21 @@ public class DatasetRDAMapper { dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN); } } - else if(rdaProperty.contains("description")){ - if(dmp.getEthicalIssuesDescription() == null){ - dmp.setEthicalIssuesDescription(rdaValue); - } - else{ - dmp.setEthicalIssuesDescription(dmp.getEthicalIssuesDescription() + ", " + rdaValue); - } - } - else if(rdaProperty.contains("report")){ - try { - dmp.setEthicalIssuesReport(URI.create(rdaValue)); - } catch (IllegalArgumentException e) { - logger.warn(e.getLocalizedMessage() + ". Skipping url parsing"); - } - } +// else if(rdaProperty.contains("description")){ +// if(dmp.getEthicalIssuesDescription() == null){ +// dmp.setEthicalIssuesDescription(rdaValue); +// } +// else{ +// dmp.setEthicalIssuesDescription(dmp.getEthicalIssuesDescription() + ", " + rdaValue); +// } +// } +// else if(rdaProperty.contains("report")){ +// try { +// dmp.setEthicalIssuesReport(URI.create(rdaValue)); +// } catch (IllegalArgumentException e) { +// logger.warn(e.getLocalizedMessage() + ". Skipping url parsing"); +// } +// } } } List foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes, @@ -282,7 +301,14 @@ public class DatasetRDAMapper { }*/ List qaNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.data_quality_assurance"); if (!qaNodes.isEmpty() && rda.getDataQualityAssurance() != null && !rda.getDataQualityAssurance().isEmpty()) { - properties.put(qaNodes.get(0).get("id").asText(), rda.getDataQualityAssurance().get(0)); + ObjectMapper m = new ObjectMapper(); + List qas = new ArrayList<>(rda.getDataQualityAssurance()); + if(!qas.isEmpty()){ + properties.put(qaNodes.get(0).get("id").asText(), m.writeValueAsString(qas)); + if(rda.getAdditionalProperties().containsKey("otherDQAID")){ + properties.put((String)rda.getAdditionalProperties().get("otherDQAID"), rda.getAdditionalProperties().get("otherDQA")); + } + } } List preservationNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.preservation_statement"); @@ -295,25 +321,28 @@ public class DatasetRDAMapper { properties.put(issuedNodes.get(0).get("id").asText(), rda.getIssued()); } - if (rda.getDistribution() != null) { + if (rda.getDistribution() != null && !rda.getDistribution().isEmpty()) { properties.putAll(DistributionRDAMapper.toProperties(rda.getDistribution().get(0), datasetDescriptionObj)); } if (rda.getKeyword() != null) { List keywordIds = rda.getAdditionalProperties().entrySet().stream().filter(entry -> entry.getKey().startsWith("keyword")).map(entry -> entry.getValue().toString()).collect(Collectors.toList()); - boolean takeAll = false; - if (keywordIds.size() < rda.getKeyword().size()) { - takeAll = true; - } - for (int i = 0; i < keywordIds.size(); i++) { - if (takeAll) { - List tags = new ArrayList<>(); - for (String keyword : rda.getKeyword()) { - tags.add(mapper.writeValueAsString(toTagEntity(keyword))); - } - properties.put(keywordIds.get(i), tags); - } else { - properties.put(keywordIds.get(i), mapper.writeValueAsString(toTagEntity(rda.getKeyword().get(i)))); +// boolean takeAll = false; +// if (keywordIds.size() < rda.getKeyword().size()) { +// takeAll = true; +// } + if(!rda.getKeyword().isEmpty()){ + for (int i = 0; i < keywordIds.size(); i++) { +// if (takeAll) { +// List tags = new ArrayList<>(); +// for (String keyword : rda.getKeyword()) { +// tags.add(mapper.writeValueAsString(toTagEntity(keyword))); +// } +// properties.put(keywordIds.get(i), tags); +// } else { +// properties.put(keywordIds.get(i), mapper.writeValueAsString(toTagEntity(rda.getKeyword().get(i)))); +// } + properties.put(keywordIds.get(i), rda.getKeyword().get(i)); } } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java index 413a41b9c..00eae64b7 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java @@ -86,12 +86,15 @@ public class DistributionRDAMapper { if(node.get("value").isArray()){ Iterator iter = node.get("value").elements(); List formats = new ArrayList<>(); + int i = 1; while(iter.hasNext()) { - String format = JavaToJson.objectStringToJson(iter.next().asText()); + JsonNode current = iter.next(); + String format = JavaToJson.objectStringToJson(current.asText()); try { Map result = new ObjectMapper().readValue(format, HashMap.class); format = result.get("label"); formats.add(format); + rda.setAdditionalProperty("format" + i++, new ObjectMapper().readTree(current.asText())); } catch(JsonProcessingException e){ logger.warn(e.getMessage()); @@ -206,7 +209,19 @@ public class DistributionRDAMapper { break; case FORMAT: if (rda.getFormat() != null && !rda.getFormat().isEmpty()) { - properties.put(distributionNode.get("id").asText(), rda.getFormat().get(0)); + Map additionalProperties = rda.getAdditionalProperties(); + List standardFormats = new ArrayList<>(); + ObjectMapper mapper = new ObjectMapper(); + rda.getAdditionalProperties().forEach((key, value) -> { + try { + if (key.matches("format\\d+")) { + standardFormats.add(additionalProperties.get(key)); + properties.put(distributionNode.get("id").asText(), mapper.writeValueAsString(standardFormats)); + } + } catch (JsonProcessingException e) { + logger.error(e.getMessage(), e); + } + }); } break; case LICENSE: diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java index 7e14486fc..5fb92aab1 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java @@ -40,6 +40,7 @@ public class HostRDAMapper { break; case BACKUP_TYPE: rda.setBackupType(rdaValue); + rda.setAdditionalProperty(ImportPropertyName.BACKUP_TYPE.getName(), node.get("id").asText()); break; case CERTIFIED_WITH: rda.setCertifiedWith(Host.CertifiedWith.fromValue(rdaValue)); @@ -145,7 +146,14 @@ public class HostRDAMapper { properties.put(entry.getValue().toString(), rda.getGeoLocation().value()); break; case PID_SYSTEM: - properties.put(entry.getValue().toString(), rda.getPidSystem().get(0).value()); + List pids = new ArrayList<>(); + ObjectMapper mapper = new ObjectMapper(); + for(PidSystem pid: rda.getPidSystem()){ + pids.add(pid.value()); + } + if(!pids.isEmpty()){ + properties.put(entry.getValue().toString(), mapper.writeValueAsString(pids)); + } break; case STORAGE_TYPE: properties.put(entry.getValue().toString(), rda.getStorageType()); From ce386ac04702cbf558d3cf68adfac8c99db0f4e4 Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Wed, 19 Jan 2022 12:12:20 +0200 Subject: [PATCH 10/11] Refactor Remote Fetcher's json parser --- .../logic/proxy/fetching/RemoteFetcher.java | 15 +- .../proxy/fetching/RemoteFetcherUtils.java | 133 +++++++----------- 2 files changed, 51 insertions(+), 97 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 53b408323..e19c14aee 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -304,23 +304,12 @@ public class RemoteFetcher { if (response.getHeaders().get("Content-Type").get(0).contains("json")) { DocumentContext jsonContext = JsonPath.parse(response.getBody()); - if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) { - results = RemoteFetcherUtils.getFromJsonWithSource(jsonContext, jsonDataPath); - } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu - results = RemoteFetcherUtils.getFromJsonWithParsedData(jsonContext, jsonDataPath); - } else if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) { + if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) { results = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType); - } else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) { - results = RemoteFetcherUtils.getFromJsonWithType(jsonContext, jsonDataPath); } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) { results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath); } else { - results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + (jsonDataPath.getFieldsUrlConfiguration().getName() != null ? jsonDataPath.getFieldsUrlConfiguration().getName(): "") - + (jsonDataPath.getFieldsUrlConfiguration().getDescription() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getDescription(): "") - + (jsonDataPath.getFieldsUrlConfiguration().getUri() !=null ? "," + jsonDataPath.getFieldsUrlConfiguration().getUri() : "") - + (jsonDataPath.getFieldsUrlConfiguration().getId() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getId(): "") + "]"), - new HashMap<>(1, 1)); + results = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath); } results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue))) .collect(Collectors.toList())); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java index a518ca2f7..ca2781f81 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java @@ -1,67 +1,30 @@ package eu.eudat.logic.proxy.fetching; +import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; import eu.eudat.logic.proxy.config.DataUrlConfiguration; import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.fetching.entities.Results; import net.minidev.json.JSONArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.*; import java.util.stream.Collectors; public class RemoteFetcherUtils { + private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class); - public static Results getFromJsonWithSource(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - return new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription() - + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() - + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"), + public static Results getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { + return new Results(parseData(jsonContext, jsonDataPath), new HashMap<>(1, 1)); } - public static Results getFromJsonWithParsedData(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - Results results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() - + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), - new HashMap<>(1, 1)); - List> fixedResults = results.getResults().stream().map(item -> { - for (int i = 0; i < 2; i++) { - String id; - if (i == 0) { - id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", ""); - } else { - id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", ""); - } - if (!(item.get(id) instanceof String)) { - Object obj = item.get(id); - if (obj instanceof JSONArray) { - JSONArray jarr = (JSONArray) obj; - if (jarr.get(0) instanceof String) { - item.put(id, jarr.get(0).toString()); - } else { - for (int j = 0; j < jarr.size(); j++) { - mapToMap(id, (Map)jarr.get(j), item, i == 1); - } - } - } else { - if (obj instanceof Map) { - mapToMap(id, (Map) obj, item, i == 1); - } else if (obj != null){ - item.put(id, obj.toString()); - } - } - } - } - return item; - }).collect(Collectors.toList()); - - return new Results(fixedResults, new HashMap<>(1, 1)); - } - public static Results getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcher remoteFetcher, String requestBody, String requestType) { - Results results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getPath() - + "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"), + Results results = new Results(parseData(jsonContext, jsonDataPath), new HashMap<>(1, 1)); List> multiResults = results.getResults().stream().map(result -> { @@ -74,34 +37,8 @@ public class RemoteFetcherUtils { return new Results(multiResults, new HashMap<>(1, 1)); } - public static Results getFromJsonWithType(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - List> tempRes = jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName() - + "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]"); - List> finalRes = new ArrayList<>(); - tempRes.forEach(map -> { - Map resMap = new HashMap<>(); - map.forEach((key, value) -> { - if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) { - resMap.put("tags", ((JSONArray) value).toJSONString()); - } else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) { - resMap.put(key, ((JSONArray) value).toJSONString()); - } else { - resMap.put(key, (String) value); - } - }); - - finalRes.add(resMap); - }); - - return new Results(finalRes, - new HashMap<>(1, 1)); - } - public static Results getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - Results results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName() - + "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"), + Results results = new Results(parseData(jsonContext, jsonDataPath), new HashMap<>(1, 1)); results.getResults().stream().forEach(entry -> { String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", "")); @@ -112,17 +49,45 @@ public class RemoteFetcherUtils { return results; } - private static void mapToMap(String key, Map source, Map destination, boolean isTitle) { - if (source != null) { - String content = source.get("content"); - /*if (isTitle) { - String classId = source.get("classid"); - if (classId.equals("main title")) { - destination.put(key, content); + private static List> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { + List > rawData = jsonContext.read(jsonDataPath.getPath()); + List> parsedData = new ArrayList<>(); + rawData.forEach(stringObjectMap -> { + parsedData.add(new LinkedHashMap<>()); + Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> { + String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1); + Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0); + try { + String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration())); + if (value != null) { + value = value.replace("'", ""); + if (stringObjectMap.containsKey(value)) { + parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri")))); + } + } + } catch (IllegalAccessException | InvocationTargetException e) { + logger.error(e.getLocalizedMessage(), e); } - } else {*/ - destination.put(key, content); -// } + }); + }); + return parsedData; + } + + private static String normalizeValue(Object value, boolean jsonString) { + if (value instanceof JSONArray) { + if (jsonString) { + return ((JSONArray)value).toJSONString(); + } + JSONArray jarr = (JSONArray) value; + if (jarr.get(0) instanceof String) { + return jarr.get(0).toString(); + } else { + return ((Map)jarr.get(0)).get("content"); + + } + } else if (value instanceof Map) { + return ((Map)value).get("content"); } + return value != null ? value.toString() : null; } } From 8f56b1874b2dff32b5ffb84df93d87f28045fe4e Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Wed, 19 Jan 2022 17:51:30 +0200 Subject: [PATCH 11/11] Add extra prefilling mapping fields and add better support for prefilling tags --- .../models/data/datasetwizard/DatasetWizardModel.java | 11 ++++++++++- .../src/main/resources/externalUrls/ExternalUrls.xml | 11 +++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java index 1ad2d666f..cf4e6da80 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/datasetwizard/DatasetWizardModel.java @@ -344,7 +344,16 @@ public class DatasetWizardModel implements DataModel method.getName().equals(methodName)).collect(Collectors.toList()).get(0); Class[] params = setterMethod.getParameterTypes(); - setterMethod.invoke(datasetWizardModel, new ObjectMapper().readValue(value, params[0])); + ObjectMapper mapper = new ObjectMapper(); + //GK: Tags Special logic + if (prefillingMapping.getTarget().equals("tags")) { + List rawTags = (List) mapper.readValue(value, params[0]); + if (rawTags.get(0) instanceof String) { + List parsedTags = rawTags.stream().map(rawTag -> new Tag((String) rawTag, (String) rawTag)).collect(Collectors.toList()); + value = mapper.writeValueAsString(parsedTags); + } + } + setterMethod.invoke(datasetWizardModel, mapper.readValue(value, params[0])); }catch (InvocationTargetException | IllegalAccessException | JsonProcessingException e) { throw e; } diff --git a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml index 770948fb9..7d055d3de 100644 --- a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml +++ b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml @@ -1033,10 +1033,17 @@ + + + + + + + + - - +