From 1715463422e3112b125daece4b90a8858b4f816a Mon Sep 17 00:00:00 2001 From: Aldo Mihasi Date: Fri, 14 Jul 2023 13:56:08 +0300 Subject: [PATCH] fix bugs in rda json export when rda semantics don't correspond to valid dataset field input type --- .../models/rda/mapper/CostRDAMapper.java | 22 ++++++++++--- .../models/rda/mapper/DatasetIdRDAMapper.java | 12 ++++++- .../models/rda/mapper/DatasetRDAMapper.java | 14 ++++++++- .../rda/mapper/DistributionRDAMapper.java | 13 ++++++-- .../eudat/models/rda/mapper/DmpRDAMapper.java | 1 - .../models/rda/mapper/HostRDAMapper.java | 31 +++++++++++++++---- .../models/rda/mapper/LicenseRDAMapper.java | 4 +++ .../models/rda/mapper/MetadataRDAMapper.java | 3 ++ .../mapper/SecurityAndPrivacyRDAMapper.java | 4 +++ .../mapper/TechnicalResourceRDAMapper.java | 4 +++ 10 files changed, 93 insertions(+), 15 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java index 0c6a340f8..6f31e570e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/CostRDAMapper.java @@ -6,6 +6,7 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.models.rda.Cost; import eu.eudat.models.rda.PidSystem; @@ -34,11 +35,14 @@ public class CostRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dmp.cost")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{ @@ -64,12 +68,22 @@ public class CostRDAMapper { rdaMap.put(key, rda); } if(rdaProperty.contains("value")){ - rda.setValue(Double.valueOf(rdaValue)); + try { + rda.setValue(Double.valueOf(rdaValue)); + } + catch (NumberFormatException e) { + logger.warn("Dmp cost value " + rdaValue + " is not valid. Cost value will not be set."); + } } else if(rdaProperty.contains("currency_code")){ - HashMap result = - new ObjectMapper().readValue(rdaValue, HashMap.class); - rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value"))); + try { + HashMap result = + new ObjectMapper().readValue(rdaValue, HashMap.class); + rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value"))); + } + catch (Exception e) { + logger.warn("Dmp cost currency code is not valid and will not be set."); + } } else if(rdaProperty.contains("title")){ Iterator iter = node.get("value").elements(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java index f3e690e75..50d535348 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetIdRDAMapper.java @@ -2,6 +2,7 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.rda.DatasetId; import org.slf4j.Logger; @@ -30,11 +31,14 @@ public class DatasetIdRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dataset.dataset_id")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{ @@ -76,7 +80,13 @@ public class DatasetIdRDAMapper { rda.setIdentifier(value); break; case TYPE: - rda.setType(DatasetId.Type.fromValue(value)); + try { + rda.setType(DatasetId.Type.fromValue(value)); + } + catch (IllegalArgumentException e){ + logger.warn("Type " + value + " from semantic rda.dataset.dataset_id.type was not found. Setting type to OTHER."); + rda.setType(DatasetId.Type.OTHER); + } break; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java index 4bcf83017..d7b29e2b3 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DatasetRDAMapper.java @@ -3,6 +3,7 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.entities.DatasetProfile; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.elastic.criteria.DatasetCriteria; import eu.eudat.elastic.entities.Tag; import eu.eudat.logic.managers.DatasetManager; @@ -74,7 +75,14 @@ public class DatasetRDAMapper { } List languageNodes = JsonSearcher.findNodes(datasetDescriptionObj, "schematics", "rda.dataset.language"); if (!languageNodes.isEmpty() && !languageNodes.get(0).get("value").asText().isEmpty()) { - rda.setLanguage(Language.fromValue(languageNodes.get(0).get("value").asText())); + String lang = languageNodes.get(0).get("value").asText(); + try { + rda.setLanguage(Language.fromValue(lang)); + } + catch (IllegalArgumentException e){ + logger.warn("Language " + lang + " from semantic rda.dataset.language was not found. Setting '" + dataset.getProfile().getLanguage() +"' as language from the dataset profile."); + rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(dataset.getProfile().getLanguage())); + } } else { rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(dataset.getProfile().getLanguage())); } @@ -188,6 +196,7 @@ public class DatasetRDAMapper { return Collections.singletonList(new Contributor()); } }).flatMap(Collection::stream).collect(Collectors.toList())); + dmp.setContributor(dmp.getContributor().stream().filter(contributor -> contributor.getContributorId() != null && contributor.getName() != null).collect(Collectors.toList())); } List costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "schematics", "rda.dmp.cost"); if (!costNodes.isEmpty()) { @@ -199,11 +208,14 @@ public class DatasetRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dmp.ethical_issues")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{ diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java index 547927c94..046d73d91 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DistributionRDAMapper.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.logic.utilities.json.JsonSearcher; @@ -74,8 +75,13 @@ public class DistributionRDAMapper { } break; case DATA_ACCESS: - rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue)); - rda.setAdditionalProperty(ImportPropertyName.DATA_ACCESS.getName(), node.get("id").asText()); + try { + rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue)); + rda.setAdditionalProperty(ImportPropertyName.DATA_ACCESS.getName(), node.get("id").asText()); + } + catch (IllegalArgumentException e) { + logger.warn("Distribution data access " + rdaValue + " from semantic distribution.data_access is not valid. Data access will not be set set."); + } break; case BYTE_SIZE: rda.setByteSize(Integer.parseInt(rdaValue)); @@ -376,11 +382,14 @@ public class DistributionRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dataset.distribution")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } return rdaProperty; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java index 2cf81d3bd..2b3a2f61a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/DmpRDAMapper.java @@ -3,7 +3,6 @@ package eu.eudat.models.rda.mapper; import eu.eudat.data.entities.*; import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.utilities.helpers.StreamDistinctBy; -import eu.eudat.models.rda.Cost; import eu.eudat.models.rda.Dmp; import eu.eudat.models.rda.DmpId; import net.minidev.json.JSONObject; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java index ca69c5cfb..74e5c2e64 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/HostRDAMapper.java @@ -3,6 +3,7 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.models.rda.Host; import eu.eudat.models.rda.PidSystem; @@ -22,11 +23,14 @@ public class HostRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dataset.distribution.host")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{ @@ -55,8 +59,13 @@ public class HostRDAMapper { rda.setAdditionalProperty(ImportPropertyName.BACKUP_TYPE.getName(), node.get("id").asText()); break; case CERTIFIED_WITH: - rda.setCertifiedWith(Host.CertifiedWith.fromValue(rdaValue)); - rda.setAdditionalProperty(ImportPropertyName.CERTIFIED_WITH.getName(), node.get("id").asText()); + try { + rda.setCertifiedWith(Host.CertifiedWith.fromValue(rdaValue)); + rda.setAdditionalProperty(ImportPropertyName.CERTIFIED_WITH.getName(), node.get("id").asText()); + } + catch (IllegalArgumentException e) { + logger.warn("Distribution host certified with " + rdaValue + "from semantic distribution.host.certified_with is not valid. Certified_with will not be set set."); + } break; case DESCRIPTION: rda.setDescription(rdaValue); @@ -70,8 +79,13 @@ public class HostRDAMapper { logger.warn(e.getLocalizedMessage() + ". Try to pass value as is"); } } - rda.setGeoLocation(Host.GeoLocation.fromValue(rdaValue)); - rda.setAdditionalProperty(ImportPropertyName.GEO_LOCATION.getName(), node.get("id").asText()); + try { + rda.setGeoLocation(Host.GeoLocation.fromValue(rdaValue)); + rda.setAdditionalProperty(ImportPropertyName.GEO_LOCATION.getName(), node.get("id").asText()); + } + catch (IllegalArgumentException e) { + logger.warn("Distribution host geo location " + rdaValue + "from semantic distribution.host.geo_location is not valid. Geo location will not be set set."); + } break; case PID_SYSTEM: try{ @@ -101,8 +115,13 @@ public class HostRDAMapper { rda.setAdditionalProperty(ImportPropertyName.STORAGE_TYPE.getName(), node.get("id").asText()); break; case SUPPORT_VERSIONING: - rda.setSupportVersioning(Host.SupportVersioning.fromValue(rdaValue)); - rda.setAdditionalProperty(ImportPropertyName.SUPPORT_VERSIONING.getName(), node.get("id").asText()); + try { + rda.setSupportVersioning(Host.SupportVersioning.fromValue(rdaValue)); + rda.setAdditionalProperty(ImportPropertyName.SUPPORT_VERSIONING.getName(), node.get("id").asText()); + } + catch (IllegalArgumentException e) { + logger.warn("Distribution host support versioning " + rdaValue + "from semantic distribution.host.support_versioning is not valid. Support versioning will not be set set."); + } break; case TITLE: rda.setTitle(rdaValue); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java index 8b2cc84e3..d7e178c14 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/LicenseRDAMapper.java @@ -1,6 +1,7 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.rda.License; import org.slf4j.Logger; @@ -20,11 +21,14 @@ public class LicenseRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dataset.distribution.license")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{ diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/MetadataRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/MetadataRDAMapper.java index fd9eb1d13..e8c28b7bb 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/MetadataRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/MetadataRDAMapper.java @@ -23,11 +23,14 @@ public class MetadataRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dataset.metadata")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{ diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java index 85f494917..1d560b91b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/SecurityAndPrivacyRDAMapper.java @@ -1,6 +1,7 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.models.rda.SecurityAndPrivacy; import org.slf4j.Logger; @@ -19,11 +20,14 @@ public class SecurityAndPrivacyRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dataset.security_and_privacy")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{ diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java index 709ee5ef3..31ad744fe 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/rda/mapper/TechnicalResourceRDAMapper.java @@ -1,6 +1,7 @@ package eu.eudat.models.rda.mapper; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.models.rda.TechnicalResource; import org.slf4j.Logger; @@ -19,11 +20,14 @@ public class TechnicalResourceRDAMapper { String rdaProperty = ""; JsonNode schematics = node.get("schematics"); if(schematics.isArray()){ + int index = 0; for(JsonNode schematic: schematics){ if(schematic.asText().startsWith("rda.dataset.technical_resource")){ rdaProperty = schematic.asText(); + ((ArrayNode)schematics).remove(index); break; } + index++; } } else{