fix bugs in rda json export when rda semantics don't correspond to valid dataset field input type

This commit is contained in:
Bernaldo Mihasi 2023-07-14 13:56:08 +03:00
parent a8ce14af70
commit 1715463422
10 changed files with 93 additions and 15 deletions

View File

@ -6,6 +6,7 @@ import java.util.stream.Collectors;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.logic.utilities.json.JavaToJson;
import eu.eudat.models.rda.Cost; import eu.eudat.models.rda.Cost;
import eu.eudat.models.rda.PidSystem; import eu.eudat.models.rda.PidSystem;
@ -34,11 +35,14 @@ public class CostRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dmp.cost")){ if(schematic.asText().startsWith("rda.dmp.cost")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{
@ -64,12 +68,22 @@ public class CostRDAMapper {
rdaMap.put(key, rda); rdaMap.put(key, rda);
} }
if(rdaProperty.contains("value")){ if(rdaProperty.contains("value")){
rda.setValue(Double.valueOf(rdaValue)); try {
rda.setValue(Double.valueOf(rdaValue));
}
catch (NumberFormatException e) {
logger.warn("Dmp cost value " + rdaValue + " is not valid. Cost value will not be set.");
}
} }
else if(rdaProperty.contains("currency_code")){ else if(rdaProperty.contains("currency_code")){
HashMap<String,String> result = try {
new ObjectMapper().readValue(rdaValue, HashMap.class); HashMap<String, String> result =
rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value"))); new ObjectMapper().readValue(rdaValue, HashMap.class);
rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value")));
}
catch (Exception e) {
logger.warn("Dmp cost currency code is not valid and will not be set.");
}
} }
else if(rdaProperty.contains("title")){ else if(rdaProperty.contains("title")){
Iterator<JsonNode> iter = node.get("value").elements(); Iterator<JsonNode> iter = node.get("value").elements();

View File

@ -2,6 +2,7 @@ package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.rda.DatasetId; import eu.eudat.models.rda.DatasetId;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -30,11 +31,14 @@ public class DatasetIdRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dataset.dataset_id")){ if(schematic.asText().startsWith("rda.dataset.dataset_id")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{
@ -76,7 +80,13 @@ public class DatasetIdRDAMapper {
rda.setIdentifier(value); rda.setIdentifier(value);
break; break;
case TYPE: case TYPE:
rda.setType(DatasetId.Type.fromValue(value)); try {
rda.setType(DatasetId.Type.fromValue(value));
}
catch (IllegalArgumentException e){
logger.warn("Type " + value + " from semantic rda.dataset.dataset_id.type was not found. Setting type to OTHER.");
rda.setType(DatasetId.Type.OTHER);
}
break; break;
} }
} }

View File

@ -3,6 +3,7 @@ package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.entities.DatasetProfile; import eu.eudat.data.entities.DatasetProfile;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.elastic.criteria.DatasetCriteria; import eu.eudat.elastic.criteria.DatasetCriteria;
import eu.eudat.elastic.entities.Tag; import eu.eudat.elastic.entities.Tag;
import eu.eudat.logic.managers.DatasetManager; import eu.eudat.logic.managers.DatasetManager;
@ -74,7 +75,14 @@ public class DatasetRDAMapper {
} }
List<JsonNode> languageNodes = JsonSearcher.findNodes(datasetDescriptionObj, "schematics", "rda.dataset.language"); List<JsonNode> languageNodes = JsonSearcher.findNodes(datasetDescriptionObj, "schematics", "rda.dataset.language");
if (!languageNodes.isEmpty() && !languageNodes.get(0).get("value").asText().isEmpty()) { if (!languageNodes.isEmpty() && !languageNodes.get(0).get("value").asText().isEmpty()) {
rda.setLanguage(Language.fromValue(languageNodes.get(0).get("value").asText())); String lang = languageNodes.get(0).get("value").asText();
try {
rda.setLanguage(Language.fromValue(lang));
}
catch (IllegalArgumentException e){
logger.warn("Language " + lang + " from semantic rda.dataset.language was not found. Setting '" + dataset.getProfile().getLanguage() +"' as language from the dataset profile.");
rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(dataset.getProfile().getLanguage()));
}
} else { } else {
rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(dataset.getProfile().getLanguage())); rda.setLanguage(LanguageRDAMapper.mapLanguageIsoToRDAIso(dataset.getProfile().getLanguage()));
} }
@ -188,6 +196,7 @@ public class DatasetRDAMapper {
return Collections.singletonList(new Contributor()); return Collections.singletonList(new Contributor());
} }
}).flatMap(Collection::stream).collect(Collectors.toList())); }).flatMap(Collection::stream).collect(Collectors.toList()));
dmp.setContributor(dmp.getContributor().stream().filter(contributor -> contributor.getContributorId() != null && contributor.getName() != null).collect(Collectors.toList()));
} }
List<JsonNode> costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "schematics", "rda.dmp.cost"); List<JsonNode> costNodes = JsonSearcher.findNodes(datasetDescriptionObj, "schematics", "rda.dmp.cost");
if (!costNodes.isEmpty()) { if (!costNodes.isEmpty()) {
@ -199,11 +208,14 @@ public class DatasetRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dmp.ethical_issues")){ if(schematic.asText().startsWith("rda.dmp.ethical_issues")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{

View File

@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.logic.utilities.helpers.MyStringUtils;
import eu.eudat.logic.utilities.json.JavaToJson; import eu.eudat.logic.utilities.json.JavaToJson;
import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.logic.utilities.json.JsonSearcher;
@ -74,8 +75,13 @@ public class DistributionRDAMapper {
} }
break; break;
case DATA_ACCESS: case DATA_ACCESS:
rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue)); try {
rda.setAdditionalProperty(ImportPropertyName.DATA_ACCESS.getName(), node.get("id").asText()); rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue));
rda.setAdditionalProperty(ImportPropertyName.DATA_ACCESS.getName(), node.get("id").asText());
}
catch (IllegalArgumentException e) {
logger.warn("Distribution data access " + rdaValue + " from semantic distribution.data_access is not valid. Data access will not be set set.");
}
break; break;
case BYTE_SIZE: case BYTE_SIZE:
rda.setByteSize(Integer.parseInt(rdaValue)); rda.setByteSize(Integer.parseInt(rdaValue));
@ -376,11 +382,14 @@ public class DistributionRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dataset.distribution")){ if(schematic.asText().startsWith("rda.dataset.distribution")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
return rdaProperty; return rdaProperty;

View File

@ -3,7 +3,6 @@ package eu.eudat.models.rda.mapper;
import eu.eudat.data.entities.*; import eu.eudat.data.entities.*;
import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.utilities.helpers.StreamDistinctBy; import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
import eu.eudat.models.rda.Cost;
import eu.eudat.models.rda.Dmp; import eu.eudat.models.rda.Dmp;
import eu.eudat.models.rda.DmpId; import eu.eudat.models.rda.DmpId;
import net.minidev.json.JSONObject; import net.minidev.json.JSONObject;

View File

@ -3,6 +3,7 @@ package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.logic.utilities.helpers.MyStringUtils;
import eu.eudat.models.rda.Host; import eu.eudat.models.rda.Host;
import eu.eudat.models.rda.PidSystem; import eu.eudat.models.rda.PidSystem;
@ -22,11 +23,14 @@ public class HostRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dataset.distribution.host")){ if(schematic.asText().startsWith("rda.dataset.distribution.host")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{
@ -55,8 +59,13 @@ public class HostRDAMapper {
rda.setAdditionalProperty(ImportPropertyName.BACKUP_TYPE.getName(), node.get("id").asText()); rda.setAdditionalProperty(ImportPropertyName.BACKUP_TYPE.getName(), node.get("id").asText());
break; break;
case CERTIFIED_WITH: case CERTIFIED_WITH:
rda.setCertifiedWith(Host.CertifiedWith.fromValue(rdaValue)); try {
rda.setAdditionalProperty(ImportPropertyName.CERTIFIED_WITH.getName(), node.get("id").asText()); rda.setCertifiedWith(Host.CertifiedWith.fromValue(rdaValue));
rda.setAdditionalProperty(ImportPropertyName.CERTIFIED_WITH.getName(), node.get("id").asText());
}
catch (IllegalArgumentException e) {
logger.warn("Distribution host certified with " + rdaValue + "from semantic distribution.host.certified_with is not valid. Certified_with will not be set set.");
}
break; break;
case DESCRIPTION: case DESCRIPTION:
rda.setDescription(rdaValue); rda.setDescription(rdaValue);
@ -70,8 +79,13 @@ public class HostRDAMapper {
logger.warn(e.getLocalizedMessage() + ". Try to pass value as is"); logger.warn(e.getLocalizedMessage() + ". Try to pass value as is");
} }
} }
rda.setGeoLocation(Host.GeoLocation.fromValue(rdaValue)); try {
rda.setAdditionalProperty(ImportPropertyName.GEO_LOCATION.getName(), node.get("id").asText()); rda.setGeoLocation(Host.GeoLocation.fromValue(rdaValue));
rda.setAdditionalProperty(ImportPropertyName.GEO_LOCATION.getName(), node.get("id").asText());
}
catch (IllegalArgumentException e) {
logger.warn("Distribution host geo location " + rdaValue + "from semantic distribution.host.geo_location is not valid. Geo location will not be set set.");
}
break; break;
case PID_SYSTEM: case PID_SYSTEM:
try{ try{
@ -101,8 +115,13 @@ public class HostRDAMapper {
rda.setAdditionalProperty(ImportPropertyName.STORAGE_TYPE.getName(), node.get("id").asText()); rda.setAdditionalProperty(ImportPropertyName.STORAGE_TYPE.getName(), node.get("id").asText());
break; break;
case SUPPORT_VERSIONING: case SUPPORT_VERSIONING:
rda.setSupportVersioning(Host.SupportVersioning.fromValue(rdaValue)); try {
rda.setAdditionalProperty(ImportPropertyName.SUPPORT_VERSIONING.getName(), node.get("id").asText()); rda.setSupportVersioning(Host.SupportVersioning.fromValue(rdaValue));
rda.setAdditionalProperty(ImportPropertyName.SUPPORT_VERSIONING.getName(), node.get("id").asText());
}
catch (IllegalArgumentException e) {
logger.warn("Distribution host support versioning " + rdaValue + "from semantic distribution.host.support_versioning is not valid. Support versioning will not be set set.");
}
break; break;
case TITLE: case TITLE:
rda.setTitle(rdaValue); rda.setTitle(rdaValue);

View File

@ -1,6 +1,7 @@
package eu.eudat.models.rda.mapper; package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.rda.License; import eu.eudat.models.rda.License;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -20,11 +21,14 @@ public class LicenseRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dataset.distribution.license")){ if(schematic.asText().startsWith("rda.dataset.distribution.license")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{

View File

@ -23,11 +23,14 @@ public class MetadataRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dataset.metadata")){ if(schematic.asText().startsWith("rda.dataset.metadata")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{

View File

@ -1,6 +1,7 @@
package eu.eudat.models.rda.mapper; package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.logic.utilities.helpers.MyStringUtils;
import eu.eudat.models.rda.SecurityAndPrivacy; import eu.eudat.models.rda.SecurityAndPrivacy;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -19,11 +20,14 @@ public class SecurityAndPrivacyRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dataset.security_and_privacy")){ if(schematic.asText().startsWith("rda.dataset.security_and_privacy")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{

View File

@ -1,6 +1,7 @@
package eu.eudat.models.rda.mapper; package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.logic.utilities.helpers.MyStringUtils;
import eu.eudat.models.rda.TechnicalResource; import eu.eudat.models.rda.TechnicalResource;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -19,11 +20,14 @@ public class TechnicalResourceRDAMapper {
String rdaProperty = ""; String rdaProperty = "";
JsonNode schematics = node.get("schematics"); JsonNode schematics = node.get("schematics");
if(schematics.isArray()){ if(schematics.isArray()){
int index = 0;
for(JsonNode schematic: schematics){ for(JsonNode schematic: schematics){
if(schematic.asText().startsWith("rda.dataset.technical_resource")){ if(schematic.asText().startsWith("rda.dataset.technical_resource")){
rdaProperty = schematic.asText(); rdaProperty = schematic.asText();
((ArrayNode)schematics).remove(index);
break; break;
} }
index++;
} }
} }
else{ else{