diff --git a/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoader.java b/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoader.java index e279014..71c3760 100644 --- a/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoader.java +++ b/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoader.java @@ -4,6 +4,9 @@ import java.util.List; public interface ConfigLoader { List getDOIFunders(); + List getRelatedIdentifiers(); + List getAcceptedPidTypes(); + PidFieldNames getPidFieldNames(); byte[] getLogo(); ZenodoConfig getZenodoConfig(); } diff --git a/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoaderImpl.java b/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoaderImpl.java index 450072a..7e3a85a 100644 --- a/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoaderImpl.java +++ b/src/main/java/eu/eudat/depositinterface/zenodorepository/config/ConfigLoaderImpl.java @@ -19,11 +19,18 @@ public class ConfigLoaderImpl implements ConfigLoader{ private static final ObjectMapper mapper = new ObjectMapper(); private List doiFunders = new ArrayList<>(); + private List relatedIdentifiers = new ArrayList<>(); + private List acceptedPidTypes = new ArrayList<>(); + private PidFieldNames pidFieldNames = new PidFieldNames(); private ZenodoConfig zenodoConfig; - @Autowired private Environment environment; + @Autowired + public ConfigLoaderImpl(Environment environment){ + this.environment = environment; + } + @Override public List getDOIFunders() { if (doiFunders == null || doiFunders.isEmpty()) { @@ -37,6 +44,35 @@ public class ConfigLoaderImpl implements ConfigLoader{ return doiFunders; } + @Override + public List getRelatedIdentifiers() { + if (relatedIdentifiers == null || relatedIdentifiers.isEmpty()) { + BufferedReader ids = new BufferedReader(new InputStreamReader(getStreamFromPath("relatedIdentifiers.txt"))); + relatedIdentifiers = ids.lines().collect(Collectors.toList()); + } + return relatedIdentifiers; + } + + @Override + public List getAcceptedPidTypes() { + if (acceptedPidTypes == null || acceptedPidTypes.isEmpty()) { + BufferedReader ids = new BufferedReader(new InputStreamReader(getStreamFromPath("acceptedPidTypes.txt"))); + acceptedPidTypes = ids.lines().collect(Collectors.toList()); + } + return acceptedPidTypes; + } + + @Override + public PidFieldNames getPidFieldNames() { + try { + pidFieldNames = mapper.readValue(getStreamFromPath("datasetFieldsPid.json"), PidFieldNames.class); + } + catch (IOException e){ + logger.error(e.getLocalizedMessage(), e); + } + return pidFieldNames; + } + @Override public ZenodoConfig getZenodoConfig() { if (zenodoConfig == null) { diff --git a/src/main/java/eu/eudat/depositinterface/zenodorepository/config/PidFieldNames.java b/src/main/java/eu/eudat/depositinterface/zenodorepository/config/PidFieldNames.java new file mode 100644 index 0000000..873494b --- /dev/null +++ b/src/main/java/eu/eudat/depositinterface/zenodorepository/config/PidFieldNames.java @@ -0,0 +1,31 @@ +package eu.eudat.depositinterface.zenodorepository.config; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class PidFieldNames { + @JsonProperty("pidName") + private String pidName; + @JsonProperty("pidTypeName") + private String pidTypeName; + + public PidFieldNames() {} + + public PidFieldNames(String pidName, String pidTypeName) { + this.pidName = pidName; + this.pidTypeName = pidTypeName; + } + + public String getPidName() { + return pidName; + } + public void setPidName(String pidName) { + this.pidName = pidName; + } + + public String getPidTypeName() { + return pidTypeName; + } + public void setPidTypeName(String pidTypeName) { + this.pidTypeName = pidTypeName; + } +} diff --git a/src/main/java/eu/eudat/depositinterface/zenodorepository/interfaces/ZenodoDeposit.java b/src/main/java/eu/eudat/depositinterface/zenodorepository/interfaces/ZenodoDeposit.java index 58c781c..5a25ba6 100644 --- a/src/main/java/eu/eudat/depositinterface/zenodorepository/interfaces/ZenodoDeposit.java +++ b/src/main/java/eu/eudat/depositinterface/zenodorepository/interfaces/ZenodoDeposit.java @@ -55,7 +55,7 @@ public class ZenodoDeposit implements RepositoryDeposit { HttpHeaders headers = new HttpHeaders(); headers.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON)); headers.setContentType(MediaType.APPLICATION_JSON); - eu.eudat.depositinterface.zenodorepository.models.ZenodoDeposit deposit = DMPToZenodoMapper.fromDMP(dmpDepositModel, "argos", "ARGOS", "https://argos.openaire.eu/", this.configLoader.getDOIFunders()); + eu.eudat.depositinterface.zenodorepository.models.ZenodoDeposit deposit = DMPToZenodoMapper.fromDMP(dmpDepositModel); HttpEntity request = new HttpEntity<>(deposit, headers); Map createResponse; diff --git a/src/main/java/eu/eudat/depositinterface/zenodorepository/mapper/DMPToZenodoMapper.java b/src/main/java/eu/eudat/depositinterface/zenodorepository/mapper/DMPToZenodoMapper.java index 64b8e73..a6a2133 100644 --- a/src/main/java/eu/eudat/depositinterface/zenodorepository/mapper/DMPToZenodoMapper.java +++ b/src/main/java/eu/eudat/depositinterface/zenodorepository/mapper/DMPToZenodoMapper.java @@ -3,21 +3,126 @@ package eu.eudat.depositinterface.zenodorepository.mapper; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.eudat.depositinterface.enums.FieldType; import eu.eudat.depositinterface.models.*; +import eu.eudat.depositinterface.zenodorepository.config.ConfigLoader; import eu.eudat.depositinterface.zenodorepository.config.DOIFunder; +import eu.eudat.depositinterface.zenodorepository.config.PidFieldNames; import eu.eudat.depositinterface.zenodorepository.models.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; import java.time.Instant; import java.util.*; import java.util.stream.Collectors; +@Component public class DMPToZenodoMapper { + private static final Logger logger = LoggerFactory.getLogger(DMPToZenodoMapper.class); private static final ObjectMapper objectMapper = new ObjectMapper(); - public static ZenodoDeposit fromDMP(DMPDepositModel dmp, String zenodoCommunity, String zenodoAffiliation, String domain, List doiFunders) throws JsonProcessingException { + private static ConfigLoader configLoader; + private static Environment environment; + + private static PidFieldNames pidFieldNames; + + @Autowired + public DMPToZenodoMapper(ConfigLoader configL, Environment env){ + configLoader = configL; + pidFieldNames = configLoader.getPidFieldNames(); + environment = env; + } + + private static List findSchemanticValues(String relatedId, List fields){ + return fields.stream().filter(f -> f.getSchematics().contains(relatedId)).collect(Collectors.toList()); + } + + private static Set extractSchemanticValues(List fields, List acceptedPidTypes) throws JsonProcessingException{ + Set values = new HashSet<>(); + for(DatasetFieldsDepositModel field: fields){ + String value = (String) field.getValue(); + if(value != null && !value.isEmpty()) { + switch (FieldType.fromName(field.getRenderStyleType())) { + case FREE_TEXT: + case TEXT_AREA: + case RICH_TEXT_AREA: + case RADIO_BOX: + case DATE_PICKER: + values.add(value); + break; + case COMBO_BOX: + if (field.isMultiple()) { + List selected = objectMapper.readValue(value, new TypeReference>() {}); + values.addAll(selected); + } + else { + values.add(value); + } + break; + case REGISTRIES: + case SERVICES: + case EXTERNAL_DATASETS: + case DATA_REPOSITORIES: + case PUB_REPOSITORIES: + case JOURNAL_REPOSITORIES: + case TAXONOMIES: + case PUBLICATIONS: + if (field.isMultiple()) { + List selected = objectMapper.readValue(value, new TypeReference>() {}); + for (String s : selected) { + Map valueMap = objectMapper.readValue(s, new TypeReference>() {}); + String pid = valueMap.get(pidFieldNames.getPidName()); + String pidType = valueMap.get(pidFieldNames.getPidTypeName()); + if (acceptedPidTypes.contains(pidType)) { + values.add(pid); + } + } + } + else { + Map valueMap = objectMapper.readValue(value, new TypeReference>() {}); + String pid = valueMap.get(pidFieldNames.getPidName()); + String pidType = valueMap.get(pidFieldNames.getPidTypeName()); + if (acceptedPidTypes.contains(pidType)) { + values.add(pid); + } + } + break; + case ORGANIZATIONS: + case RESEARCHERS: + if (field.isMultiple()) { + List selected = objectMapper.readValue(value, new TypeReference>() {}); + for (String s : selected) { + Map valueMap = objectMapper.readValue(s, new TypeReference>() {}); + String pid = valueMap.get("reference"); + if(pid != null) { + values.add(pid); + } + } + } + else { + Map valueMap = objectMapper.readValue(value, new TypeReference>() {}); + String pid = valueMap.get("reference"); + if(pid != null) { + values.add(pid); + } + } + break; + case DATASET_IDENTIFIER: + Map valueMap = objectMapper.readValue(value, new TypeReference>() {}); + values.add(valueMap.get("identifier")); + break; + } + } + } + return values; + } + + public static ZenodoDeposit fromDMP(DMPDepositModel dmp) throws JsonProcessingException { Map extraProperties = dmp.getExtraProperties() != null ? new org.json.JSONObject(dmp.getExtraProperties()).toMap() : new HashMap<>(); ZenodoDeposit deposit = new ZenodoDeposit(); @@ -31,115 +136,21 @@ public class DMPToZenodoMapper { List keywords = new ArrayList<>(); List references = new ArrayList<>(); - //objectMapper.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES); + List acceptedPidTypes = configLoader.getAcceptedPidTypes(); + for(DatasetDepositModel dataset: dmp.getDatasets()){ - for(DatasetFieldsDepositModel field: dataset.getFields()){ - for(String schematic: field.getSchematics()){ - if(schematic.contains("zenodo.")){ - if(field.getValue() != null && !field.getValue().isEmpty()){ - if(schematic.contains("related_identifiers")){ - if(schematic.endsWith("relation")){ - Optional relatorOptional = relatedIdentifiers.stream().filter(r -> r.getRelation() == null).findAny(); - if(relatorOptional.isPresent()){ - relatorOptional.get().setRelation(field.getValue()); - } - else{ - ZenodoRelator relator = new ZenodoRelator(); - relator.setRelation(field.getValue()); - relatedIdentifiers.add(relator); - } - } - else if(schematic.endsWith("identifier")){ - Optional relatorOptional = relatedIdentifiers.stream().filter(r -> r.getIdentifier() == null).findAny(); - if(relatorOptional.isPresent()){ - relatorOptional.get().setIdentifier(field.getValue()); - } - else{ - ZenodoRelator relator = new ZenodoRelator(); - relator.setIdentifier(field.getValue()); - relatedIdentifiers.add(relator); - } - } - else{ - JsonNode relatorNodes = objectMapper.readTree(field.getValue()); - if(relatorNodes.isArray()){ - List relators = objectMapper.readValue(field.getValue(), new TypeReference>(){}); - relatedIdentifiers.addAll(relators); - } - else{ - ZenodoRelator relator = objectMapper.readValue(field.getValue(), ZenodoRelator.class); - relatedIdentifiers.add(relator); - } - } - } - else if(schematic.contains("communities")){ - JsonNode communitiesNodes = objectMapper.readTree(field.getValue()); - if(communitiesNodes.isArray()){ - List communitiesList = objectMapper.readValue(field.getValue(), new TypeReference>(){}); - communities.addAll(communitiesList); - } - else{ - ZenodoComunity community = objectMapper.readValue(field.getValue(), ZenodoComunity.class); - communities.add(community); - } - } - else if(schematic.contains("contributors")){ - JsonNode contributorsNodes = objectMapper.readTree(field.getValue()); - if(contributorsNodes.isArray()){ - List contributorsList = objectMapper.readValue(field.getValue(), new TypeReference>(){}); - contributors.addAll(contributorsList); - } - else{ - ZenodoContributor contributor = objectMapper.readValue(field.getValue(), ZenodoContributor.class); - contributors.add(contributor); - } - } - else if(schematic.contains("creators")){ - JsonNode creatorsNodes = objectMapper.readTree(field.getValue()); - if(creatorsNodes.isArray()){ - List creatorsList = objectMapper.readValue(field.getValue(), new TypeReference>(){}); - creators.addAll(creatorsList); - } - else{ - ZenodoCreator creator = objectMapper.readValue(field.getValue(), ZenodoCreator.class); - creators.add(creator); - } - } - else if(schematic.contains("grants")){ - JsonNode grantsNodes = objectMapper.readTree(field.getValue()); - if(grantsNodes.isArray()){ - List grantsList = objectMapper.readValue(field.getValue(), new TypeReference>(){}); - grants.addAll(grantsList); - } - else{ - ZenodoGrant grant = objectMapper.readValue(field.getValue(), ZenodoGrant.class); - grants.add(grant); - } - } - else if(schematic.contains("keywords")){ - if(field.getValue().startsWith("[")){ - List keywordsList = objectMapper.readValue(field.getValue(), new TypeReference>(){}); - keywords.addAll(keywordsList); - } - else{ - keywords.add(field.getValue()); - } - } - else if(schematic.contains("references")){ - if(field.getValue().startsWith("[")){ - List referencesList = objectMapper.readValue(field.getValue(), new TypeReference>(){}); - references.addAll(referencesList); - } - else{ - references.add(field.getValue()); - } - } - else{ - schematicsMap.put(schematic.replace("zenodo.", ""), field.getValue()); - } - } - } + + for(String relatedId: configLoader.getRelatedIdentifiers()){ + + List fields = findSchemanticValues(relatedId, dataset.getFields()); + Set values = extractSchemanticValues(fields, acceptedPidTypes); + for(String value: values){ + ZenodoRelator relator = new ZenodoRelator(); + relator.setRelation(relatedId.substring(relatedId.lastIndexOf(".") + 1)); + relator.setIdentifier(value); + relatedIdentifiers.add(relator); } + } } schematicsMap.put("related_identifiers", relatedIdentifiers); @@ -160,7 +171,8 @@ public class DMPToZenodoMapper { deposit.getMetadata().setPublicationType("datamanagementplan"); deposit.getMetadata().setDescription((dmp.getDescription() != null && !dmp.getDescription().isEmpty() ? dmp.getDescription() : "

")); deposit.getMetadata().setVersion(String.valueOf(dmp.getVersion())); - if(zenodoCommunity != null && !zenodoAffiliation.isEmpty()) { + String zenodoCommunity = environment.getProperty("zenodo.community"); + if(zenodoCommunity != null && !zenodoCommunity.isEmpty()) { ZenodoComunity community = new ZenodoComunity(); community.setIdentifier(zenodoCommunity); deposit.getMetadata().getCommunities().add(community); @@ -188,10 +200,11 @@ public class DMPToZenodoMapper { } if (dmp.isPublic()) { ZenodoRelator relator = new ZenodoRelator(); - relator.setIdentifier(domain + "/external/zenodo/" + dmp.getId().toString()); + relator.setIdentifier(environment.getProperty("zenodo.domain") + "/external/zenodo/" + dmp.getId().toString()); relator.setRelation("isIdenticalTo"); deposit.getMetadata().getRelatedIdentifiers().add(relator); } + String zenodoAffiliation = environment.getProperty("zenodo.affiliation"); List contributors1 = dmp.getUsers().stream().map(userDMP -> { ZenodoContributor contributor = new ZenodoContributor(); contributor.setName(userDMP.getUser().getName()); @@ -229,7 +242,7 @@ public class DMPToZenodoMapper { String grantReferenceHead = dmp.getGrant().getReference().split(":")[0]; if (grantReferenceHead.equals("openaire")) { String grantReferenceTail = dmp.getGrant().getReference().split(":")[3]; - DOIFunder doiFunder = doiFunders.stream() + DOIFunder doiFunder = configLoader.getDOIFunders().stream() .filter(doiFunder1 -> dmp.getGrant().getFunder().getLabel().contains(doiFunder1.getFunder()) || doiFunder1.getFunder().contains(dmp.getGrant().getFunder().getLabel())) .findFirst().orElse(null); if (doiFunder != null) { diff --git a/src/main/resources/acceptedPidTypes.txt b/src/main/resources/acceptedPidTypes.txt new file mode 100644 index 0000000..c8bfcc9 --- /dev/null +++ b/src/main/resources/acceptedPidTypes.txt @@ -0,0 +1,43 @@ +actrn +ark +bibcode +nct +drks +doi +euctr +data.europa.eu +epo_id +GRID +gsk +GeoPass +GBIF +hal +handle +isrctn +ichushi +ISNI +jprn +mag_id +NAID +NCID +oai +orcid_pending +orcid +OrgPeg +PANGAEA +PIC +epo_nr_epodoc +pdb +pmc +pmid +RNSR +ROR +RRID +UNKNOWN +VIAF +who +arXiv +info:eu-repo/dai +orcidworkid +urn +w3id \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 7a587a5..5e67098 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,3 +2,6 @@ configuration.doi_funder=DOI_Funder.json configuration.zenodo.logo=${CONFIGURATION_LOGO_ZENODO} storage.temp=${STORAGE_TMP_ZENODO} configuration.zenodo=${CONFIGURATION_ZENODO} +zenodo.community=argos +zenodo.affiliation=ARGOS +zenodo.domain=https://argos.openaire.eu/ \ No newline at end of file diff --git a/src/main/resources/datasetFieldsPid.json b/src/main/resources/datasetFieldsPid.json new file mode 100644 index 0000000..f0af605 --- /dev/null +++ b/src/main/resources/datasetFieldsPid.json @@ -0,0 +1,4 @@ +{ + "pidName": "pid", + "pidTypeName": "pidTypeField" +} \ No newline at end of file diff --git a/src/main/resources/relatedIdentifiers.txt b/src/main/resources/relatedIdentifiers.txt new file mode 100644 index 0000000..92cf8de --- /dev/null +++ b/src/main/resources/relatedIdentifiers.txt @@ -0,0 +1,32 @@ +zenodo.related_identifiers.isCitedBy +zenodo.related_identifiers.cites +zenodo.related_identifiers.isSupplementTo +zenodo.related_identifiers.isSupplementedBy +zenodo.related_identifiers.isContinuedBy +zenodo.related_identifiers.continues +zenodo.related_identifiers.isDescribedBy +zenodo.related_identifiers.describes +zenodo.related_identifiers.hasMetadata +zenodo.related_identifiers.isMetadataFor +zenodo.related_identifiers.isNewVersionOf +zenodo.related_identifiers.isPreviousVersionOf +zenodo.related_identifiers.isPartOf +zenodo.related_identifiers.hasPart +zenodo.related_identifiers.isReferencedBy +zenodo.related_identifiers.references +zenodo.related_identifiers.isDocumentedBy +zenodo.related_identifiers.documents +zenodo.related_identifiers.isCompiledBy +zenodo.related_identifiers.compiles +zenodo.related_identifiers.isVariantFormOf +zenodo.related_identifiers.isOriginalFormof +zenodo.related_identifiers.isIdenticalTo +zenodo.related_identifiers.isAlternateIdentifier +zenodo.related_identifiers.isReviewedBy +zenodo.related_identifiers.reviews +zenodo.related_identifiers.isDerivedFrom +zenodo.related_identifiers.isSourceOf +zenodo.related_identifiers.requires +zenodo.related_identifiers.isRequiredBy +zenodo.related_identifiers.isObsoletedBy +zenodo.related_identifiers.obsoletes \ No newline at end of file