From 3d0e0c84aa29fd851aef297a971a7fa5345b5d2f Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Thu, 5 Dec 2019 11:56:51 +0100 Subject: [PATCH] Mapping.Source as complex object --- .../publishing/ckan2zenodo/Transformer.java | 144 ++++++++++-------- .../ckan2zenodo/TransformerManager.java | 52 +------ .../publishing/ckan2zenodo/commons/IS.java | 70 ++++++++- .../publishing/ckan2zenodo/model/Mapping.java | 22 ++- .../model/faults/ConfigurationException.java | 29 ++++ .../model/faults/TransformationException.java | 30 ++++ .../java/org/gcube/tests/TestCommons.java | 20 ++- .../org/gcube/tests/TransformationTests.java | 29 ++-- .../java/org/gcube/tests/ZenodoTests.java | 25 +-- .../src/test/resources/ResearchObject.xml | 92 ++++++----- 10 files changed, 328 insertions(+), 185 deletions(-) create mode 100644 ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/ConfigurationException.java create mode 100644 ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/TransformationException.java diff --git a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java index 8c2c1ec..cf0ada5 100644 --- a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java +++ b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java @@ -1,6 +1,5 @@ package org.gcube.data.publishing.ckan2zenodo; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -10,6 +9,7 @@ import org.gcube.data.publishing.ckan2zenodo.commons.Parsing; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.Mapping; import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Regexp; +import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Contributor; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.AccessRights; @@ -30,22 +30,22 @@ public class Transformer { @NonNull private List mappings; - - - - public ZenodoDeposition transform(CkanItemDescriptor toTransform, ZenodoDeposition deposition) throws IOException { + + + + public ZenodoDeposition transform(CkanItemDescriptor toTransform, ZenodoDeposition deposition) throws TransformationException { log.debug("Transforming "+toTransform+". Existing Deposition is : "+deposition); - + if(deposition==null) deposition=new ZenodoDeposition(); - + // default mappings DepositionMetadata meta=deposition.getMetadata(); if(meta==null) meta=new DepositionMetadata(); - + meta.setTitle(toTransform.getTitle()); meta.setDescription(toTransform.getNotes()); - + // Access Right if(toTransform.isOpen()) meta.setAccess_right(AccessRights.open); @@ -53,18 +53,18 @@ public class Transformer { meta.setAccess_right(AccessRights.restricted); meta.setAccess_conditions("Visit the VRE "+toTransform.getVRE()+" to access it."); } - + meta.setLicense(toTransform.getLicenseId()); - + meta.setKeywords(new ArrayList(toTransform.getTags())); - + //Related Identifiers String itemUrl=toTransform.getItemUrl(); ArrayList relatedIdentifiers=new ArrayList<>(); if(itemUrl!=null) relatedIdentifiers.add(new RelatedIdentifier(itemUrl,RelatedIdentifier.Relation.compiles)); - + meta.setRelated_identifiers(relatedIdentifiers); - + //Contributors ArrayList contributors=new ArrayList<>(); String authorName=toTransform.getAuthor(); @@ -73,7 +73,7 @@ public class Transformer { author.setName(authorName); contributors.add(author); } - + String maintainerName=toTransform.getAuthor(); if(maintainerName!=null) { Contributor maintainer=new Contributor(Contributor.Type.DataCurator); @@ -81,59 +81,83 @@ public class Transformer { contributors.add(maintainer); } meta.setContributors(contributors); - + meta.setVersion(toTransform.getVersion()); - - + + // Dates - - + + deposition.setMetadata(meta); - + // profile specific mappings return applyMappings(toTransform, deposition); } - - - private ZenodoDeposition applyMappings(CkanItemDescriptor source, ZenodoDeposition target) throws IOException { - ObjectMapper mapper=Parsing.getMapper(); - DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); - DocumentContext targetCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(mapper.writeValueAsString(target)); - - for(Mapping mapping:mappings) { - List sourceValues=sourceCtx.read(mapping.getSource()); - if(sourceValues!=null) - for(String sourceValue:sourceValues) { - String resultingValue=sourceValue; - - // apply regexps - for(Regexp regexp:mapping.getRegexp()) { - switch(regexp.getType()) { - case extract : { - Pattern p=Pattern.compile(regexp.getTarget()); - Matcher m = p.matcher(resultingValue); - if(m.find()) - resultingValue=m.group(); - else resultingValue=null; - break; - } - case replace : { - resultingValue=resultingValue.replaceAll(regexp.getTarget(), regexp.getReplacement()); - break; - } - } - - + + + private ZenodoDeposition applyMappings(CkanItemDescriptor source, ZenodoDeposition target) throws TransformationException { + try{ + ObjectMapper mapper=Parsing.getMapper(); + DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); + DocumentContext targetCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(mapper.writeValueAsString(target)); + + for(Mapping mapping:mappings) { + try { + // extract source + List sourceValues=new ArrayList<>(); + switch(mapping.getSource().getType()) { + case constant : { + sourceValues.add(mapping.getSource().getValue()); + break; } - - // apply value mapping - - resultingValue =mapping.getValueMapping().getOrDefault(sourceValue, sourceValue); - targetCtx.add(mapping.getTarget(),resultingValue); + case jsonPath : { + sourceValues.addAll(sourceCtx.read(mapping.getSource().getValue())); + break; + } + } + + for(String sourceValue:sourceValues) { + String resultingValue=sourceValue; + + // apply regexps + for(Regexp regexp:mapping.getRegexp()) { + switch(regexp.getType()) { + case extract : { + Pattern p=Pattern.compile(regexp.getTarget()); + Matcher m = p.matcher(resultingValue); + if(m.find()) + resultingValue=m.group(); + else resultingValue=null; + break; + } + case replace : { + if(resultingValue!=null) { + String replacement=regexp.getReplacement()!=null?regexp.getReplacement():""; + resultingValue=resultingValue.replaceAll(regexp.getTarget(), replacement); + break; + } + } + } + + + } + // apply value mappings + resultingValue =mapping.getValueMapping().getOrDefault(sourceValue, resultingValue); + + + // apply resulting value + targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement(),resultingValue); + } + }catch(Throwable t) { + throw new TransformationException("Exception while applying "+mapping,t); } + } + + return mapper.readValue(targetCtx.jsonString(), ZenodoDeposition.class); + }catch(Throwable t) { + log.error("Unable to transform "+source+" using previous "+target,t); + throw new TransformationException("Unable to translate "+source.getName(),t); } - - return mapper.readValue(targetCtx.jsonString(), ZenodoDeposition.class); } - + } diff --git a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java index 3f57558..e5f5e1d 100644 --- a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java +++ b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java @@ -35,60 +35,12 @@ public class TransformerManager { public Transformer getByProfile(String profile) throws Exception { for(GenericResource r: IS.queryForGenericResources("Ckan-Zenodo-Mappings")){ if (r.profile().name().equals(profile)) - return new Transformer(readMappings(r)); + return new Transformer(IS.readMappings(r)); } throw new Exception("No transformer found for profile "+profile); } - private static ArrayList readMappings(GenericResource res){ - // ByteArrayInputStream input = new ByteArrayInputStream( - // res.profile()..toString().getBytes("UTF-8")); - // Document doc = builder.parse(input); - // XPath xPath = XPathFactory.newInstance().newXPath(); - // String expression = "/class/student"; - // NodeList nodeList = (NodeList) xPath.compile(expression).evaluate( - // doc, XPathConstants.NODESET); - - ArrayList toReturn=new ArrayList(); - - Element root=res.profile().body(); - NodeList mappings=root.getElementsByTagName("mapping"); - - for(int i = 0; i values=new HashMap<>(); - NodeList valueMappings=mapping.getElementsByTagName("valueMapping"); - for(int j = 0; i regularExpressions=new ArrayList<>(); - NodeList regexpDeclarations=mapping.getElementsByTagName("regexp"); - for(int j = 0; i queryForGenericResources(String secondaryType){ log.debug("Querying for Service Endpoints [secondary type : {} ]",secondaryType); - + SimpleQuery query = queryFor(GenericResource.class); query.addCondition("$resource/Profile/SecondaryType/text() eq '"+secondaryType+"'"); @@ -30,16 +38,68 @@ public class IS { public static List queryForServiceEndpoints(String category, String platformName){ log.debug("Querying for Service Endpoints [category : {} , platformName : {}]",category,platformName); - + SimpleQuery query = queryFor(ServiceEndpoint.class); query.addCondition("$resource/Profile/Category/text() eq '"+category+"'") .addCondition("$resource/Profile/Platform/Name/text() eq '"+platformName+"'"); - + DiscoveryClient client = clientFor(ServiceEndpoint.class); return client.submit(query); } - + public static ArrayList readMappings(GenericResource res) throws ConfigurationException{ + + ArrayList toReturn=new ArrayList(); + try { + + Element root=res.profile().body(); + NodeList mappings=root.getElementsByTagName("mapping"); + + for(int i = 0; i values=new HashMap<>(); + NodeList valueMappings=mapping.getElementsByTagName("valueMapping"); + for(int j = 0; j regularExpressions=new ArrayList<>(); + NodeList regexpDeclarations=mapping.getElementsByTagName("regexp"); + for(int j = 0; j valueMapping; @NonNull diff --git a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/ConfigurationException.java b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/ConfigurationException.java new file mode 100644 index 0000000..70b86d0 --- /dev/null +++ b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/ConfigurationException.java @@ -0,0 +1,29 @@ +package org.gcube.data.publishing.ckan2zenodo.model.faults; + +public class ConfigurationException extends Exception { + + public ConfigurationException() { + // TODO Auto-generated constructor stub + } + + public ConfigurationException(String arg0) { + super(arg0); + // TODO Auto-generated constructor stub + } + + public ConfigurationException(Throwable arg0) { + super(arg0); + // TODO Auto-generated constructor stub + } + + public ConfigurationException(String arg0, Throwable arg1) { + super(arg0, arg1); + // TODO Auto-generated constructor stub + } + + public ConfigurationException(String arg0, Throwable arg1, boolean arg2, boolean arg3) { + super(arg0, arg1, arg2, arg3); + // TODO Auto-generated constructor stub + } + +} diff --git a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/TransformationException.java b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/TransformationException.java new file mode 100644 index 0000000..977eb27 --- /dev/null +++ b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/faults/TransformationException.java @@ -0,0 +1,30 @@ +package org.gcube.data.publishing.ckan2zenodo.model.faults; + +public class TransformationException extends Exception { + + public TransformationException() { + // TODO Auto-generated constructor stub + } + + public TransformationException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public TransformationException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + public TransformationException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public TransformationException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + +} diff --git a/ckan2zenodo-library/src/test/java/org/gcube/tests/TestCommons.java b/ckan2zenodo-library/src/test/java/org/gcube/tests/TestCommons.java index 493d43c..eefd1d6 100644 --- a/ckan2zenodo-library/src/test/java/org/gcube/tests/TestCommons.java +++ b/ckan2zenodo-library/src/test/java/org/gcube/tests/TestCommons.java @@ -4,7 +4,11 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import org.gcube.data.publishing.ckan2zenodo.Fixer; +import org.gcube.data.publishing.ckan2zenodo.Transformer; +import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials; +import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -37,5 +41,19 @@ public class TestCommons { return mapper; } - + static final ZenodoDeposition readAndTransform(String jsonFile, Transformer transformer,ZenodoDeposition...depositions) throws Exception { + try{ + String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile)); + CkanItemDescriptor desc=new CkanItemDescriptor(json); + System.out.println("Going to transform : "+desc.getContent()); + System.out.println("Result : "); + ZenodoDeposition dep=transformer.transform(desc, depositions!=null?depositions[0]:null); + System.out.println(dep); + System.out.println("As JSON : "); + System.out.println(Fixer.fixSending(getMapper().writeValueAsString(dep))); + return dep; + }catch(Throwable t) { + throw new Exception("Errors with json file "+jsonFile,t); + } + } } diff --git a/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java b/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java index 6aa17ca..b548749 100644 --- a/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java +++ b/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java @@ -3,8 +3,11 @@ package org.gcube.tests; import java.io.IOException; import java.util.Collections; +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.Resources; import org.gcube.data.publishing.ckan2zenodo.Fixer; import org.gcube.data.publishing.ckan2zenodo.Transformer; +import org.gcube.data.publishing.ckan2zenodo.commons.IS; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; import org.junit.BeforeClass; @@ -25,33 +28,19 @@ public class TransformationTests { @Test public void transform() throws Exception { Transformer basic=new Transformer(Collections.EMPTY_LIST); - readAndTransform("/simpleItem.json",basic); - readAndTransform("/FSKXModel.json",basic); - readAndTransform("/ResearchObject.json",basic); + TestCommons.readAndTransform("/simpleItem.json",basic); + TestCommons.readAndTransform("/FSKXModel.json",basic); + TestCommons.readAndTransform("/ResearchObject.json",basic); } @Test - public void transformWithMappings() throws IOException { - - + public void transformWithMappings() throws Exception { + GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); + TestCommons.readAndTransform("/ResearchObject.json", new Transformer(IS.readMappings(res))); } - private static final void readAndTransform(String jsonFile, Transformer transformer) throws Exception { - try{ - String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile)); - CkanItemDescriptor desc=new CkanItemDescriptor(json); - System.out.println("Going to transform : "+desc.getContent()); - System.out.println("Result : "); - ZenodoDeposition dep=transformer.transform(desc, null); - System.out.println(dep); - System.out.println("As JSON : "); - System.out.println(Fixer.fixSending(mapper.writeValueAsString(dep))); - }catch(Throwable t) { - throw new Exception("Errors with json file "+jsonFile,t); - } - } } diff --git a/ckan2zenodo-library/src/test/java/org/gcube/tests/ZenodoTests.java b/ckan2zenodo-library/src/test/java/org/gcube/tests/ZenodoTests.java index 0266c91..a6522a6 100644 --- a/ckan2zenodo-library/src/test/java/org/gcube/tests/ZenodoTests.java +++ b/ckan2zenodo-library/src/test/java/org/gcube/tests/ZenodoTests.java @@ -6,10 +6,14 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Date; +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.Resources; import org.gcube.data.publishing.ckan2zenodo.Transformer; import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo; +import org.gcube.data.publishing.ckan2zenodo.commons.IS; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials; +import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Creator; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; @@ -60,14 +64,15 @@ public class ZenodoTests { } -// @Test -// public void createFromSimpleItem() throws JsonProcessingException, ZenodoException { -// Zenodo z=new Zenodo(credentials); -// String json=TestCommons.convertStreamToString(this.getClass().getResourceAsStream("/simpleItem.json")); -// Transformer defaultTransformer=new Transformer(); -// CkanItemDescriptor desc=new CkanItemDescriptor("sampleontable_in_prevre", json); -// ZenodoDeposition dep=z.createNew(); -// defaultTransformer.transform(desc, dep); -// System.out.println(z.updateMetadata(dep.getId(), dep.getMetadata())); -// } + @Test + public void createFromSimpleItem() throws ConfigurationException, Exception { + Zenodo z=new Zenodo(credentials); + + GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); + ZenodoDeposition dep=z.createNew(); + dep=TestCommons.readAndTransform("/ResearchObject.json", new Transformer(IS.readMappings(res)),dep); + + + System.out.println(z.updateMetadata(dep.getId(), dep.getMetadata())); + } } diff --git a/ckan2zenodo-library/src/test/resources/ResearchObject.xml b/ckan2zenodo-library/src/test/resources/ResearchObject.xml index 8f58471..5d1a708 100644 --- a/ckan2zenodo-library/src/test/resources/ResearchObject.xml +++ b/ckan2zenodo-library/src/test/resources/ResearchObject.xml @@ -1,38 +1,58 @@ - 4adeaca2-8e32-4507-8937-d891629998e2 - GenericResource - - - Ckan-Zenodo-Mappings - ResearchObject - Simple mappings tests - - - - $.extras[?(@.key=='Author')].value - $.metadata.contributors[0].name - - ([A-Za-z]*, [A-Za-z]*)(?=,) - - - - $.extras[?(@.key=='Author')].value - $.metadata.contributors[0].type - - .* - Producer - - - - $.extras[?(@.key=='Author')].value - $.metadata.contributors[0].orcid - - orcid.org/.* - - - - - - - + 4adeaca2-8e32-4507-8937-d891629998e2 + GenericResource + + + Ckan-Zenodo-Mappings + ResearchObject + Simple mappings tests + + + + + + dataset + + $.metadata + upload_type + + + + + + $.extras[?(@.key=='Author')].value + + $.metadata.contributors[0] + name + + ([A-Za-z]*, [A-Za-z]*)(?=,) + + + + + Producer + + $.metadata.contributors[0] + type + + + + + + + + \ No newline at end of file