From 96315e79247ebaf71b3354647fdf1314dd0c68fe Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Thu, 18 Feb 2021 12:14:29 +0100 Subject: [PATCH 1/4] updated gcat dependency --- CHANGELOG.md | 7 +++++++ pom.xml | 14 ++++---------- .../data/publishing/ckan2zenodo/Ckan2Zenodo.java | 10 ++++++++++ src/test/java/org/gcube/tests/OneHitTest.java | 2 +- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bc3e8f..151638b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). # Changelog for org.gcube.data.publishing.ckan2zenodo-library +## [v0.0.4-SNAPSHOT] 2021-02-18 +- Changed coordinartes for gcat-client + +## [v0.0.3-SNAPSHOT] 2020-06-30 +- Changed coordinartes for gcat-client + + ## [v0.0.2] 2020-06-30 ### Enhancements diff --git a/pom.xml b/pom.xml index 3bbeaad..4bcc6e5 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.gcube.data.publishing ckan2zenodo-library - 0.0.2 + 0.0.4-SNAPSHOT CKAN 2 Zenodo Library Library to publish d4science CKAN items into Zenodo @@ -31,7 +31,7 @@ org.gcube.distribution gcube-bom - 1.5.0 + 2.0.0 pom import @@ -40,15 +40,9 @@ - org.gcube.data-publishing + org.gcube.data-catalogue gcat-client - [1.2.0,2.0.0-SNAPSHOT) - - - com.fasterxml.jackson.core - jackson-core - - + [2.0.0-SNAPSHOT,3.0.0) diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2Zenodo.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2Zenodo.java index 8bd2de6..67caf97 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2Zenodo.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2Zenodo.java @@ -96,4 +96,14 @@ public interface Ckan2Zenodo { */ public ZenodoDeposition publish(ZenodoDeposition dep, CkanItemDescriptor toUpdate) throws ZenodoException, ConfigurationException, InvalidItemException, MalformedURLException; + + /** + * Checks environment configuration + * + * -gCat is present + * -Zenodo credentials are present + * + * + * + */ } diff --git a/src/test/java/org/gcube/tests/OneHitTest.java b/src/test/java/org/gcube/tests/OneHitTest.java index c43417a..29eb1b7 100644 --- a/src/test/java/org/gcube/tests/OneHitTest.java +++ b/src/test/java/org/gcube/tests/OneHitTest.java @@ -26,7 +26,7 @@ public class OneHitTest { TokenSetter.set(scope); Ckan2Zenodo client=new Ckan2ZenodoImpl(); - String toPublishItemName="f4292d0e-c94f-4542-bfa3-25f78638fc1b"; + String toPublishItemName="my_test_bc_service"; // Get the item representation CkanItemDescriptor item=client.read(toPublishItemName); -- 2.17.1 From 81048172b2a6ddbbff2595e900c46ffa8bd67539 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Mon, 8 Mar 2021 17:12:28 +0100 Subject: [PATCH 2/4] Improved mappings --- CHANGELOG.md | 4 +- out.json | 1 + pom.xml | 13 +- .../ckan2zenodo/TransformerManager.java | 2 +- .../publishing/ckan2zenodo/Translator.java | 141 +++++++++--- .../publishing/ckan2zenodo/commons/IS.java | 127 +++------- .../ckan2zenodo/commons/Parsing.java | 2 +- .../ckan2zenodo/model/parsing/Filter.java | 20 ++ .../ckan2zenodo/model/parsing/Mapping.java | 90 ++------ .../ckan2zenodo/model/parsing/Mappings.java | 26 +++ .../ckan2zenodo/model/parsing/Regexp.java | 19 ++ .../model/parsing/ResourceFilter.java | 31 ++- .../ckan2zenodo/model/parsing/Source.java | 20 ++ .../model/parsing/TargetElement.java | 24 ++ .../ckan2zenodo/model/parsing/TargetPath.java | 31 +++ .../ckan2zenodo/model/parsing/Value.java | 27 +++ src/test/java/org/gcube/tests/OneHitTest.java | 4 +- src/test/java/org/gcube/tests/Previewer.java | 34 +++ .../java/org/gcube/tests/TestCommons.java | 22 +- .../org/gcube/tests/TransformationTests.java | 45 +++- .../java/org/gcube/tests/ZenodoTests.java | 5 +- src/test/resources/ResearchObject.xml | 8 +- src/test/resources/blue_cloud_dataset.out.xml | 0 src/test/resources/blue_cloud_dataset.xml | 216 +++++++++++------- .../resources/blue_cloud_deliverable.json | 120 ++++++++++ src/test/resources/blue_cloud_deliverable.xml | 70 ++++++ src/test/resources/simple.xml | 30 +++ 27 files changed, 819 insertions(+), 313 deletions(-) create mode 100644 out.json create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Filter.java create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java create mode 100644 src/test/java/org/gcube/tests/Previewer.java delete mode 100644 src/test/resources/blue_cloud_dataset.out.xml create mode 100644 src/test/resources/blue_cloud_deliverable.json create mode 100644 src/test/resources/blue_cloud_deliverable.xml create mode 100644 src/test/resources/simple.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 151638b..009639c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for org.gcube.data.publishing.ckan2zenodo-library ## [v0.0.4-SNAPSHOT] 2021-02-18 -- Changed coordinartes for gcat-client +- Improved mapping language : target record +- Improved mapping language : value iteration + ## [v0.0.3-SNAPSHOT] 2020-06-30 - Changed coordinartes for gcat-client diff --git a/out.json b/out.json new file mode 100644 index 0000000..98b5791 --- /dev/null +++ b/out.json @@ -0,0 +1 @@ +{"metadata":{"upload_type":"other","creators":[{"name":"Rossi, Pietro, pietro.rossi@acme.org"},{"name":"Verdi, Luca, lica.verdi@amce.org"},{"name":"Bianchi, Elena"}],"contributors":[{"name":"Ferrari, Paolo, paolo.ferrari@acme.org","type":"Other"},{"name":"Russo, Pietro, pietro.russo@amce.org","type":"Other"},{"name":"Giorgino, Giorgio","type":"Other"}],"communities":[{"identifier":"blue-cloud"}],"grants":[{"id":"862409"}]},"doiurl":"https://doi.org/null"} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 4bcc6e5..7cbf6d7 100644 --- a/pom.xml +++ b/pom.xml @@ -39,12 +39,19 @@ - - org.gcube.data-catalogue + + + + + + + + org.gcube.data-publishing gcat-client - [2.0.0-SNAPSHOT,3.0.0) + [1.0.0-SNAPSHOT,2.0.0) + org.gcube.core common-encryption diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java index 487d5f3..9daeeee 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java @@ -29,7 +29,7 @@ public class TransformerManager { public Translator getByProfile(String profile) throws ConfigurationException { for(GenericResource r: IS.queryForGenericResources("Ckan-Zenodo-Mappings")){ if (r.profile().name().equals(profile)) - return new Translator(IS.readMappings(r),IS.readResourceFilters(r)); + return new Translator(IS.readMappings(r)); } throw new ConfigurationException("No specific mapping for the catalogue item has been configured. " + "By continuing with the upload some metadata might not be upload to Zenodo."); diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java index 8e703a6..13f85a3 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java @@ -5,9 +5,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,11 +17,13 @@ import org.gcube.data.publishing.ckan2zenodo.commons.Parsing; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.CkanResource; import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Filter; import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Regexp; import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter.Filter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.TargetElement; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Value; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Contributor; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Creator; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; @@ -56,6 +60,10 @@ public class Translator { this(mappings,ResourceFilter.PASS_ALL); } + public Translator(Mappings m) { + this(m.getMappings(),m.getResourceFilters()); + } + public ZenodoDeposition transform(CkanItemDescriptor toTransform, ZenodoDeposition deposition) throws TransformationException { log.debug("Transforming "+toTransform+". Existing Deposition is : "+deposition); @@ -154,24 +162,33 @@ public class Translator { DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); DocumentContext targetCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(mapper.writeValueAsString(target)); + // FOR EACH MAPPING + // IF source + // For each source value ; NB GET FIRST MATCHING Value path + // APPLY TRANSFORMATIONS + // FIND/INIT TARGET PATH (OPTS isArray, replace) + // SET TARGET ELEMENT (OPTS replace, regexp) + // IF SINGLE -> property + // iF MULITPLE -> object + // ELSE TODO for(Mapping mapping:mappings) { + log.debug("Applying {} ",mapping); try { - // extract source + // "EVALUATE SURCE VALUES" List sourceValues=new ArrayList<>(); for(Value v: mapping.getSource().getValues()) { - - String actualValue=null; + List actualValues=new ArrayList(); switch(v.getType()) { case constant : { - actualValue=v.getValue(); + actualValues.add(v.getValue()); break; } case jsonPath : { for(String s: ((Collection) sourceCtx.read(v.getValue()))){ if(s!=null) { s=s.trim(); - if(!s.isEmpty())actualValue=s; + if(!s.isEmpty())actualValues.add(s); } } @@ -180,24 +197,50 @@ public class Translator { } - // Adding to actual values - if(actualValue!=null) { + // Applygin splits + for(String foundVal:actualValues) { if(v.getSplit()!=null) - for(String toAdd:actualValue.split(v.getSplit())) + for(String toAdd:foundVal.split(v.getSplit())) sourceValues.add(toAdd.trim()); - else sourceValues.add(actualValue); + else sourceValues.add(foundVal); } + + if(!sourceValues.isEmpty()) break; } + log.debug("Found matching "+sourceValues); + // ************** INIT TARGET PATH + // CHECK INIT TARGET PATH + ArrayList> resultingValueList=new ArrayList>(); + if(!sourceValues.isEmpty()) { + List targetElementFound=targetCtx.read(mapping.getTargetPath().getValue()); + if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null || + !mapping.getTargetPath().getAppend()) { + // targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing")); + JsonPath path=JsonPath.compile(mapping.getTargetPath().getValue()); + + switch(mapping.getTargetPath().getType()) { + case array : + targetCtx=targetCtx.set(path,new Object[sourceValues.size()]); + break; + case map : + targetCtx=targetCtx.set(path,null); + break; + } + } + } + + // ************** PREPARE VALUE FOR EACH SOURCE VALUE for(String sourceValue:sourceValues) { String resultingValue=sourceValue; + log.debug("Managing "+resultingValue); // apply regexps for(Regexp regexp:mapping.getRegexp()) { @@ -221,31 +264,63 @@ public class Translator { } + // apply value mappings resultingValue =mapping.getValueMapping().getOrDefault(sourceValue, resultingValue); - // check if targetPath exists - List targetElementFound=targetCtx.read(mapping.getTargetPath()); - if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null) { - // targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing")); - targetCtx=Parsing.addElement(targetCtx, mapping.getTargetPath()); - } - + // NEW : Multiple Target Elements can be simple or structure + + Map resultingTargetElements=new HashMap(); + + for(TargetElement el:mapping.getTargetElements()) { + if(el.getConstant()!=null) resultingValue=el.getConstant(); + else + if(el.getAppend()){ + String original=((List)targetCtx.read(mapping.getTargetPath()+"."+el.getValue())).get(0); + if(original!=null && !original.isEmpty()) + resultingValue=original+resultingValue; + } + + resultingTargetElements.put(el.getValue(), resultingValue); - if(mapping.getTargetElement().getAppend()){ - String original=((List)targetCtx.read(mapping.getTargetPath()+"."+mapping.getTargetElement().getTargetElement())).get(0); - if(original!=null && !original.isEmpty()) - resultingValue=original+resultingValue; } - - - - - - targetCtx=targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement().getTargetElement(), - resultingValue); + + resultingValueList.add(resultingTargetElements); + + // close loop on values } + + JsonPath path=JsonPath.compile(mapping.getTargetPath().getValue()); + switch(mapping.getTargetPath().getType()) { + case array : + targetCtx.set(path, resultingValueList); + break; + case map : + for(Map resultingTargetElements : resultingValueList) { + log.debug("Applying "+resultingTargetElements+ " to "+mapping.getTargetPath()); + targetCtx.set(path, resultingTargetElements); + } + break; + } + + + // for(Map resultingTargetElements : resultingValueList) { + // log.debug("Applying "+resultingTargetElements+ " to "+mapping.getTargetPath()); + // // Apply object + // + // + // switch(mapping.getTargetPath().getType()) { + // case array : + // targetCtx=targetCtx.add(path, resultingTargetElements); + // break; + // case map : + // targetCtx.set(path, resultingTargetElements); + // break; + // } + // } + + }catch(Throwable t) { throw new TransformationException("Exception while applying "+mapping,t); } @@ -261,9 +336,9 @@ public class Translator { public List filterResources(CkanItemDescriptor source) throws TransformationException{ - ObjectMapper mapper=Parsing.getMapper(); - DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); - try { + ObjectMapper mapper=Parsing.getMapper(); + DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); + try { HashSet toReturn=new HashSet(); for(Filter f:resourceFilter.getFilters()) { JSONArray filtered=sourceCtx.read(f.getConditions().get(0)); diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java index f2805c9..842adae 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java @@ -3,24 +3,22 @@ package org.gcube.data.publishing.ckan2zenodo.commons; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; +import java.io.StringReader; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Unmarshaller; +import javax.xml.transform.stream.StreamSource; + import org.gcube.common.encryption.StringEncrypter; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.TargetElement; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; import lombok.extern.slf4j.Slf4j; @@ -53,69 +51,18 @@ public class IS { return client.submit(query); } - public static ArrayList readMappings(GenericResource res) throws ConfigurationException{ + public static Mappings readMappings(GenericResource res) throws ConfigurationException{ - ArrayList toReturn=new ArrayList(); try { + String body=res.profile().bodyAsString(); + + // create JAXB context and unmarshaller + Unmarshaller um=JAXBContext.newInstance(Mappings.class).createUnmarshaller(); + - Element root=res.profile().body(); - NodeList mappings=root.getElementsByTagName("mapping"); + return (Mappings) um.unmarshal(new StreamSource(new StringReader(body))); - for(int i = 0; i values=new HashMap<>(); - NodeList valueMappings=mapping.getElementsByTagName("valueMapping"); - for(int j = 0; j regularExpressions=new ArrayList<>(); - NodeList regexpDeclarations=mapping.getElementsByTagName("regexp"); - for(int j = 0; j filtersList=new ArrayList<>(); - Element root=res.profile().body(); - NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter"); - for(int i=0;i conditions=new ArrayList<>(); - NodeList conditionNodes=filter.getElementsByTagName("condition"); - for(int j=0;j filtersList=new ArrayList<>(); +// Element root=res.profile().body(); +// NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter"); +// for(int i=0;i conditions=new ArrayList<>(); +// NodeList conditionNodes=filter.getElementsByTagName("condition"); +// for(int j=0;j conditions; +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java index 775b013..0eb5df9 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java @@ -4,74 +4,34 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import lombok.ToString; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; -@RequiredArgsConstructor -@Getter -@ToString +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +@XmlAccessorType(XmlAccessType.FIELD) public class Mapping { - @RequiredArgsConstructor - @Getter - @ToString - public static class Regexp{ - public static enum Type{ - replace,extract - } - @NonNull - private Type type; - @NonNull - private String target; - @Setter - private String replacement; - } - - @Getter - @ToString - public static class Source{ - - @RequiredArgsConstructor - @Getter - @ToString - public static class Value{ - public static enum Type{ - jsonPath,constant - } - @Setter - private String split; - @NonNull - private Type type; - @NonNull - private String value; - } - - private List values=new ArrayList<>(); - } - - @Getter - @ToString - @RequiredArgsConstructor - public static class TargetElement{ - @Setter - private Boolean append = false; - @NonNull - private String targetElement; - } - - - @NonNull + @XmlElement private Source source; - @NonNull - private String targetPath; - @NonNull - private TargetElement targetElement; - @NonNull - private HashMap valueMapping; - @NonNull - private List regexp; + + @XmlElement + private TargetPath targetPath; + + + + @XmlElement(name="targetElement", required= true) + private List targetElements=new ArrayList(); + + @XmlElement + private HashMap valueMapping=new HashMap(); + + + @XmlElement + private List regexp=new ArrayList(); } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java new file mode 100644 index 0000000..1d55629 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java @@ -0,0 +1,26 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import lombok.Data; +import lombok.NoArgsConstructor; + + +@NoArgsConstructor +@Data +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class Mappings { + + + @XmlElement + private ResourceFilter resourceFilters; + + @XmlElement(name="mapping") + private List mappings; +} diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java new file mode 100644 index 0000000..1f84acc --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java @@ -0,0 +1,19 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import lombok.Data; + +@Data +@XmlAccessorType(XmlAccessType.FIELD) +public class Regexp{ + public static enum RegexpType{ + replace,extract + } + + private RegexpType type=RegexpType.extract; + + private String target; + private String replacement; +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java index 2d39e27..973c4b4 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java @@ -1,30 +1,27 @@ package org.gcube.data.publishing.ckan2zenodo.model.parsing; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.ToString; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; -@RequiredArgsConstructor -@ToString -@Getter +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) public class ResourceFilter { public static final ResourceFilter PASS_ALL=new ResourceFilter(Collections.singletonList( new Filter(Collections.singletonList("$.resources[?(@.format)]")))); - @RequiredArgsConstructor - @ToString - @Getter - public static class Filter { - @NonNull - private List conditions; - } - - @NonNull - private List filters; + @XmlElement(name="filter", required= false) + private List filters=new ArrayList(); } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java new file mode 100644 index 0000000..1ba5849 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java @@ -0,0 +1,20 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class Source{ + + @XmlElement(name="value", required= true) + private List values=new ArrayList<>(); +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java new file mode 100644 index 0000000..9ee2435 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java @@ -0,0 +1,24 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class TargetElement{ + + @XmlAttribute(required=false) + private Boolean append = false; + + @XmlAttribute(required=false) + private String constant= null; + + @XmlValue + private String Value; +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java new file mode 100644 index 0000000..077f81d --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java @@ -0,0 +1,31 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class TargetPath { + + + public static enum ElementType { + map,array + } + + + + @XmlAttribute(required=false) + private ElementType type=ElementType.map; + + @XmlAttribute(required=false) + private Boolean append=false; + + @XmlValue + private String value; +} diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java new file mode 100644 index 0000000..7aeee45 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java @@ -0,0 +1,27 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class Value{ + public static enum ValueType{ + jsonPath,constant + } + + @XmlAttribute(required=false) + private String split; + + @XmlAttribute(required=false) + private ValueType type=ValueType.constant; + + @XmlValue + private String value; +} \ No newline at end of file diff --git a/src/test/java/org/gcube/tests/OneHitTest.java b/src/test/java/org/gcube/tests/OneHitTest.java index 29eb1b7..47c4ab8 100644 --- a/src/test/java/org/gcube/tests/OneHitTest.java +++ b/src/test/java/org/gcube/tests/OneHitTest.java @@ -26,7 +26,7 @@ public class OneHitTest { TokenSetter.set(scope); Ckan2Zenodo client=new Ckan2ZenodoImpl(); - String toPublishItemName="my_test_bc_service"; + String toPublishItemName="a_sample_deliverable_4_zenodo"; // Get the item representation CkanItemDescriptor item=client.read(toPublishItemName); @@ -55,7 +55,7 @@ public class OneHitTest { System.out.println("Publishing.. "); //Finalize - System.out.println("DONE : "+client.publish(preview, item)); +// System.out.println("DONE : "+client.publish(preview, item)); } diff --git a/src/test/java/org/gcube/tests/Previewer.java b/src/test/java/org/gcube/tests/Previewer.java new file mode 100644 index 0000000..87c2908 --- /dev/null +++ b/src/test/java/org/gcube/tests/Previewer.java @@ -0,0 +1,34 @@ +package org.gcube.tests; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.Resources; +import org.gcube.data.publishing.ckan2zenodo.Translator; +import org.gcube.data.publishing.ckan2zenodo.commons.IS; +import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; +import org.junit.Test; + +public class Previewer { + + + @Test + public void preview() throws Exception { + String sourceItem="/blue_cloud_deliverable.json"; + String genResource="/blue_cloud_deliverable.xml"; + + GenericResource resource=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream(genResource)); + Mappings m=IS.readMappings(resource); + + try(FileOutputStream fos = new FileOutputStream("out.json")){ + TestCommons.getMapper().writeValue(fos, TestCommons.readAndTransform(sourceItem, new Translator(m))); + }; + + + } + + +} diff --git a/src/test/java/org/gcube/tests/TestCommons.java b/src/test/java/org/gcube/tests/TestCommons.java index 74f6b25..065bedf 100644 --- a/src/test/java/org/gcube/tests/TestCommons.java +++ b/src/test/java/org/gcube/tests/TestCommons.java @@ -1,6 +1,7 @@ package org.gcube.tests; import java.io.IOException; +import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; @@ -33,6 +34,7 @@ public class TestCommons { static Map getDefinedMappings(){ HashMap toReturn=new HashMap(); toReturn.put("/blue_cloud_dataset.json", "/blue_cloud_dataset.xml"); + toReturn.put("/blue_cloud_deliverable.json", "/blue_cloud_deliverable.xml"); return toReturn; } @@ -51,19 +53,23 @@ public class TestCommons { } static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,ZenodoDeposition...depositions) throws Exception { + return readAndTransform(jsonFile, transformer, System.out, depositions); + } + + static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,PrintStream os, ZenodoDeposition...depositions) throws Exception { try{ String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile)); CkanItemDescriptor desc=new CkanItemDescriptor(json); - System.out.println("Going to transform : "+desc.getContent()); - System.out.println("Result : "); + os.println("Going to transform : "+desc.getContent()); + os.println("Result : "); ZenodoDeposition dep=transformer.transform(desc, (depositions!=null&&depositions.length>0)?depositions[0]:null); - System.out.println(dep); - System.out.println("As JSON : "); - System.out.println(Fixer.fixSending(getMapper().writeValueAsString(dep))); - System.out.println("Filtered resources : "); + os.println(dep); + os.println("As JSON : "); + os.println(Fixer.fixSending(getMapper().writeValueAsString(dep))); + os.println("Filtered resources : "); for(CkanResource res : transformer.filterResources(desc)) { - System.out.println("AS JSON : "); - System.out.println(Fixer.fixSending(getMapper().writeValueAsString(res))); + os.println("AS JSON : "); + os.println(Fixer.fixSending(getMapper().writeValueAsString(res))); } return dep; }catch(Throwable t) { diff --git a/src/test/java/org/gcube/tests/TransformationTests.java b/src/test/java/org/gcube/tests/TransformationTests.java index a2c9dd2..271bd7b 100644 --- a/src/test/java/org/gcube/tests/TransformationTests.java +++ b/src/test/java/org/gcube/tests/TransformationTests.java @@ -1,5 +1,9 @@ package org.gcube.tests; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -9,6 +13,11 @@ import org.gcube.data.publishing.ckan2zenodo.Translator; import org.gcube.data.publishing.ckan2zenodo.commons.IS; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.CkanResource; +import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Filter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Regexp; import org.junit.BeforeClass; import org.junit.Test; @@ -28,6 +37,39 @@ public class TransformationTests { } + @Test + public void testUnmarshalling() throws ConfigurationException { + List toCheck=new ArrayList(); + toCheck.add("/simple.xml"); + toCheck.addAll(mappings.values()); + for(String resFile:toCheck) { + System.out.println("Checking "+resFile); + GenericResource resource=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream(resFile)); + Mappings m=IS.readMappings(resource); + assertTrue(m.getResourceFilters()!=null); + assertTrue(m.getResourceFilters().getFilters()!=null); + m.getResourceFilters().getFilters().forEach((Filter f)->{ + assertTrue(f.getConditions()!=null&&!f.getConditions().isEmpty());}); + assertTrue(m.getMappings()!=null); + m.getMappings().forEach((Mapping map)->{ +// assertTrue(map.getSource()!=); + }); + new Translator(m); + } + } + @Test + public void testMarhalling() { + + +// Mappings m=new Mappings(); +// Mapping m1=new Mapping(); +// m1. +// +// m.getMappings().add(new Mapping()); +// + + } + @Test public void transform() throws Exception { @@ -35,6 +77,7 @@ public class TransformationTests { TestCommons.readAndTransform("/simpleItem.json",basic); TestCommons.readAndTransform("/FSKXModel.json",basic); TestCommons.readAndTransform("/ResearchObject.json",basic); + } @@ -60,7 +103,7 @@ public class TransformationTests { @Test public void filterResources() throws Exception { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); - Translator t=new Translator(IS.readMappings(res),IS.readResourceFilters(res)); + Translator t=new Translator(IS.readMappings(res)); String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream("/ResearchObject.json")); CkanItemDescriptor desc=new CkanItemDescriptor(json); diff --git a/src/test/java/org/gcube/tests/ZenodoTests.java b/src/test/java/org/gcube/tests/ZenodoTests.java index 7bbeb22..b822051 100644 --- a/src/test/java/org/gcube/tests/ZenodoTests.java +++ b/src/test/java/org/gcube/tests/ZenodoTests.java @@ -79,7 +79,7 @@ public class ZenodoTests { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); ZenodoDeposition dep=z.createNew(); - Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res)); + Translator tran=new Translator(IS.readMappings(res)); dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep); dep=z.updateMetadata(dep); @@ -107,7 +107,7 @@ public class ZenodoTests { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); ZenodoDeposition dep=z.createNew(); - Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res)); + Translator tran=new Translator(IS.readMappings(res)); dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep); dep=z.updateMetadata(dep); @@ -159,6 +159,7 @@ public class ZenodoTests { ZenodoDeposition dep=TestCommons.readAndTransform(entry.getKey(), translator,z.createNew()); dep=z.updateMetadata(dep); System.out.println("Created "+dep); + //No payload to publish // System.out.println("Published "+z.publish(dep)); } } diff --git a/src/test/resources/ResearchObject.xml b/src/test/resources/ResearchObject.xml index c9de4c4..986abba 100644 --- a/src/test/resources/ResearchObject.xml +++ b/src/test/resources/ResearchObject.xml @@ -7,14 +7,16 @@ ResearchObject Simple mappings tests - + + + + + $.resources[?(@.format)] - - diff --git a/src/test/resources/blue_cloud_dataset.out.xml b/src/test/resources/blue_cloud_dataset.out.xml deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/resources/blue_cloud_dataset.xml b/src/test/resources/blue_cloud_dataset.xml index 204ad14..a205bc2 100644 --- a/src/test/resources/blue_cloud_dataset.xml +++ b/src/test/resources/blue_cloud_dataset.xml @@ -7,32 +7,33 @@ Dataset Simple mappings tests - - - $.resources[?(@.format)] - - - - + + + - + + + $.resources[?(@.format)] + + - dataset - + dataset + $.metadata upload_type - - + + - $.extras[?(@.key=='Identity:External Identifier')].value + $.extras[?(@.key=='Identity:External + Identifier')].value $.metadata.relatedIdentifier[1] identifier - + isAlternateIdentifier @@ -40,19 +41,20 @@ $.metadata.relatedIdentifier[1] type - - - - - + + + + + - $.extras[?(@.key=='Identity:RelatedPaper')].value + $.extras[?(@.key=='Identity:RelatedPaper')].value + $.metadata.relatedIdentifier[2] identifier - + isCitedBy @@ -60,21 +62,23 @@ $.metadata.relatedIdentifier[2] type - - - + + + - $.extras[?(@.key=='Coverage:Semantic Coverage')].value + $.extras[?(@.key=='Coverage:Semantic + Coverage')].value $.metadata - keywords[*] + keywords[*] - - + + - $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.metadata.dates[0] start @@ -84,11 +88,12 @@ [/ ] - + - $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.metadata.dates[0] end @@ -98,27 +103,29 @@ [/ ] - + Valid $.metadata.dates[0] - type + type - - + + - - + + - $.extras[?(@.key=='AccessMode:Accessibility')].value + $.extras[?(@.key=='AccessMode:Accessibility')].value + $.metadata - access_conditions + access_conditions + ^ AccessMode.Accessibility : @@ -128,14 +135,16 @@ ; - + - $.extras[?(@.key=='AccessMode:Availability')].value + $.extras[?(@.key=='AccessMode:Availability')].value + $.metadata - access_conditions + access_conditions + ^ AccessMode.Availability : @@ -145,14 +154,16 @@ ; - + - $.extras[?(@.key=='AccessMode:AccessibilityMode')].value + $.extras[?(@.key=='AccessMode:AccessibilityMode')].value + $.metadata - access_conditions + access_conditions + ^ AccessMode.AccessibilityMode : @@ -167,7 +178,8 @@ - $.extras[?(@.key=='TechnicalDetails:ProcessingDegree')].value + $.extras[?(@.key=='TechnicalDetails:ProcessingDegree')].value + $.metadata notes @@ -184,7 +196,8 @@ - $.extras[?(@.key=='TechnicalDetails:ManifestationType')].value + $.extras[?(@.key=='TechnicalDetails:ManifestationType')].value + $.metadata notes @@ -201,16 +214,18 @@ - $.extras[?(@.key=='TechnicalDetails:Language')].value + $.extras[?(@.key=='TechnicalDetails:Language')].value + $.metadata language - + - $.extras[?(@.key=='TechnicalDetails:Size')].value + $.extras[?(@.key=='TechnicalDetails:Size')].value + $.metadata notes @@ -223,11 +238,12 @@ ; - + - $.extras[?(@.key=='TechnicalDetails:DiskSize')].value + $.extras[?(@.key=='TechnicalDetails:DiskSize')].value + $.metadata notes @@ -240,11 +256,12 @@ ; - + - $.extras[?(@.key=='TechnicalDetails:Format')].value + $.extras[?(@.key=='TechnicalDetails:Format')].value + $.metadata notes @@ -257,11 +274,12 @@ ; - + - $.extras[?(@.key=='TechnicalDetails:FormatSchema')].value + $.extras[?(@.key=='TechnicalDetails:FormatSchema')].value + $.metadata notes @@ -279,7 +297,8 @@ - $.extras[?(@.key=='DataProtection:PersonalData')].value + $.extras[?(@.key=='DataProtection:PersonalData')].value + $.metadata notes @@ -296,7 +315,8 @@ - $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.metadata notes @@ -314,13 +334,15 @@ - $.extras[?(@.key=='DataProtection:Consent of the data subject')].value + $.extras[?(@.key=='DataProtection:Consent of the + data subject')].value $.metadata notes ^ - DataProtection.Consent of the data subject : + DataProtection.Consent of the data subject : + $ @@ -328,33 +350,41 @@ - + - $.extras[?(@.key=='DataProtection:Consent obtained also covers the envisaged transfer of the personal data outside the EU')].value + $.extras[?(@.key=='DataProtection:Consent + obtained also covers the envisaged transfer of the personal data + outside the EU')].value $.metadata notes ^ - DataProtection.Consent obtained also covers the envisaged transfer of the personal data outside the EU : + DataProtection.Consent obtained also covers the + envisaged transfer of the personal data outside the EU : + $ ; - - + + - $.extras[?(@.key=='DataProtection:Personal data was manifestly made public by the data subject')].value + $.extras[?(@.key=='DataProtection:Personal data + was manifestly made public by the data subject')].value $.metadata notes ^ - DataProtection.Personal data was manifestly made public by the data subject : + DataProtection.Personal data was manifestly made + public by the data subject : $ @@ -365,13 +395,15 @@ - $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.metadata notes ^ - DataProtection.DataProtectionDirective : + DataProtection.DataProtectionDirective : + $ @@ -383,7 +415,8 @@ - $.extras[?(@.key=='Rights:IP/Copyrights')].value + $.extras[?(@.key=='Rights:IP/Copyrights')].value + $.metadata notes @@ -396,11 +429,12 @@ ; - + - $.extras[?(@.key=='Rights:Field/Scope of use')].value + $.extras[?(@.key=='Rights:Field/Scope of + use')].value $.metadata notes @@ -413,12 +447,13 @@ ; - - + + - $.extras[?(@.key=='Rights:Basic rights')].value + $.extras[?(@.key=='Rights:Basic rights')].value + $.metadata notes @@ -436,7 +471,8 @@ - $.extras[?(@.key=='Rights:Restrictions on use')].value + $.extras[?(@.key=='Rights:Restrictions on + use')].value $.metadata notes @@ -450,11 +486,12 @@ - + - $.extras[?(@.key=='Rights:Sublicense rights')].value + $.extras[?(@.key=='Rights:Sublicense + rights')].value $.metadata notes @@ -471,7 +508,8 @@ - $.extras[?(@.key=='Rights:Territory of use')].value + $.extras[?(@.key=='Rights:Territory of + use')].value $.metadata notes @@ -488,7 +526,8 @@ - $.extras[?(@.key=='Rights:License term')].value + $.extras[?(@.key=='Rights:License term')].value + $.metadata notes @@ -501,18 +540,20 @@ ; - + - $.extras[?(@.key=='Rights:Requirement of non-disclosure (confidentiality mark)')].value + $.extras[?(@.key=='Rights:Requirement of + non-disclosure (confidentiality mark)')].value $.metadata notes ^ - Rights.Requirement of non-disclosure (confidentiality mark) : + Rights.Requirement of non-disclosure (confidentiality + mark) : $ @@ -524,7 +565,8 @@ - $.extras[?(@.key=='Attribution:Attribution requirements')].value + $.extras[?(@.key=='Attribution:Attribution + requirements')].value $.metadata notes @@ -542,7 +584,8 @@ - $.extras[?(@.key=='Attribution:Attribution requirements')].value + $.extras[?(@.key=='Attribution:Attribution + requirements')].value $.metadata notes @@ -560,7 +603,8 @@ - $.extras[?(@.key=='Attribution:Distribution requirements')].value + $.extras[?(@.key=='Attribution:Distribution + requirements')].value $.metadata notes diff --git a/src/test/resources/blue_cloud_deliverable.json b/src/test/resources/blue_cloud_deliverable.json new file mode 100644 index 0000000..e6f0e56 --- /dev/null +++ b/src/test/resources/blue_cloud_deliverable.json @@ -0,0 +1,120 @@ +{ + "rating": 0.0, + "license_title": "Creative Commons Attribution Share-Alike 4.0", + "maintainer": "Candela Leonardo", + "relationships_as_object": [], + "private": false, + "maintainer_email": "leonardo.candela@isti.cnr.it", + "num_tags": 1, + "id": "ae46038a-10e9-4e62-a6c6-758e71a3428f", + "metadata_created": "2021-02-25T09:55:42.319211", + "metadata_modified": "2021-02-25T15:54:50.597047", + "author": "Candela Leonardo", + "author_email": "leonardo.candela@isti.cnr.it", + "acquire_url": "", + "state": "active", + "version": "1", + "creator_user_id": "be67d795-9fc8-4a8d-a62d-3113b724f83c", + "type": "dataset", + "resources": [ + { + "mimetype": null, + "cache_url": null, + "hash": "", + "description": "", + "name": "Deliverable", + "format": "PDF", + "url": "https://data-pre.d4science.net/RgA7", + "datastore_active": false, + "cache_last_updated": null, + "package_id": "ae46038a-10e9-4e62-a6c6-758e71a3428f", + "created": "2021-02-25T09:57:53.477077", + "state": "active", + "mimetype_inner": null, + "last_modified": null, + "position": 0, + "revision_id": "f8875fc2-e7c4-4949-a122-343a646f05e4", + "url_type": null, + "id": "e2363d7f-3c67-40bc-b6bb-9111bfcf063b", + "resource_type": null, + "size": null + } + ], + "num_resources": 1, + "tags": [ + { + "vocabulary_id": null, + "state": "active", + "display_name": "Text mining", + "id": "331ceb06-9ec3-4a10-a46a-5124f4dc19f2", + "name": "Text mining" + } + ], + "groups": [], + "license_id": "CC-BY-SA-4.0", + "relationships_as_subject": [], + "organization": { + "description": "The preVRE organisation", + "created": "2019-05-02T12:41:26.149836", + "title": "preVRE", + "name": "prevre", + "is_organization": true, + "state": "active", + "image_url": "https://pre.d4science.org/image/layout_set_logo?img_id=12949941", + "revision_id": "a44aebb5-5f88-4d53-94d2-bfb17b8d340b", + "type": "organization", + "id": "46ce33da-dbc1-4116-b41d-eb961435a53d", + "approval_status": "approved" + }, + "name": "a_sample_deliverable_4_zenodo", + "isopen": true, + "url": "", + "notes": "This is a sample deliverable created for testing the publish2Zenodo \r\nfacility", + "owner_org": "46ce33da-dbc1-4116-b41d-eb961435a53d", + "extras": [ + { + "key": "Deliverable Author", + "value": "Rossi, Pietro, pietro.rossi@acme.org" + }, + { + "key": "Deliverable Author", + "value": "Verdi, Luca, lica.verdi@amce.org" + }, + { + "key": "Deliverable Author", + "value": "Bianchi, Elena" + }, + { + "key": "Deliverable Contributor", + "value": "Ferrari, Paolo, paolo.ferrari@acme.org" + }, + { + "key": "Deliverable Contributor", + "value": "Russo, Pietro, pietro.russo@amce.org" + }, + { + "key": "Deliverable Contributor", + "value": "Giorgino, Giorgio" + }, + { + "key": "Deliverable ID", + "value": "Dx.y" + }, + { + "key": "Item URL", + "value": "https://data-pre.d4science.org/ctlg/preVRE/a_sample_deliverable_4_zenodo" + }, + { + "key": "Publication Date", + "value": "2021-02-25 " + }, + { + "key": "system:type", + "value": "Deliverable" + } + ], + "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", + "ratings_count": 0, + "title": "A sample deliverable 4 Zenodo", + "revision_id": "cbd34661-8303-4e7b-8935-beb111ad1925" +} \ No newline at end of file diff --git a/src/test/resources/blue_cloud_deliverable.xml b/src/test/resources/blue_cloud_deliverable.xml new file mode 100644 index 0000000..9d08279 --- /dev/null +++ b/src/test/resources/blue_cloud_deliverable.xml @@ -0,0 +1,70 @@ + + 12345 + GenericResource + + + Ckan-Zenodo-Mappings + Deliverable + Simple mappings tests + + + + + + + $.resources[?(@.format)] + + + + + + other + + $.metadata + upload_type + + + + + $.extras[?(@.key=='Deliverable Author')].value + + $.metadata.creators + name + + + + + + + + $.extras[?(@.key=='Deliverable Contributor')].value + + $.metadata.contributors + name + type + + + + + + + 862409 + + $.metadata.grants + id + + + + + blue-cloud + + $.metadata.communities + identifier + + + + + + + + \ No newline at end of file diff --git a/src/test/resources/simple.xml b/src/test/resources/simple.xml new file mode 100644 index 0000000..9573cc3 --- /dev/null +++ b/src/test/resources/simple.xml @@ -0,0 +1,30 @@ + + 12345 + GenericResource + + + Ckan-Zenodo-Mappings + Deliverable + Simple mappings tests + + + + + $.resources[?(@.format)] + + + + + + other + + $.metadata + upload_type + + + + + + + + \ No newline at end of file -- 2.17.1 From d3b599033acd7d12b192b01b15ed1430a6cc3726 Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Mon, 8 Mar 2021 17:39:24 +0100 Subject: [PATCH 3/4] Testing and fixes --- out.json | 2 +- .../publishing/ckan2zenodo/Translator.java | 7 ++- .../ckan2zenodo/model/parsing/TargetPath.java | 2 +- src/test/java/org/gcube/tests/OneHitTest.java | 2 +- .../resources/blue_cloud_deliverable.out.xml | 0 src/test/resources/blue_cloud_deliverable.xml | 58 +++++++++---------- 6 files changed, 37 insertions(+), 34 deletions(-) create mode 100644 src/test/resources/blue_cloud_deliverable.out.xml diff --git a/out.json b/out.json index 98b5791..27c5b6c 100644 --- a/out.json +++ b/out.json @@ -1 +1 @@ -{"metadata":{"upload_type":"other","creators":[{"name":"Rossi, Pietro, pietro.rossi@acme.org"},{"name":"Verdi, Luca, lica.verdi@amce.org"},{"name":"Bianchi, Elena"}],"contributors":[{"name":"Ferrari, Paolo, paolo.ferrari@acme.org","type":"Other"},{"name":"Russo, Pietro, pietro.russo@amce.org","type":"Other"},{"name":"Giorgino, Giorgio","type":"Other"}],"communities":[{"identifier":"blue-cloud"}],"grants":[{"id":"862409"}]},"doiurl":"https://doi.org/null"} \ No newline at end of file +{"metadata":{"upload_type":"other","publication_date":"2021-03-08T16:30:21.000774+0000","title":"A sample deliverable 4 Zenodo","creators":[{"name":"Candela Leonardo"},{"name":"Candela Leonardo"}],"description":"This is a sample deliverable created for testing the publish2Zenodo \r\nfacility","access_right":"open","license":"CC-BY-SA-4.0","keywords":["Text mining"],"related_identifiers":[{"identifier":"https://data-pre.d4science.org/ctlg/preVRE/a_sample_deliverable_4_zenodo","relation":"isCompiledBy"}],"contributors":[{"name":"Candela Leonardo","type":"Producer"},{"name":"Candela Leonardo","type":"DataCurator"},{"name":"D4Science","type":"HostingInstitution"}],"version":"1"},"doiurl":"https://doi.org/null"} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java index 13f85a3..9b503c2 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java @@ -294,12 +294,15 @@ public class Translator { JsonPath path=JsonPath.compile(mapping.getTargetPath().getValue()); switch(mapping.getTargetPath().getType()) { case array : - targetCtx.set(path, resultingValueList); + targetCtx=targetCtx.set(path, resultingValueList); break; case map : + for(Map resultingTargetElements : resultingValueList) { log.debug("Applying "+resultingTargetElements+ " to "+mapping.getTargetPath()); - targetCtx.set(path, resultingTargetElements); + for(Entry e:resultingTargetElements.entrySet()) + targetCtx=targetCtx.put(path, e.getKey(),e.getValue()); + } break; } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java index 077f81d..10eeab7 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java @@ -24,7 +24,7 @@ public class TargetPath { private ElementType type=ElementType.map; @XmlAttribute(required=false) - private Boolean append=false; + private Boolean append=true; @XmlValue private String value; diff --git a/src/test/java/org/gcube/tests/OneHitTest.java b/src/test/java/org/gcube/tests/OneHitTest.java index 47c4ab8..e3e04f5 100644 --- a/src/test/java/org/gcube/tests/OneHitTest.java +++ b/src/test/java/org/gcube/tests/OneHitTest.java @@ -55,7 +55,7 @@ public class OneHitTest { System.out.println("Publishing.. "); //Finalize -// System.out.println("DONE : "+client.publish(preview, item)); + System.out.println("DONE : "+client.publish(preview, item)); } diff --git a/src/test/resources/blue_cloud_deliverable.out.xml b/src/test/resources/blue_cloud_deliverable.out.xml new file mode 100644 index 0000000..e69de29 diff --git a/src/test/resources/blue_cloud_deliverable.xml b/src/test/resources/blue_cloud_deliverable.xml index 9d08279..cd33b0b 100644 --- a/src/test/resources/blue_cloud_deliverable.xml +++ b/src/test/resources/blue_cloud_deliverable.xml @@ -24,43 +24,43 @@ upload_type - - - $.extras[?(@.key=='Deliverable Author')].value - - $.metadata.creators - name - + + + + + + + - - - $.extras[?(@.key=='Deliverable Contributor')].value - - $.metadata.contributors - name - type - + + + + + + + + - - - 862409 - - $.metadata.grants - id - + + + + + + + - - - blue-cloud - - $.metadata.communities - identifier - + + + + + + + -- 2.17.1 From 0aa481aa3914e481b68606ee65c1d5df8402ca9c Mon Sep 17 00:00:00 2001 From: Fabio Sinibaldi Date: Mon, 8 Mar 2021 17:39:56 +0100 Subject: [PATCH 4/4] release --- CHANGELOG.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 009639c..993c568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). # Changelog for org.gcube.data.publishing.ckan2zenodo-library -## [v0.0.4-SNAPSHOT] 2021-02-18 +## [v0.0.4] 2021-02-18 - Improved mapping language : target record - Improved mapping language : value iteration diff --git a/pom.xml b/pom.xml index 7cbf6d7..88309f5 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.gcube.data.publishing ckan2zenodo-library - 0.0.4-SNAPSHOT + 0.0.4 CKAN 2 Zenodo Library Library to publish d4science CKAN items into Zenodo -- 2.17.1