diff --git a/CHANGELOG.md b/CHANGELOG.md index 151638b..009639c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm # Changelog for org.gcube.data.publishing.ckan2zenodo-library ## [v0.0.4-SNAPSHOT] 2021-02-18 -- Changed coordinartes for gcat-client +- Improved mapping language : target record +- Improved mapping language : value iteration + ## [v0.0.3-SNAPSHOT] 2020-06-30 - Changed coordinartes for gcat-client diff --git a/out.json b/out.json new file mode 100644 index 0000000..98b5791 --- /dev/null +++ b/out.json @@ -0,0 +1 @@ +{"metadata":{"upload_type":"other","creators":[{"name":"Rossi, Pietro, pietro.rossi@acme.org"},{"name":"Verdi, Luca, lica.verdi@amce.org"},{"name":"Bianchi, Elena"}],"contributors":[{"name":"Ferrari, Paolo, paolo.ferrari@acme.org","type":"Other"},{"name":"Russo, Pietro, pietro.russo@amce.org","type":"Other"},{"name":"Giorgino, Giorgio","type":"Other"}],"communities":[{"identifier":"blue-cloud"}],"grants":[{"id":"862409"}]},"doiurl":"https://doi.org/null"} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 4bcc6e5..7cbf6d7 100644 --- a/pom.xml +++ b/pom.xml @@ -39,12 +39,19 @@ - - org.gcube.data-catalogue + + + + + + + + org.gcube.data-publishing gcat-client - [2.0.0-SNAPSHOT,3.0.0) + [1.0.0-SNAPSHOT,2.0.0) + org.gcube.core common-encryption diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java index 487d5f3..9daeeee 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java @@ -29,7 +29,7 @@ public class TransformerManager { public Translator getByProfile(String profile) throws ConfigurationException { for(GenericResource r: IS.queryForGenericResources("Ckan-Zenodo-Mappings")){ if (r.profile().name().equals(profile)) - return new Translator(IS.readMappings(r),IS.readResourceFilters(r)); + return new Translator(IS.readMappings(r)); } throw new ConfigurationException("No specific mapping for the catalogue item has been configured. " + "By continuing with the upload some metadata might not be upload to Zenodo."); diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java index 8e703a6..13f85a3 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java @@ -5,9 +5,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,11 +17,13 @@ import org.gcube.data.publishing.ckan2zenodo.commons.Parsing; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.CkanResource; import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Filter; import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Regexp; import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter.Filter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.TargetElement; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Value; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Contributor; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Creator; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; @@ -56,6 +60,10 @@ public class Translator { this(mappings,ResourceFilter.PASS_ALL); } + public Translator(Mappings m) { + this(m.getMappings(),m.getResourceFilters()); + } + public ZenodoDeposition transform(CkanItemDescriptor toTransform, ZenodoDeposition deposition) throws TransformationException { log.debug("Transforming "+toTransform+". Existing Deposition is : "+deposition); @@ -154,24 +162,33 @@ public class Translator { DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); DocumentContext targetCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(mapper.writeValueAsString(target)); + // FOR EACH MAPPING + // IF source + // For each source value ; NB GET FIRST MATCHING Value path + // APPLY TRANSFORMATIONS + // FIND/INIT TARGET PATH (OPTS isArray, replace) + // SET TARGET ELEMENT (OPTS replace, regexp) + // IF SINGLE -> property + // iF MULITPLE -> object + // ELSE TODO for(Mapping mapping:mappings) { + log.debug("Applying {} ",mapping); try { - // extract source + // "EVALUATE SURCE VALUES" List sourceValues=new ArrayList<>(); for(Value v: mapping.getSource().getValues()) { - - String actualValue=null; + List actualValues=new ArrayList(); switch(v.getType()) { case constant : { - actualValue=v.getValue(); + actualValues.add(v.getValue()); break; } case jsonPath : { for(String s: ((Collection) sourceCtx.read(v.getValue()))){ if(s!=null) { s=s.trim(); - if(!s.isEmpty())actualValue=s; + if(!s.isEmpty())actualValues.add(s); } } @@ -180,24 +197,50 @@ public class Translator { } - // Adding to actual values - if(actualValue!=null) { + // Applygin splits + for(String foundVal:actualValues) { if(v.getSplit()!=null) - for(String toAdd:actualValue.split(v.getSplit())) + for(String toAdd:foundVal.split(v.getSplit())) sourceValues.add(toAdd.trim()); - else sourceValues.add(actualValue); + else sourceValues.add(foundVal); } + + if(!sourceValues.isEmpty()) break; } + log.debug("Found matching "+sourceValues); + // ************** INIT TARGET PATH + // CHECK INIT TARGET PATH + ArrayList> resultingValueList=new ArrayList>(); + if(!sourceValues.isEmpty()) { + List targetElementFound=targetCtx.read(mapping.getTargetPath().getValue()); + if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null || + !mapping.getTargetPath().getAppend()) { + // targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing")); + JsonPath path=JsonPath.compile(mapping.getTargetPath().getValue()); + + switch(mapping.getTargetPath().getType()) { + case array : + targetCtx=targetCtx.set(path,new Object[sourceValues.size()]); + break; + case map : + targetCtx=targetCtx.set(path,null); + break; + } + } + } + + // ************** PREPARE VALUE FOR EACH SOURCE VALUE for(String sourceValue:sourceValues) { String resultingValue=sourceValue; + log.debug("Managing "+resultingValue); // apply regexps for(Regexp regexp:mapping.getRegexp()) { @@ -221,31 +264,63 @@ public class Translator { } + // apply value mappings resultingValue =mapping.getValueMapping().getOrDefault(sourceValue, resultingValue); - // check if targetPath exists - List targetElementFound=targetCtx.read(mapping.getTargetPath()); - if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null) { - // targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing")); - targetCtx=Parsing.addElement(targetCtx, mapping.getTargetPath()); - } - + // NEW : Multiple Target Elements can be simple or structure + + Map resultingTargetElements=new HashMap(); + + for(TargetElement el:mapping.getTargetElements()) { + if(el.getConstant()!=null) resultingValue=el.getConstant(); + else + if(el.getAppend()){ + String original=((List)targetCtx.read(mapping.getTargetPath()+"."+el.getValue())).get(0); + if(original!=null && !original.isEmpty()) + resultingValue=original+resultingValue; + } + + resultingTargetElements.put(el.getValue(), resultingValue); - if(mapping.getTargetElement().getAppend()){ - String original=((List)targetCtx.read(mapping.getTargetPath()+"."+mapping.getTargetElement().getTargetElement())).get(0); - if(original!=null && !original.isEmpty()) - resultingValue=original+resultingValue; } - - - - - - targetCtx=targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement().getTargetElement(), - resultingValue); + + resultingValueList.add(resultingTargetElements); + + // close loop on values } + + JsonPath path=JsonPath.compile(mapping.getTargetPath().getValue()); + switch(mapping.getTargetPath().getType()) { + case array : + targetCtx.set(path, resultingValueList); + break; + case map : + for(Map resultingTargetElements : resultingValueList) { + log.debug("Applying "+resultingTargetElements+ " to "+mapping.getTargetPath()); + targetCtx.set(path, resultingTargetElements); + } + break; + } + + + // for(Map resultingTargetElements : resultingValueList) { + // log.debug("Applying "+resultingTargetElements+ " to "+mapping.getTargetPath()); + // // Apply object + // + // + // switch(mapping.getTargetPath().getType()) { + // case array : + // targetCtx=targetCtx.add(path, resultingTargetElements); + // break; + // case map : + // targetCtx.set(path, resultingTargetElements); + // break; + // } + // } + + }catch(Throwable t) { throw new TransformationException("Exception while applying "+mapping,t); } @@ -261,9 +336,9 @@ public class Translator { public List filterResources(CkanItemDescriptor source) throws TransformationException{ - ObjectMapper mapper=Parsing.getMapper(); - DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); - try { + ObjectMapper mapper=Parsing.getMapper(); + DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); + try { HashSet toReturn=new HashSet(); for(Filter f:resourceFilter.getFilters()) { JSONArray filtered=sourceCtx.read(f.getConditions().get(0)); diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java index f2805c9..842adae 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java @@ -3,24 +3,22 @@ package org.gcube.data.publishing.ckan2zenodo.commons; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; +import java.io.StringReader; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Unmarshaller; +import javax.xml.transform.stream.StreamSource; + import org.gcube.common.encryption.StringEncrypter; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.TargetElement; -import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; import lombok.extern.slf4j.Slf4j; @@ -53,69 +51,18 @@ public class IS { return client.submit(query); } - public static ArrayList readMappings(GenericResource res) throws ConfigurationException{ + public static Mappings readMappings(GenericResource res) throws ConfigurationException{ - ArrayList toReturn=new ArrayList(); try { + String body=res.profile().bodyAsString(); + + // create JAXB context and unmarshaller + Unmarshaller um=JAXBContext.newInstance(Mappings.class).createUnmarshaller(); + - Element root=res.profile().body(); - NodeList mappings=root.getElementsByTagName("mapping"); + return (Mappings) um.unmarshal(new StreamSource(new StringReader(body))); - for(int i = 0; i values=new HashMap<>(); - NodeList valueMappings=mapping.getElementsByTagName("valueMapping"); - for(int j = 0; j regularExpressions=new ArrayList<>(); - NodeList regexpDeclarations=mapping.getElementsByTagName("regexp"); - for(int j = 0; j filtersList=new ArrayList<>(); - Element root=res.profile().body(); - NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter"); - for(int i=0;i conditions=new ArrayList<>(); - NodeList conditionNodes=filter.getElementsByTagName("condition"); - for(int j=0;j filtersList=new ArrayList<>(); +// Element root=res.profile().body(); +// NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter"); +// for(int i=0;i conditions=new ArrayList<>(); +// NodeList conditionNodes=filter.getElementsByTagName("condition"); +// for(int j=0;j conditions; +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java index 775b013..0eb5df9 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java @@ -4,74 +4,34 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import lombok.ToString; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; -@RequiredArgsConstructor -@Getter -@ToString +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +@XmlAccessorType(XmlAccessType.FIELD) public class Mapping { - @RequiredArgsConstructor - @Getter - @ToString - public static class Regexp{ - public static enum Type{ - replace,extract - } - @NonNull - private Type type; - @NonNull - private String target; - @Setter - private String replacement; - } - - @Getter - @ToString - public static class Source{ - - @RequiredArgsConstructor - @Getter - @ToString - public static class Value{ - public static enum Type{ - jsonPath,constant - } - @Setter - private String split; - @NonNull - private Type type; - @NonNull - private String value; - } - - private List values=new ArrayList<>(); - } - - @Getter - @ToString - @RequiredArgsConstructor - public static class TargetElement{ - @Setter - private Boolean append = false; - @NonNull - private String targetElement; - } - - - @NonNull + @XmlElement private Source source; - @NonNull - private String targetPath; - @NonNull - private TargetElement targetElement; - @NonNull - private HashMap valueMapping; - @NonNull - private List regexp; + + @XmlElement + private TargetPath targetPath; + + + + @XmlElement(name="targetElement", required= true) + private List targetElements=new ArrayList(); + + @XmlElement + private HashMap valueMapping=new HashMap(); + + + @XmlElement + private List regexp=new ArrayList(); } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java new file mode 100644 index 0000000..1d55629 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java @@ -0,0 +1,26 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import lombok.Data; +import lombok.NoArgsConstructor; + + +@NoArgsConstructor +@Data +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class Mappings { + + + @XmlElement + private ResourceFilter resourceFilters; + + @XmlElement(name="mapping") + private List mappings; +} diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java new file mode 100644 index 0000000..1f84acc --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java @@ -0,0 +1,19 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import lombok.Data; + +@Data +@XmlAccessorType(XmlAccessType.FIELD) +public class Regexp{ + public static enum RegexpType{ + replace,extract + } + + private RegexpType type=RegexpType.extract; + + private String target; + private String replacement; +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java index 2d39e27..973c4b4 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java @@ -1,30 +1,27 @@ package org.gcube.data.publishing.ckan2zenodo.model.parsing; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.ToString; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; -@RequiredArgsConstructor -@ToString -@Getter +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) public class ResourceFilter { public static final ResourceFilter PASS_ALL=new ResourceFilter(Collections.singletonList( new Filter(Collections.singletonList("$.resources[?(@.format)]")))); - @RequiredArgsConstructor - @ToString - @Getter - public static class Filter { - @NonNull - private List conditions; - } - - @NonNull - private List filters; + @XmlElement(name="filter", required= false) + private List filters=new ArrayList(); } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java new file mode 100644 index 0000000..1ba5849 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java @@ -0,0 +1,20 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class Source{ + + @XmlElement(name="value", required= true) + private List values=new ArrayList<>(); +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java new file mode 100644 index 0000000..9ee2435 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java @@ -0,0 +1,24 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class TargetElement{ + + @XmlAttribute(required=false) + private Boolean append = false; + + @XmlAttribute(required=false) + private String constant= null; + + @XmlValue + private String Value; +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java new file mode 100644 index 0000000..077f81d --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java @@ -0,0 +1,31 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class TargetPath { + + + public static enum ElementType { + map,array + } + + + + @XmlAttribute(required=false) + private ElementType type=ElementType.map; + + @XmlAttribute(required=false) + private Boolean append=false; + + @XmlValue + private String value; +} diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java new file mode 100644 index 0000000..7aeee45 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java @@ -0,0 +1,27 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class Value{ + public static enum ValueType{ + jsonPath,constant + } + + @XmlAttribute(required=false) + private String split; + + @XmlAttribute(required=false) + private ValueType type=ValueType.constant; + + @XmlValue + private String value; +} \ No newline at end of file diff --git a/src/test/java/org/gcube/tests/OneHitTest.java b/src/test/java/org/gcube/tests/OneHitTest.java index 29eb1b7..47c4ab8 100644 --- a/src/test/java/org/gcube/tests/OneHitTest.java +++ b/src/test/java/org/gcube/tests/OneHitTest.java @@ -26,7 +26,7 @@ public class OneHitTest { TokenSetter.set(scope); Ckan2Zenodo client=new Ckan2ZenodoImpl(); - String toPublishItemName="my_test_bc_service"; + String toPublishItemName="a_sample_deliverable_4_zenodo"; // Get the item representation CkanItemDescriptor item=client.read(toPublishItemName); @@ -55,7 +55,7 @@ public class OneHitTest { System.out.println("Publishing.. "); //Finalize - System.out.println("DONE : "+client.publish(preview, item)); +// System.out.println("DONE : "+client.publish(preview, item)); } diff --git a/src/test/java/org/gcube/tests/Previewer.java b/src/test/java/org/gcube/tests/Previewer.java new file mode 100644 index 0000000..87c2908 --- /dev/null +++ b/src/test/java/org/gcube/tests/Previewer.java @@ -0,0 +1,34 @@ +package org.gcube.tests; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.Resources; +import org.gcube.data.publishing.ckan2zenodo.Translator; +import org.gcube.data.publishing.ckan2zenodo.commons.IS; +import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; +import org.junit.Test; + +public class Previewer { + + + @Test + public void preview() throws Exception { + String sourceItem="/blue_cloud_deliverable.json"; + String genResource="/blue_cloud_deliverable.xml"; + + GenericResource resource=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream(genResource)); + Mappings m=IS.readMappings(resource); + + try(FileOutputStream fos = new FileOutputStream("out.json")){ + TestCommons.getMapper().writeValue(fos, TestCommons.readAndTransform(sourceItem, new Translator(m))); + }; + + + } + + +} diff --git a/src/test/java/org/gcube/tests/TestCommons.java b/src/test/java/org/gcube/tests/TestCommons.java index 74f6b25..065bedf 100644 --- a/src/test/java/org/gcube/tests/TestCommons.java +++ b/src/test/java/org/gcube/tests/TestCommons.java @@ -1,6 +1,7 @@ package org.gcube.tests; import java.io.IOException; +import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; @@ -33,6 +34,7 @@ public class TestCommons { static Map getDefinedMappings(){ HashMap toReturn=new HashMap(); toReturn.put("/blue_cloud_dataset.json", "/blue_cloud_dataset.xml"); + toReturn.put("/blue_cloud_deliverable.json", "/blue_cloud_deliverable.xml"); return toReturn; } @@ -51,19 +53,23 @@ public class TestCommons { } static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,ZenodoDeposition...depositions) throws Exception { + return readAndTransform(jsonFile, transformer, System.out, depositions); + } + + static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,PrintStream os, ZenodoDeposition...depositions) throws Exception { try{ String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile)); CkanItemDescriptor desc=new CkanItemDescriptor(json); - System.out.println("Going to transform : "+desc.getContent()); - System.out.println("Result : "); + os.println("Going to transform : "+desc.getContent()); + os.println("Result : "); ZenodoDeposition dep=transformer.transform(desc, (depositions!=null&&depositions.length>0)?depositions[0]:null); - System.out.println(dep); - System.out.println("As JSON : "); - System.out.println(Fixer.fixSending(getMapper().writeValueAsString(dep))); - System.out.println("Filtered resources : "); + os.println(dep); + os.println("As JSON : "); + os.println(Fixer.fixSending(getMapper().writeValueAsString(dep))); + os.println("Filtered resources : "); for(CkanResource res : transformer.filterResources(desc)) { - System.out.println("AS JSON : "); - System.out.println(Fixer.fixSending(getMapper().writeValueAsString(res))); + os.println("AS JSON : "); + os.println(Fixer.fixSending(getMapper().writeValueAsString(res))); } return dep; }catch(Throwable t) { diff --git a/src/test/java/org/gcube/tests/TransformationTests.java b/src/test/java/org/gcube/tests/TransformationTests.java index a2c9dd2..271bd7b 100644 --- a/src/test/java/org/gcube/tests/TransformationTests.java +++ b/src/test/java/org/gcube/tests/TransformationTests.java @@ -1,5 +1,9 @@ package org.gcube.tests; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -9,6 +13,11 @@ import org.gcube.data.publishing.ckan2zenodo.Translator; import org.gcube.data.publishing.ckan2zenodo.commons.IS; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.CkanResource; +import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Filter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Regexp; import org.junit.BeforeClass; import org.junit.Test; @@ -28,6 +37,39 @@ public class TransformationTests { } + @Test + public void testUnmarshalling() throws ConfigurationException { + List toCheck=new ArrayList(); + toCheck.add("/simple.xml"); + toCheck.addAll(mappings.values()); + for(String resFile:toCheck) { + System.out.println("Checking "+resFile); + GenericResource resource=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream(resFile)); + Mappings m=IS.readMappings(resource); + assertTrue(m.getResourceFilters()!=null); + assertTrue(m.getResourceFilters().getFilters()!=null); + m.getResourceFilters().getFilters().forEach((Filter f)->{ + assertTrue(f.getConditions()!=null&&!f.getConditions().isEmpty());}); + assertTrue(m.getMappings()!=null); + m.getMappings().forEach((Mapping map)->{ +// assertTrue(map.getSource()!=); + }); + new Translator(m); + } + } + @Test + public void testMarhalling() { + + +// Mappings m=new Mappings(); +// Mapping m1=new Mapping(); +// m1. +// +// m.getMappings().add(new Mapping()); +// + + } + @Test public void transform() throws Exception { @@ -35,6 +77,7 @@ public class TransformationTests { TestCommons.readAndTransform("/simpleItem.json",basic); TestCommons.readAndTransform("/FSKXModel.json",basic); TestCommons.readAndTransform("/ResearchObject.json",basic); + } @@ -60,7 +103,7 @@ public class TransformationTests { @Test public void filterResources() throws Exception { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); - Translator t=new Translator(IS.readMappings(res),IS.readResourceFilters(res)); + Translator t=new Translator(IS.readMappings(res)); String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream("/ResearchObject.json")); CkanItemDescriptor desc=new CkanItemDescriptor(json); diff --git a/src/test/java/org/gcube/tests/ZenodoTests.java b/src/test/java/org/gcube/tests/ZenodoTests.java index 7bbeb22..b822051 100644 --- a/src/test/java/org/gcube/tests/ZenodoTests.java +++ b/src/test/java/org/gcube/tests/ZenodoTests.java @@ -79,7 +79,7 @@ public class ZenodoTests { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); ZenodoDeposition dep=z.createNew(); - Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res)); + Translator tran=new Translator(IS.readMappings(res)); dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep); dep=z.updateMetadata(dep); @@ -107,7 +107,7 @@ public class ZenodoTests { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); ZenodoDeposition dep=z.createNew(); - Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res)); + Translator tran=new Translator(IS.readMappings(res)); dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep); dep=z.updateMetadata(dep); @@ -159,6 +159,7 @@ public class ZenodoTests { ZenodoDeposition dep=TestCommons.readAndTransform(entry.getKey(), translator,z.createNew()); dep=z.updateMetadata(dep); System.out.println("Created "+dep); + //No payload to publish // System.out.println("Published "+z.publish(dep)); } } diff --git a/src/test/resources/ResearchObject.xml b/src/test/resources/ResearchObject.xml index c9de4c4..986abba 100644 --- a/src/test/resources/ResearchObject.xml +++ b/src/test/resources/ResearchObject.xml @@ -7,14 +7,16 @@ ResearchObject Simple mappings tests - + + + + + $.resources[?(@.format)] - - diff --git a/src/test/resources/blue_cloud_dataset.out.xml b/src/test/resources/blue_cloud_dataset.out.xml deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/resources/blue_cloud_dataset.xml b/src/test/resources/blue_cloud_dataset.xml index 204ad14..a205bc2 100644 --- a/src/test/resources/blue_cloud_dataset.xml +++ b/src/test/resources/blue_cloud_dataset.xml @@ -7,32 +7,33 @@ Dataset Simple mappings tests - - - $.resources[?(@.format)] - - - - + + + - + + + $.resources[?(@.format)] + + - dataset - + dataset + $.metadata upload_type - - + + - $.extras[?(@.key=='Identity:External Identifier')].value + $.extras[?(@.key=='Identity:External + Identifier')].value $.metadata.relatedIdentifier[1] identifier - + isAlternateIdentifier @@ -40,19 +41,20 @@ $.metadata.relatedIdentifier[1] type - - - - - + + + + + - $.extras[?(@.key=='Identity:RelatedPaper')].value + $.extras[?(@.key=='Identity:RelatedPaper')].value + $.metadata.relatedIdentifier[2] identifier - + isCitedBy @@ -60,21 +62,23 @@ $.metadata.relatedIdentifier[2] type - - - + + + - $.extras[?(@.key=='Coverage:Semantic Coverage')].value + $.extras[?(@.key=='Coverage:Semantic + Coverage')].value $.metadata - keywords[*] + keywords[*] - - + + - $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.metadata.dates[0] start @@ -84,11 +88,12 @@ [/ ] - + - $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.extras[?(@.key=='Coverage:TimeCoverage')].value + $.metadata.dates[0] end @@ -98,27 +103,29 @@ [/ ] - + Valid $.metadata.dates[0] - type + type - - + + - - + + - $.extras[?(@.key=='AccessMode:Accessibility')].value + $.extras[?(@.key=='AccessMode:Accessibility')].value + $.metadata - access_conditions + access_conditions + ^ AccessMode.Accessibility : @@ -128,14 +135,16 @@ ; - + - $.extras[?(@.key=='AccessMode:Availability')].value + $.extras[?(@.key=='AccessMode:Availability')].value + $.metadata - access_conditions + access_conditions + ^ AccessMode.Availability : @@ -145,14 +154,16 @@ ; - + - $.extras[?(@.key=='AccessMode:AccessibilityMode')].value + $.extras[?(@.key=='AccessMode:AccessibilityMode')].value + $.metadata - access_conditions + access_conditions + ^ AccessMode.AccessibilityMode : @@ -167,7 +178,8 @@ - $.extras[?(@.key=='TechnicalDetails:ProcessingDegree')].value + $.extras[?(@.key=='TechnicalDetails:ProcessingDegree')].value + $.metadata notes @@ -184,7 +196,8 @@ - $.extras[?(@.key=='TechnicalDetails:ManifestationType')].value + $.extras[?(@.key=='TechnicalDetails:ManifestationType')].value + $.metadata notes @@ -201,16 +214,18 @@ - $.extras[?(@.key=='TechnicalDetails:Language')].value + $.extras[?(@.key=='TechnicalDetails:Language')].value + $.metadata language - + - $.extras[?(@.key=='TechnicalDetails:Size')].value + $.extras[?(@.key=='TechnicalDetails:Size')].value + $.metadata notes @@ -223,11 +238,12 @@ ; - + - $.extras[?(@.key=='TechnicalDetails:DiskSize')].value + $.extras[?(@.key=='TechnicalDetails:DiskSize')].value + $.metadata notes @@ -240,11 +256,12 @@ ; - + - $.extras[?(@.key=='TechnicalDetails:Format')].value + $.extras[?(@.key=='TechnicalDetails:Format')].value + $.metadata notes @@ -257,11 +274,12 @@ ; - + - $.extras[?(@.key=='TechnicalDetails:FormatSchema')].value + $.extras[?(@.key=='TechnicalDetails:FormatSchema')].value + $.metadata notes @@ -279,7 +297,8 @@ - $.extras[?(@.key=='DataProtection:PersonalData')].value + $.extras[?(@.key=='DataProtection:PersonalData')].value + $.metadata notes @@ -296,7 +315,8 @@ - $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.metadata notes @@ -314,13 +334,15 @@ - $.extras[?(@.key=='DataProtection:Consent of the data subject')].value + $.extras[?(@.key=='DataProtection:Consent of the + data subject')].value $.metadata notes ^ - DataProtection.Consent of the data subject : + DataProtection.Consent of the data subject : + $ @@ -328,33 +350,41 @@ - + - $.extras[?(@.key=='DataProtection:Consent obtained also covers the envisaged transfer of the personal data outside the EU')].value + $.extras[?(@.key=='DataProtection:Consent + obtained also covers the envisaged transfer of the personal data + outside the EU')].value $.metadata notes ^ - DataProtection.Consent obtained also covers the envisaged transfer of the personal data outside the EU : + DataProtection.Consent obtained also covers the + envisaged transfer of the personal data outside the EU : + $ ; - - + + - $.extras[?(@.key=='DataProtection:Personal data was manifestly made public by the data subject')].value + $.extras[?(@.key=='DataProtection:Personal data + was manifestly made public by the data subject')].value $.metadata notes ^ - DataProtection.Personal data was manifestly made public by the data subject : + DataProtection.Personal data was manifestly made + public by the data subject : $ @@ -365,13 +395,15 @@ - $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value + $.metadata notes ^ - DataProtection.DataProtectionDirective : + DataProtection.DataProtectionDirective : + $ @@ -383,7 +415,8 @@ - $.extras[?(@.key=='Rights:IP/Copyrights')].value + $.extras[?(@.key=='Rights:IP/Copyrights')].value + $.metadata notes @@ -396,11 +429,12 @@ ; - + - $.extras[?(@.key=='Rights:Field/Scope of use')].value + $.extras[?(@.key=='Rights:Field/Scope of + use')].value $.metadata notes @@ -413,12 +447,13 @@ ; - - + + - $.extras[?(@.key=='Rights:Basic rights')].value + $.extras[?(@.key=='Rights:Basic rights')].value + $.metadata notes @@ -436,7 +471,8 @@ - $.extras[?(@.key=='Rights:Restrictions on use')].value + $.extras[?(@.key=='Rights:Restrictions on + use')].value $.metadata notes @@ -450,11 +486,12 @@ - + - $.extras[?(@.key=='Rights:Sublicense rights')].value + $.extras[?(@.key=='Rights:Sublicense + rights')].value $.metadata notes @@ -471,7 +508,8 @@ - $.extras[?(@.key=='Rights:Territory of use')].value + $.extras[?(@.key=='Rights:Territory of + use')].value $.metadata notes @@ -488,7 +526,8 @@ - $.extras[?(@.key=='Rights:License term')].value + $.extras[?(@.key=='Rights:License term')].value + $.metadata notes @@ -501,18 +540,20 @@ ; - + - $.extras[?(@.key=='Rights:Requirement of non-disclosure (confidentiality mark)')].value + $.extras[?(@.key=='Rights:Requirement of + non-disclosure (confidentiality mark)')].value $.metadata notes ^ - Rights.Requirement of non-disclosure (confidentiality mark) : + Rights.Requirement of non-disclosure (confidentiality + mark) : $ @@ -524,7 +565,8 @@ - $.extras[?(@.key=='Attribution:Attribution requirements')].value + $.extras[?(@.key=='Attribution:Attribution + requirements')].value $.metadata notes @@ -542,7 +584,8 @@ - $.extras[?(@.key=='Attribution:Attribution requirements')].value + $.extras[?(@.key=='Attribution:Attribution + requirements')].value $.metadata notes @@ -560,7 +603,8 @@ - $.extras[?(@.key=='Attribution:Distribution requirements')].value + $.extras[?(@.key=='Attribution:Distribution + requirements')].value $.metadata notes diff --git a/src/test/resources/blue_cloud_deliverable.json b/src/test/resources/blue_cloud_deliverable.json new file mode 100644 index 0000000..e6f0e56 --- /dev/null +++ b/src/test/resources/blue_cloud_deliverable.json @@ -0,0 +1,120 @@ +{ + "rating": 0.0, + "license_title": "Creative Commons Attribution Share-Alike 4.0", + "maintainer": "Candela Leonardo", + "relationships_as_object": [], + "private": false, + "maintainer_email": "leonardo.candela@isti.cnr.it", + "num_tags": 1, + "id": "ae46038a-10e9-4e62-a6c6-758e71a3428f", + "metadata_created": "2021-02-25T09:55:42.319211", + "metadata_modified": "2021-02-25T15:54:50.597047", + "author": "Candela Leonardo", + "author_email": "leonardo.candela@isti.cnr.it", + "acquire_url": "", + "state": "active", + "version": "1", + "creator_user_id": "be67d795-9fc8-4a8d-a62d-3113b724f83c", + "type": "dataset", + "resources": [ + { + "mimetype": null, + "cache_url": null, + "hash": "", + "description": "", + "name": "Deliverable", + "format": "PDF", + "url": "https://data-pre.d4science.net/RgA7", + "datastore_active": false, + "cache_last_updated": null, + "package_id": "ae46038a-10e9-4e62-a6c6-758e71a3428f", + "created": "2021-02-25T09:57:53.477077", + "state": "active", + "mimetype_inner": null, + "last_modified": null, + "position": 0, + "revision_id": "f8875fc2-e7c4-4949-a122-343a646f05e4", + "url_type": null, + "id": "e2363d7f-3c67-40bc-b6bb-9111bfcf063b", + "resource_type": null, + "size": null + } + ], + "num_resources": 1, + "tags": [ + { + "vocabulary_id": null, + "state": "active", + "display_name": "Text mining", + "id": "331ceb06-9ec3-4a10-a46a-5124f4dc19f2", + "name": "Text mining" + } + ], + "groups": [], + "license_id": "CC-BY-SA-4.0", + "relationships_as_subject": [], + "organization": { + "description": "The preVRE organisation", + "created": "2019-05-02T12:41:26.149836", + "title": "preVRE", + "name": "prevre", + "is_organization": true, + "state": "active", + "image_url": "https://pre.d4science.org/image/layout_set_logo?img_id=12949941", + "revision_id": "a44aebb5-5f88-4d53-94d2-bfb17b8d340b", + "type": "organization", + "id": "46ce33da-dbc1-4116-b41d-eb961435a53d", + "approval_status": "approved" + }, + "name": "a_sample_deliverable_4_zenodo", + "isopen": true, + "url": "", + "notes": "This is a sample deliverable created for testing the publish2Zenodo \r\nfacility", + "owner_org": "46ce33da-dbc1-4116-b41d-eb961435a53d", + "extras": [ + { + "key": "Deliverable Author", + "value": "Rossi, Pietro, pietro.rossi@acme.org" + }, + { + "key": "Deliverable Author", + "value": "Verdi, Luca, lica.verdi@amce.org" + }, + { + "key": "Deliverable Author", + "value": "Bianchi, Elena" + }, + { + "key": "Deliverable Contributor", + "value": "Ferrari, Paolo, paolo.ferrari@acme.org" + }, + { + "key": "Deliverable Contributor", + "value": "Russo, Pietro, pietro.russo@amce.org" + }, + { + "key": "Deliverable Contributor", + "value": "Giorgino, Giorgio" + }, + { + "key": "Deliverable ID", + "value": "Dx.y" + }, + { + "key": "Item URL", + "value": "https://data-pre.d4science.org/ctlg/preVRE/a_sample_deliverable_4_zenodo" + }, + { + "key": "Publication Date", + "value": "2021-02-25 " + }, + { + "key": "system:type", + "value": "Deliverable" + } + ], + "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", + "ratings_count": 0, + "title": "A sample deliverable 4 Zenodo", + "revision_id": "cbd34661-8303-4e7b-8935-beb111ad1925" +} \ No newline at end of file diff --git a/src/test/resources/blue_cloud_deliverable.xml b/src/test/resources/blue_cloud_deliverable.xml new file mode 100644 index 0000000..9d08279 --- /dev/null +++ b/src/test/resources/blue_cloud_deliverable.xml @@ -0,0 +1,70 @@ + + 12345 + GenericResource + + + Ckan-Zenodo-Mappings + Deliverable + Simple mappings tests + + + + + + + $.resources[?(@.format)] + + + + + + other + + $.metadata + upload_type + + + + + $.extras[?(@.key=='Deliverable Author')].value + + $.metadata.creators + name + + + + + + + + $.extras[?(@.key=='Deliverable Contributor')].value + + $.metadata.contributors + name + type + + + + + + + 862409 + + $.metadata.grants + id + + + + + blue-cloud + + $.metadata.communities + identifier + + + + + + + + \ No newline at end of file diff --git a/src/test/resources/simple.xml b/src/test/resources/simple.xml new file mode 100644 index 0000000..9573cc3 --- /dev/null +++ b/src/test/resources/simple.xml @@ -0,0 +1,30 @@ + + 12345 + GenericResource + + + Ckan-Zenodo-Mappings + Deliverable + Simple mappings tests + + + + + $.resources[?(@.format)] + + + + + + other + + $.metadata + upload_type + + + + + + + + \ No newline at end of file