gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved mappings

This commit is contained in:
Fabio Sinibaldi 2021-03-08 17:12:28 +01:00
parent 96315e7924
commit 81048172b2
27 changed files with 819 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -2,7 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
# Changelog for org.gcube.data.publishing.ckan2zenodo-library
## [v0.0.4-SNAPSHOT] 2021-02-18
- Changed coordinartes for gcat-client
- Improved mapping language : target record
- Improved mapping language : value iteration
## [v0.0.3-SNAPSHOT] 2020-06-30
- Changed coordinartes for gcat-client

1
out.json Normal file
View File

@ -0,0 +1 @@
{"metadata":{"upload_type":"other","creators":[{"name":"Rossi, Pietro, pietro.rossi@acme.org"},{"name":"Verdi, Luca, lica.verdi@amce.org"},{"name":"Bianchi, Elena"}],"contributors":[{"name":"Ferrari, Paolo, paolo.ferrari@acme.org","type":"Other"},{"name":"Russo, Pietro, pietro.russo@amce.org","type":"Other"},{"name":"Giorgino, Giorgio","type":"Other"}],"communities":[{"identifier":"blue-cloud"}],"grants":[{"id":"862409"}]},"doiurl":"https://doi.org/null"}

13
pom.xml
View File

@ -39,12 +39,19 @@
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.data-catalogue</groupId>
<!-- <dependency> -->
<!-- <groupId>org.gcube.data-catalogue</groupId> -->
<!-- <artifactId>gcat-client</artifactId> -->
<!-- <version>[2.0.0-SNAPSHOT,3.0.0)</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.gcube.data-publishing</groupId>
<artifactId>gcat-client</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0)</version>
<version>[1.0.0-SNAPSHOT,2.0.0)</version>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-encryption</artifactId>

2
src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java
View File

@ -29,7 +29,7 @@ public class TransformerManager {
public Translator getByProfile(String profile) throws ConfigurationException {
for(GenericResource r: IS.queryForGenericResources("Ckan-Zenodo-Mappings")){
if (r.profile().name().equals(profile))
return new Translator(IS.readMappings(r),IS.readResourceFilters(r));
return new Translator(IS.readMappings(r));
}
throw new ConfigurationException("No specific mapping for the catalogue item has been configured. "
+ "By continuing with the upload some metadata might not be upload to Zenodo.");

141
src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java
View File

@ -5,9 +5,11 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -15,11 +17,13 @@ import org.gcube.data.publishing.ckan2zenodo.commons.Parsing;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Filter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter.Filter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.TargetElement;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Value;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Contributor;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Creator;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
@ -56,6 +60,10 @@ public class Translator {
this(mappings,ResourceFilter.PASS_ALL);
}
public Translator(Mappings m) {
this(m.getMappings(),m.getResourceFilters());
}
public ZenodoDeposition transform(CkanItemDescriptor toTransform, ZenodoDeposition deposition) throws TransformationException {
log.debug("Transforming "+toTransform+". Existing Deposition is : "+deposition);
@ -154,24 +162,33 @@ public class Translator {
DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent());
DocumentContext targetCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(mapper.writeValueAsString(target));
// FOR EACH MAPPING
// IF source
// For each source value ; NB GET FIRST MATCHING Value path
// APPLY TRANSFORMATIONS
// FIND/INIT TARGET PATH (OPTS isArray, replace)
// SET TARGET ELEMENT (OPTS replace, regexp)
// IF SINGLE -> property
// iF MULITPLE -> object
// ELSE TODO
for(Mapping mapping:mappings) {
log.debug("Applying {} ",mapping);
try {
// extract source
// "EVALUATE SURCE VALUES"
List<String> sourceValues=new ArrayList<>();
for(Value v: mapping.getSource().getValues()) {
String actualValue=null;
List<String> actualValues=new ArrayList<String>();
switch(v.getType()) {
case constant : {
actualValue=v.getValue();
actualValues.add(v.getValue());
break;
}
case jsonPath : {
for(String s: ((Collection<? extends String>) sourceCtx.read(v.getValue()))){
if(s!=null) {
s=s.trim();
if(!s.isEmpty())actualValue=s;
if(!s.isEmpty())actualValues.add(s);
}
}
@ -180,24 +197,50 @@ public class Translator {
}
// Adding to actual values
if(actualValue!=null) {
// Applygin splits
for(String foundVal:actualValues) {
if(v.getSplit()!=null)
for(String toAdd:actualValue.split(v.getSplit()))
for(String toAdd:foundVal.split(v.getSplit()))
sourceValues.add(toAdd.trim());
else sourceValues.add(actualValue);
else sourceValues.add(foundVal);
}
if(!sourceValues.isEmpty()) break;
}
log.debug("Found matching "+sourceValues);
// ************** INIT TARGET PATH
// CHECK INIT TARGET PATH
ArrayList<Map<String,Object>> resultingValueList=new ArrayList<Map<String,Object>>();
if(!sourceValues.isEmpty()) {
List<String> targetElementFound=targetCtx.read(mapping.getTargetPath().getValue());
if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null ||
!mapping.getTargetPath().getAppend()) {
// targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing"));
JsonPath path=JsonPath.compile(mapping.getTargetPath().getValue());
switch(mapping.getTargetPath().getType()) {
case array :
targetCtx=targetCtx.set(path,new Object[sourceValues.size()]);
break;
case map :
targetCtx=targetCtx.set(path,null);
break;
}
}
}
// ************** PREPARE VALUE FOR EACH SOURCE VALUE
for(String sourceValue:sourceValues) {
String resultingValue=sourceValue;
log.debug("Managing "+resultingValue);
// apply regexps
for(Regexp regexp:mapping.getRegexp()) {
@ -221,31 +264,63 @@ public class Translator {
}
// apply value mappings
resultingValue =mapping.getValueMapping().getOrDefault(sourceValue, resultingValue);
// check if targetPath exists
List<String> targetElementFound=targetCtx.read(mapping.getTargetPath());
if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null) {
// targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing"));
targetCtx=Parsing.addElement(targetCtx, mapping.getTargetPath());
}
// NEW : Multiple Target Elements can be simple or structure
Map<String,Object> resultingTargetElements=new HashMap<String, Object>();
for(TargetElement el:mapping.getTargetElements()) {
if(el.getConstant()!=null) resultingValue=el.getConstant();
else
if(el.getAppend()){
String original=((List<String>)targetCtx.read(mapping.getTargetPath()+"."+el.getValue())).get(0);
if(original!=null && !original.isEmpty())
resultingValue=original+resultingValue;
}
resultingTargetElements.put(el.getValue(), resultingValue);
if(mapping.getTargetElement().getAppend()){
String original=((List<String>)targetCtx.read(mapping.getTargetPath()+"."+mapping.getTargetElement().getTargetElement())).get(0);
if(original!=null && !original.isEmpty())
resultingValue=original+resultingValue;
}
targetCtx=targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement().getTargetElement(),
resultingValue);
resultingValueList.add(resultingTargetElements);
// close loop on values
}
JsonPath path=JsonPath.compile(mapping.getTargetPath().getValue());
switch(mapping.getTargetPath().getType()) {
case array :
targetCtx.set(path, resultingValueList);
break;
case map :
for(Map<String,Object> resultingTargetElements : resultingValueList) {
log.debug("Applying "+resultingTargetElements+ " to "+mapping.getTargetPath());
targetCtx.set(path, resultingTargetElements);
}
break;
}
// for(Map<String,Object> resultingTargetElements : resultingValueList) {
// log.debug("Applying "+resultingTargetElements+ " to "+mapping.getTargetPath());
// // Apply object
//
//
// switch(mapping.getTargetPath().getType()) {
// case array :
// targetCtx=targetCtx.add(path, resultingTargetElements);
// break;
// case map :
// targetCtx.set(path, resultingTargetElements);
// break;
// }
// }
}catch(Throwable t) {
throw new TransformationException("Exception while applying "+mapping,t);
}
@ -261,9 +336,9 @@ public class Translator {
public List<CkanResource> filterResources(CkanItemDescriptor source) throws TransformationException{
ObjectMapper mapper=Parsing.getMapper();
DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent());
try {
ObjectMapper mapper=Parsing.getMapper();
DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent());
try {
HashSet<CkanResource> toReturn=new HashSet();
for(Filter f:resourceFilter.getFilters()) {
JSONArray filtered=sourceCtx.read(f.getConditions().get(0));

127
src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java
View File

@ -3,24 +3,22 @@ package org.gcube.data.publishing.ckan2zenodo.commons;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller;
import javax.xml.transform.stream.StreamSource;
import org.gcube.common.encryption.StringEncrypter;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.TargetElement;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import lombok.extern.slf4j.Slf4j;
@ -53,69 +51,18 @@ public class IS {
return client.submit(query);
}
public static ArrayList<Mapping> readMappings(GenericResource res) throws ConfigurationException{
public static Mappings readMappings(GenericResource res) throws ConfigurationException{
ArrayList<Mapping> toReturn=new ArrayList<Mapping>();
try {
String body=res.profile().bodyAsString();
// create JAXB context and unmarshaller
Unmarshaller um=JAXBContext.newInstance(Mappings.class).createUnmarshaller();
Element root=res.profile().body();
NodeList mappings=root.getElementsByTagName("mapping");
return (Mappings) um.unmarshal(new StreamSource(new StringReader(body)));
for(int i = 0; i<mappings.getLength();i++) {
Element mapping=(Element) mappings.item(i);
Element sourceElement=(Element) mapping.getElementsByTagName("source").item(0);
NodeList valuesNodeList=sourceElement.getElementsByTagName("value");
Source source=new Source();
for(int j=0;j<valuesNodeList.getLength();j++) {
Element valueElement=(Element) valuesNodeList.item(j);
Value valueObject=new Value(Value.Type.valueOf(valueElement.getAttribute("type")),
valueElement.getTextContent());
if(valueElement.hasAttribute("split"))
valueObject.setSplit(valueElement.getAttribute("split"));
source.getValues().add(valueObject);
}
String targetPath=mapping.getElementsByTagName("targetPath").item(0).getTextContent();
Element targetElement=(Element) mapping.getElementsByTagName("targetElement").item(0);
TargetElement targetObject=new TargetElement(targetElement.getTextContent());
if(targetElement.hasAttribute("append"))
targetObject.setAppend(Boolean.parseBoolean(targetElement.getAttribute("append")));
HashMap<String,String> values=new HashMap<>();
NodeList valueMappings=mapping.getElementsByTagName("valueMapping");
for(int j = 0; j<valueMappings.getLength();j++) {
Element codelistMapping=(Element) valueMappings.item(j);
String sourceValue=codelistMapping.getElementsByTagName("sourceValue").item(0).getTextContent();
String targetValue=codelistMapping.getElementsByTagName("targetValue").item(0).getTextContent();
values.put(sourceValue, targetValue);
}
ArrayList<Regexp> regularExpressions=new ArrayList<>();
NodeList regexpDeclarations=mapping.getElementsByTagName("regexp");
for(int j = 0; j<regexpDeclarations.getLength();j++) {
Element regexpElement=(Element) regexpDeclarations.item(j);
String regexpTarget=regexpElement.getElementsByTagName("target").item(0).getTextContent();
String typeName=regexpElement.getAttribute("type");
Regexp regexp=new Regexp(Regexp.Type.valueOf(typeName),regexpTarget);
if(regexp.getType().equals(Regexp.Type.replace))
regexp.setReplacement(regexpElement.getElementsByTagName("replacement").item(0).getTextContent());
regularExpressions.add(regexp);
}
toReturn.add(new Mapping(source,targetPath,targetObject,values,regularExpressions));
}
return toReturn;
}catch(Throwable t) {
log.debug("Error while parsing mapping from resource "+res.id()+" name : "+res.profile().name(),t);
throw new ConfigurationException("Invaild mapping resource "+res.id()+" name : "+res.profile().name(),t);
@ -123,29 +70,29 @@ public class IS {
}
public static ResourceFilter readResourceFilters(GenericResource res) throws ConfigurationException{
try{
ArrayList<ResourceFilter.Filter> filtersList=new ArrayList<>();
Element root=res.profile().body();
NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter");
for(int i=0;i<filters.getLength();i++) {
Element filter=(Element) filters.item(i);
ArrayList<String> conditions=new ArrayList<>();
NodeList conditionNodes=filter.getElementsByTagName("condition");
for(int j=0;j<conditionNodes.getLength();j++) {
conditions.add(conditionNodes.item(j).getTextContent());
}
if(!conditions.isEmpty())
filtersList.add(new ResourceFilter.Filter(conditions));
}
if(filtersList.isEmpty())
throw new Exception("No Filter actually declared.");
return new ResourceFilter(filtersList);
}catch(Throwable t) {
log.debug("Error while parsing filters from resource "+res.id()+" name : "+res.profile().name(),t);
throw new ConfigurationException("Invalid resource filters declared in "+res.id()+" name : "+res.profile().name(),t);
}
}
// public static ResourceFilter readResourceFilters(GenericResource res) throws ConfigurationException{
// try{
// ArrayList<ResourceFilter.Filter> filtersList=new ArrayList<>();
// Element root=res.profile().body();
// NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter");
// for(int i=0;i<filters.getLength();i++) {
// Element filter=(Element) filters.item(i);
// ArrayList<String> conditions=new ArrayList<>();
// NodeList conditionNodes=filter.getElementsByTagName("condition");
// for(int j=0;j<conditionNodes.getLength();j++) {
// conditions.add(conditionNodes.item(j).getTextContent());
// }
// if(!conditions.isEmpty())
// filtersList.add(new ResourceFilter.Filter(conditions));
// }
// if(filtersList.isEmpty())
// throw new Exception("No Filter actually declared.");
// return new ResourceFilter(filtersList);
// }catch(Throwable t) {
// log.debug("Error while parsing filters from resource "+res.id()+" name : "+res.profile().name(),t);
// throw new ConfigurationException("Invalid resource filters declared in "+res.id()+" name : "+res.profile().name(),t);
// }
// }
public static String decryptString(String toDecrypt){
try{

2
src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/Parsing.java
View File

@ -30,7 +30,7 @@ public class Parsing {
ObjectMapper mapper=new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,false);
mapper.configure(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, false);
mapper.setSerializationInclusion(Include.NON_NULL);
// mapper.setSerializationInclusion(Include.NON_NULL);
return mapper;
}

20
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Filter.java Normal file
View File

@ -0,0 +1,20 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@XmlAccessorType(XmlAccessType.FIELD)
public class Filter {
@XmlElement(name="condition")
private List<String> conditions;
}

90
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java
View File

@ -4,74 +4,34 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
@RequiredArgsConstructor
@Getter
@ToString
import lombok.Data;
import lombok.NoArgsConstructor;
@NoArgsConstructor
@Data
@XmlAccessorType(XmlAccessType.FIELD)
public class Mapping {
@RequiredArgsConstructor
@Getter
@ToString
public static class Regexp{
public static enum Type{
replace,extract
}
@NonNull
private Type type;
@NonNull
private String target;
@Setter
private String replacement;
}
@Getter
@ToString
public static class Source{
@RequiredArgsConstructor
@Getter
@ToString
public static class Value{
public static enum Type{
jsonPath,constant
}
@Setter
private String split;
@NonNull
private Type type;
@NonNull
private String value;
}
private List<Value> values=new ArrayList<>();
}
@Getter
@ToString
@RequiredArgsConstructor
public static class TargetElement{
@Setter
private Boolean append = false;
@NonNull
private String targetElement;
}
@NonNull
@XmlElement
private Source source;
@NonNull
private String targetPath;
@NonNull
private TargetElement targetElement;
@NonNull
private HashMap<String,String> valueMapping;
@NonNull
private List<Regexp> regexp;
@XmlElement
private TargetPath targetPath;
@XmlElement(name="targetElement", required= true)
private List<TargetElement> targetElements=new ArrayList<TargetElement>();
@XmlElement
private HashMap<String,String> valueMapping=new HashMap<String, String>();
@XmlElement
private List<Regexp> regexp=new ArrayList<Regexp>();
}

26
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mappings.java Normal file
View File

@ -0,0 +1,26 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import lombok.Data;
import lombok.NoArgsConstructor;
@NoArgsConstructor
@Data
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class Mappings {
@XmlElement
private ResourceFilter resourceFilters;
@XmlElement(name="mapping")
private List<Mapping> mappings;
}

19
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Regexp.java Normal file
View File

@ -0,0 +1,19 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import lombok.Data;
@Data
@XmlAccessorType(XmlAccessType.FIELD)
public class Regexp{
public static enum RegexpType{
replace,extract
}
private RegexpType type=RegexpType.extract;
private String target;
private String replacement;
}

31
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java
View File

@ -1,30 +1,27 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
@RequiredArgsConstructor
@ToString
@Getter
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@XmlAccessorType(XmlAccessType.FIELD)
public class ResourceFilter {
public static final ResourceFilter PASS_ALL=new ResourceFilter(Collections.singletonList(
new Filter(Collections.singletonList("$.resources[?(@.format)]"))));
@RequiredArgsConstructor
@ToString
@Getter
public static class Filter {
@NonNull
private List<String> conditions;
}
@NonNull
private List<Filter> filters;
@XmlElement(name="filter", required= false)
private List<Filter> filters=new ArrayList<Filter>();
}

20
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Source.java Normal file
View File

@ -0,0 +1,20 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@XmlAccessorType(XmlAccessType.FIELD)
public class Source{
@XmlElement(name="value", required= true)
private List<Value> values=new ArrayList<>();
}

24
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetElement.java Normal file
View File

@ -0,0 +1,24 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlValue;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@XmlAccessorType(XmlAccessType.FIELD)
public class TargetElement{
@XmlAttribute(required=false)
private Boolean append = false;
@XmlAttribute(required=false)
private String constant= null;
@XmlValue
private String Value;
}

31
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/TargetPath.java Normal file
View File

@ -0,0 +1,31 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlValue;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@XmlAccessorType(XmlAccessType.FIELD)
public class TargetPath {
public static enum ElementType {
map,array
}
@XmlAttribute(required=false)
private ElementType type=ElementType.map;
@XmlAttribute(required=false)
private Boolean append=false;
@XmlValue
private String value;
}

27
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Value.java Normal file
View File

@ -0,0 +1,27 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlValue;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@XmlAccessorType(XmlAccessType.FIELD)
public class Value{
public static enum ValueType{
jsonPath,constant
}
@XmlAttribute(required=false)
private String split;
@XmlAttribute(required=false)
private ValueType type=ValueType.constant;
@XmlValue
private String value;
}

4
src/test/java/org/gcube/tests/OneHitTest.java
View File

@ -26,7 +26,7 @@ public class OneHitTest {
TokenSetter.set(scope);
Ckan2Zenodo client=new Ckan2ZenodoImpl();
String toPublishItemName="my_test_bc_service";
String toPublishItemName="a_sample_deliverable_4_zenodo";
// Get the item representation
CkanItemDescriptor item=client.read(toPublishItemName);
@ -55,7 +55,7 @@ public class OneHitTest {
System.out.println("Publishing.. ");
//Finalize
System.out.println("DONE : "+client.publish(preview, item));
// System.out.println("DONE : "+client.publish(preview, item));
}

34
src/test/java/org/gcube/tests/Previewer.java Normal file
View File

@ -0,0 +1,34 @@
package org.gcube.tests;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.Resources;
import org.gcube.data.publishing.ckan2zenodo.Translator;
import org.gcube.data.publishing.ckan2zenodo.commons.IS;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings;
import org.junit.Test;
public class Previewer {
@Test
public void preview() throws Exception {
String sourceItem="/blue_cloud_deliverable.json";
String genResource="/blue_cloud_deliverable.xml";
GenericResource resource=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream(genResource));
Mappings m=IS.readMappings(resource);
try(FileOutputStream fos = new FileOutputStream("out.json")){
TestCommons.getMapper().writeValue(fos, TestCommons.readAndTransform(sourceItem, new Translator(m)));
};
}
}

22
src/test/java/org/gcube/tests/TestCommons.java
View File

@ -1,6 +1,7 @@
package org.gcube.tests;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
@ -33,6 +34,7 @@ public class TestCommons {
static Map<String,String> getDefinedMappings(){
HashMap<String,String> toReturn=new HashMap<String,String>();
toReturn.put("/blue_cloud_dataset.json", "/blue_cloud_dataset.xml");
toReturn.put("/blue_cloud_deliverable.json", "/blue_cloud_deliverable.xml");
return toReturn;
}
@ -51,19 +53,23 @@ public class TestCommons {
}
static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,ZenodoDeposition...depositions) throws Exception {
return readAndTransform(jsonFile, transformer, System.out, depositions);
}
static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,PrintStream os, ZenodoDeposition...depositions) throws Exception {
try{
String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile));
CkanItemDescriptor desc=new CkanItemDescriptor(json);
System.out.println("Going to transform : "+desc.getContent());
System.out.println("Result : ");
os.println("Going to transform : "+desc.getContent());
os.println("Result : ");
ZenodoDeposition dep=transformer.transform(desc, (depositions!=null&&depositions.length>0)?depositions[0]:null);
System.out.println(dep);
System.out.println("As JSON : ");
System.out.println(Fixer.fixSending(getMapper().writeValueAsString(dep)));
System.out.println("Filtered resources : ");
os.println(dep);
os.println("As JSON : ");
os.println(Fixer.fixSending(getMapper().writeValueAsString(dep)));
os.println("Filtered resources : ");
for(CkanResource res : transformer.filterResources(desc)) {
System.out.println("AS JSON : ");
System.out.println(Fixer.fixSending(getMapper().writeValueAsString(res)));
os.println("AS JSON : ");
os.println(Fixer.fixSending(getMapper().writeValueAsString(res)));
}
return dep;
}catch(Throwable t) {

45
src/test/java/org/gcube/tests/TransformationTests.java
View File

@ -1,5 +1,9 @@
package org.gcube.tests;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -9,6 +13,11 @@ import org.gcube.data.publishing.ckan2zenodo.Translator;
import org.gcube.data.publishing.ckan2zenodo.commons.IS;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Filter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Regexp;
import org.junit.BeforeClass;
import org.junit.Test;
@ -28,6 +37,39 @@ public class TransformationTests {
}
@Test
public void testUnmarshalling() throws ConfigurationException {
List<String> toCheck=new ArrayList<String>();
toCheck.add("/simple.xml");
toCheck.addAll(mappings.values());
for(String resFile:toCheck) {
System.out.println("Checking "+resFile);
GenericResource resource=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream(resFile));
Mappings m=IS.readMappings(resource);
assertTrue(m.getResourceFilters()!=null);
assertTrue(m.getResourceFilters().getFilters()!=null);
m.getResourceFilters().getFilters().forEach((Filter f)->{
assertTrue(f.getConditions()!=null&&!f.getConditions().isEmpty());});
assertTrue(m.getMappings()!=null);
m.getMappings().forEach((Mapping map)->{
// assertTrue(map.getSource()!=);
});
new Translator(m);
}
}
@Test
public void testMarhalling() {
// Mappings m=new Mappings();
// Mapping m1=new Mapping();
// m1.
//
// m.getMappings().add(new Mapping());
//
}
@Test
public void transform() throws Exception {
@ -35,6 +77,7 @@ public class TransformationTests {
TestCommons.readAndTransform("/simpleItem.json",basic);
TestCommons.readAndTransform("/FSKXModel.json",basic);
TestCommons.readAndTransform("/ResearchObject.json",basic);
}
@ -60,7 +103,7 @@ public class TransformationTests {
@Test
public void filterResources() throws Exception {
GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml"));
Translator t=new Translator(IS.readMappings(res),IS.readResourceFilters(res));
Translator t=new Translator(IS.readMappings(res));
String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream("/ResearchObject.json"));
CkanItemDescriptor desc=new CkanItemDescriptor(json);

5
src/test/java/org/gcube/tests/ZenodoTests.java
View File

@ -79,7 +79,7 @@ public class ZenodoTests {
GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml"));
ZenodoDeposition dep=z.createNew();
Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res));
Translator tran=new Translator(IS.readMappings(res));
dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep);
dep=z.updateMetadata(dep);
@ -107,7 +107,7 @@ public class ZenodoTests {
GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml"));
ZenodoDeposition dep=z.createNew();
Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res));
Translator tran=new Translator(IS.readMappings(res));
dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep);
dep=z.updateMetadata(dep);
@ -159,6 +159,7 @@ public class ZenodoTests {
ZenodoDeposition dep=TestCommons.readAndTransform(entry.getKey(), translator,z.createNew());
dep=z.updateMetadata(dep);
System.out.println("Created "+dep);
//No payload to publish
// System.out.println("Published "+z.publish(dep));
}
}

8
src/test/resources/ResearchObject.xml
View File

@ -7,14 +7,16 @@
<Name>ResearchObject</Name>
<Description>Simple mappings tests</Description>
<Body>
<resourceFilters>
<mappings>
<resourceFilters>
<filter>
<condition>$.resources[?(@.format)]</condition>
</filter>
</resourceFilters>
<mappings>
<mapping>
<source>

0
src/test/resources/blue_cloud_dataset.out.xml
View File

216
src/test/resources/blue_cloud_dataset.xml
View File

@ -7,32 +7,33 @@
<Name>Dataset</Name>
<Description>Simple mappings tests</Description>
<Body>
<resourceFilters>
<filter>
<condition>$.resources[?(@.format)]</condition>
</filter>
</resourceFilters>
<mappings>
<resourceFilters>
<filter>
<condition>$.resources[?(@.format)]</condition>
</filter>
</resourceFilters>
<mapping>
<source>
<value type="constant">dataset</value>
</source>
<value type="constant">dataset</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement>upload_type</targetElement>
</mapping>
<!-- Identity:External Identifier -->
<!-- Identity:External Identifier -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Identity:External Identifier')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Identity:External
Identifier')].value</value>
</source>
<targetPath>$.metadata.relatedIdentifier[1]</targetPath>
<targetElement>identifier</targetElement>
</mapping>
<mapping>
<source>
<value type="constant">isAlternateIdentifier</value>
@ -40,19 +41,20 @@
<targetPath>$.metadata.relatedIdentifier[1]</targetPath>
<targetElement>type</targetElement>
</mapping>
<!-- Identity:RelatedPaper-->
<!-- Identity:RelatedPaper -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Identity:RelatedPaper')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Identity:RelatedPaper')].value
</value>
</source>
<targetPath>$.metadata.relatedIdentifier[2]</targetPath>
<targetElement>identifier</targetElement>
</mapping>
<mapping>
<source>
<value type="constant">isCitedBy</value>
@ -60,21 +62,23 @@
<targetPath>$.metadata.relatedIdentifier[2]</targetPath>
<targetElement>type</targetElement>
</mapping>
<!-- Coverage:Semantic Coverage-->
<!-- Coverage:Semantic Coverage -->
<mapping>
<source>
<value type="jsonPath" split=";">$.extras[?(@.key=='Coverage:Semantic Coverage')].value</value>
<value type="jsonPath" split=";">$.extras[?(@.key=='Coverage:Semantic
Coverage')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement>keywords[*]</targetElement>
<targetElement>keywords[*]</targetElement>
</mapping>
<!-- Coverage:TimeCoverage-->
<!-- Coverage:TimeCoverage -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Coverage:TimeCoverage')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Coverage:TimeCoverage')].value
</value>
</source>
<targetPath>$.metadata.dates[0]</targetPath>
<targetElement>start</targetElement>
@ -84,11 +88,12 @@
<regexp type="replace">
<target>[/ ]</target>
<replacement></replacement>
</regexp>
</regexp>
</mapping>
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Coverage:TimeCoverage')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Coverage:TimeCoverage')].value
</value>
</source>
<targetPath>$.metadata.dates[0]</targetPath>
<targetElement>end</targetElement>
@ -98,27 +103,29 @@
<regexp type="replace">
<target>[/ ]</target>
<replacement></replacement>
</regexp>
</regexp>
</mapping>
<mapping>
<source>
<value type="constant">Valid</value>
</source>
<targetPath>$.metadata.dates[0]</targetPath>
<targetElement>type</targetElement>
<targetElement>type</targetElement>
</mapping>
<!-- Coverage.SpatialCoverage -->
<!-- No example given -->
<!-- No example given -->
<!-- AccessMode:Accessibility -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='AccessMode:Accessibility')].value</value>
<value type="jsonPath">$.extras[?(@.key=='AccessMode:Accessibility')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">access_conditions</targetElement>
<targetElement append="true">access_conditions
</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>AccessMode.Accessibility : </replacement>
@ -128,14 +135,16 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- AccessMode:Availability -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='AccessMode:Availability')].value</value>
<value type="jsonPath">$.extras[?(@.key=='AccessMode:Availability')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">access_conditions</targetElement>
<targetElement append="true">access_conditions
</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>AccessMode.Availability : </replacement>
@ -145,14 +154,16 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- AccessMode:AccessibilityMode -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='AccessMode:AccessibilityMode')].value</value>
<value type="jsonPath">$.extras[?(@.key=='AccessMode:AccessibilityMode')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">access_conditions</targetElement>
<targetElement append="true">access_conditions
</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>AccessMode.AccessibilityMode : </replacement>
@ -167,7 +178,8 @@
<!-- TechnicalDetails:ProcessingDegree -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:ProcessingDegree')].value</value>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:ProcessingDegree')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -184,7 +196,8 @@
<!-- TechnicalDetails:ManifestationType -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:ManifestationType')].value</value>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:ManifestationType')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -201,16 +214,18 @@
<!-- TechnicalDetails:Language -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:Language')].value</value>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:Language')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement>language</targetElement>
</mapping>
<!-- TechnicalDetails:Size -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:Size')].value</value>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:Size')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -223,11 +238,12 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- TechnicalDetails:DiskSize -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:DiskSize')].value</value>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:DiskSize')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -240,11 +256,12 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- TechnicalDetails:Format -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:Format')].value</value>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:Format')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -257,11 +274,12 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- TechnicalDetails:FormatSchema -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:FormatSchema')].value</value>
<value type="jsonPath">$.extras[?(@.key=='TechnicalDetails:FormatSchema')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -279,7 +297,8 @@
<!-- DataProtection:PersonalData -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:PersonalData')].value</value>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:PersonalData')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -296,7 +315,8 @@
<!-- DataProtection:PersonalSensitiveData -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value</value>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -314,13 +334,15 @@
<!-- DataProtection:Consent of the data subject -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:Consent of the data subject')].value</value>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:Consent of the
data subject')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>DataProtection.Consent of the data subject : </replacement>
<replacement>DataProtection.Consent of the data subject :
</replacement>
</regexp>
<regexp type="replace">
<target>$</target>
@ -328,33 +350,41 @@
</regexp>
</mapping>
<!-- DataProtection:Consent obtained also covers the envisaged transfer of the personal data outside the EU -->
<!-- DataProtection:Consent obtained also covers the envisaged transfer
of the personal data outside the EU -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:Consent obtained also covers the envisaged transfer of the personal data outside the EU')].value</value>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:Consent
obtained also covers the envisaged transfer of the personal data
outside the EU')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>DataProtection.Consent obtained also covers the envisaged transfer of the personal data outside the EU : </replacement>
<replacement>DataProtection.Consent obtained also covers the
envisaged transfer of the personal data outside the EU :
</replacement>
</regexp>
<regexp type="replace">
<target>$</target>
<replacement>; </replacement>
</regexp>
</mapping>
<!-- DataProtection:Personal data was manifestly made public by the data subject -->
<!-- DataProtection:Personal data was manifestly made public by the data
subject -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:Personal data was manifestly made public by the data subject')].value</value>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:Personal data
was manifestly made public by the data subject')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>DataProtection.Personal data was manifestly made public by the data subject : </replacement>
<replacement>DataProtection.Personal data was manifestly made
public by the data subject : </replacement>
</regexp>
<regexp type="replace">
<target>$</target>
@ -365,13 +395,15 @@
<!-- DataProtection:DataProtectionDirective -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value</value>
<value type="jsonPath">$.extras[?(@.key=='DataProtection:PersonalSensitiveData')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>DataProtection.DataProtectionDirective : </replacement>
<replacement>DataProtection.DataProtectionDirective :
</replacement>
</regexp>
<regexp type="replace">
<target>$</target>
@ -383,7 +415,8 @@
<!-- Rights:IP/Copyrights -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:IP/Copyrights')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:IP/Copyrights')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -396,11 +429,12 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- Rights:Field/Scope of use -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:Field/Scope of use')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:Field/Scope of
use')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -413,12 +447,13 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- Rights:Basic rights -->
<!-- Rights:Basic rights -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:Basic rights')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:Basic rights')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -436,7 +471,8 @@
<!-- Rights:Restrictions on use -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:Restrictions on use')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:Restrictions on
use')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -450,11 +486,12 @@
</regexp>
</mapping>
<!-- Rights:Sublicense rights -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:Sublicense rights')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:Sublicense
rights')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -471,7 +508,8 @@
<!-- Rights:Territory of use -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:Territory of use')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:Territory of
use')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -488,7 +526,8 @@
<!-- Rights:License term -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:License term')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:License term')].value
</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -501,18 +540,20 @@
<replacement>; </replacement>
</regexp>
</mapping>
<!-- Rights:Requirement of non-disclosure (confidentiality mark) -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Rights:Requirement of non-disclosure (confidentiality mark)')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Rights:Requirement of
non-disclosure (confidentiality mark)')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
<regexp type="replace">
<target>^</target>
<replacement>Rights.Requirement of non-disclosure (confidentiality mark) : </replacement>
<replacement>Rights.Requirement of non-disclosure (confidentiality
mark) : </replacement>
</regexp>
<regexp type="replace">
<target>$</target>
@ -524,7 +565,8 @@
<!-- Attribution:Attribution requirements -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Attribution:Attribution requirements')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Attribution:Attribution
requirements')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -542,7 +584,8 @@
<!-- Attribution:Display requirements -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Attribution:Attribution requirements')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Attribution:Attribution
requirements')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>
@ -560,7 +603,8 @@
<!-- Attribution:Distribution requirements -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Attribution:Distribution requirements')].value</value>
<value type="jsonPath">$.extras[?(@.key=='Attribution:Distribution
requirements')].value</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement append="true">notes</targetElement>

120
src/test/resources/blue_cloud_deliverable.json Normal file
View File

@ -0,0 +1,120 @@
{
"rating": 0.0,
"license_title": "Creative Commons Attribution Share-Alike 4.0",
"maintainer": "Candela Leonardo",
"relationships_as_object": [],
"private": false,
"maintainer_email": "leonardo.candela@isti.cnr.it",
"num_tags": 1,
"id": "ae46038a-10e9-4e62-a6c6-758e71a3428f",
"metadata_created": "2021-02-25T09:55:42.319211",
"metadata_modified": "2021-02-25T15:54:50.597047",
"author": "Candela Leonardo",
"author_email": "leonardo.candela@isti.cnr.it",
"acquire_url": "",
"state": "active",
"version": "1",
"creator_user_id": "be67d795-9fc8-4a8d-a62d-3113b724f83c",
"type": "dataset",
"resources": [
{
"mimetype": null,
"cache_url": null,
"hash": "",
"description": "",
"name": "Deliverable",
"format": "PDF",
"url": "https://data-pre.d4science.net/RgA7",
"datastore_active": false,
"cache_last_updated": null,
"package_id": "ae46038a-10e9-4e62-a6c6-758e71a3428f",
"created": "2021-02-25T09:57:53.477077",
"state": "active",
"mimetype_inner": null,
"last_modified": null,
"position": 0,
"revision_id": "f8875fc2-e7c4-4949-a122-343a646f05e4",
"url_type": null,
"id": "e2363d7f-3c67-40bc-b6bb-9111bfcf063b",
"resource_type": null,
"size": null
}
],
"num_resources": 1,
"tags": [
{
"vocabulary_id": null,
"state": "active",
"display_name": "Text mining",
"id": "331ceb06-9ec3-4a10-a46a-5124f4dc19f2",
"name": "Text mining"
}
],
"groups": [],
"license_id": "CC-BY-SA-4.0",
"relationships_as_subject": [],
"organization": {
"description": "The preVRE organisation",
"created": "2019-05-02T12:41:26.149836",
"title": "preVRE",
"name": "prevre",
"is_organization": true,
"state": "active",
"image_url": "https://pre.d4science.org/image/layout_set_logo?img_id=12949941",
"revision_id": "a44aebb5-5f88-4d53-94d2-bfb17b8d340b",
"type": "organization",
"id": "46ce33da-dbc1-4116-b41d-eb961435a53d",
"approval_status": "approved"
},
"name": "a_sample_deliverable_4_zenodo",
"isopen": true,
"url": "",
"notes": "This is a sample deliverable created for testing the publish2Zenodo \r\nfacility",
"owner_org": "46ce33da-dbc1-4116-b41d-eb961435a53d",
"extras": [
{
"key": "Deliverable Author",
"value": "Rossi, Pietro, pietro.rossi@acme.org"
},
{
"key": "Deliverable Author",
"value": "Verdi, Luca, lica.verdi@amce.org"
},
{
"key": "Deliverable Author",
"value": "Bianchi, Elena"
},
{
"key": "Deliverable Contributor",
"value": "Ferrari, Paolo, paolo.ferrari@acme.org"
},
{
"key": "Deliverable Contributor",
"value": "Russo, Pietro, pietro.russo@amce.org"
},
{
"key": "Deliverable Contributor",
"value": "Giorgino, Giorgio"
},
{
"key": "Deliverable ID",
"value": "Dx.y"
},
{
"key": "Item URL",
"value": "https://data-pre.d4science.org/ctlg/preVRE/a_sample_deliverable_4_zenodo"
},
{
"key": "Publication Date",
"value": "2021-02-25 "
},
{
"key": "system:type",
"value": "Deliverable"
}
],
"license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
"ratings_count": 0,
"title": "A sample deliverable 4 Zenodo",
"revision_id": "cbd34661-8303-4e7b-8935-beb111ad1925"
}

70
src/test/resources/blue_cloud_deliverable.xml Normal file
View File

@ -0,0 +1,70 @@
<Resource version="0.4.x">
<ID>12345</ID>
<Type>GenericResource</Type>
<Scopes></Scopes>
<Profile>
<SecondaryType>Ckan-Zenodo-Mappings</SecondaryType>
<Name>Deliverable</Name>
<Description>Simple mappings tests</Description>
<Body>
<mappings>
<resourceFilters>
<filter>
<condition>$.resources[?(@.format)]</condition>
</filter>
</resourceFilters>
<mapping>
<source>
<value type="constant">other</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement>upload_type</targetElement>
</mapping>
<mapping>
<source> <!-- $.store.book[?(@.price < 10)] -->
<value type="jsonPath">$.extras[?(@.key=='Deliverable Author')].value</value>
</source>
<targetPath append="false" type="array">$.metadata.creators</targetPath>
<targetElement>name</targetElement>
</mapping>
<!-- Contributor -> {name : ... , type : 'other'} -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Deliverable Contributor')].value</value>
</source>
<targetPath append="false" type="array">$.metadata.contributors</targetPath>
<targetElement>name</targetElement>
<targetElement constant="Other">type</targetElement>
</mapping>
<mapping>
<source>
<value type="constant">862409</value>
</source>
<targetPath type="array">$.metadata.grants</targetPath>
<targetElement>id</targetElement>
</mapping>
<mapping>
<source>
<value type="constant">blue-cloud</value>
</source>
<targetPath type="array">$.metadata.communities</targetPath>
<targetElement>identifier</targetElement>
</mapping>
</mappings>
</Body>
</Profile>
</Resource>

30
src/test/resources/simple.xml Normal file
View File

@ -0,0 +1,30 @@
<Resource version="0.4.x">
<ID>12345</ID>
<Type>GenericResource</Type>
<Scopes></Scopes>
<Profile>
<SecondaryType>Ckan-Zenodo-Mappings</SecondaryType>
<Name>Deliverable</Name>
<Description>Simple mappings tests</Description>
<Body>
<mappings>
<resourceFilters>
<filter>
<condition>$.resources[?(@.format)]</condition>
</filter>
</resourceFilters>
<mapping>
<source>
<value type="constant">other</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement>upload_type</targetElement>
</mapping>
</mappings>
</Body>
</Profile>
</Resource>