gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added Multiple source options in mappings

This commit is contained in:
Fabio Sinibaldi 2020-01-13 16:49:50 +01:00
parent 71de9fbe9b
commit 1557c39f32
8 changed files with 116 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java
View File

@ -12,8 +12,9 @@ import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter.Filter;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Contributor;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
@ -121,17 +122,28 @@ public class Translator {
try {
// extract source
List<String> sourceValues=new ArrayList<>();
switch(mapping.getSource().getType()) {
case constant : {
sourceValues.add(mapping.getSource().getValue());
break;
for(Value v: mapping.getSource().getValues()) {
switch(v.getType()) {
case constant : {
sourceValues.add(v.getValue());
break;
}
case jsonPath : {
sourceValues.addAll(sourceCtx.read(v.getValue()));
break;
}
}
if(!sourceValues.isEmpty()) break;
}
case jsonPath : {
sourceValues.addAll(sourceCtx.read(mapping.getSource().getValue()));
break;
}
}
for(String sourceValue:sourceValues) {
String resultingValue=sourceValue;

12
src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java
View File

@ -11,9 +11,10 @@ import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source.Value;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.w3c.dom.Element;
@ -61,8 +62,13 @@ public class IS {
for(int i = 0; i<mappings.getLength();i++) {
Element mapping=(Element) mappings.item(i);
Element sourceElement=(Element) mapping.getElementsByTagName("source").item(0);
Source source=new Source(Source.Type.valueOf(sourceElement.getAttribute("type")),
sourceElement.getElementsByTagName("value").item(0).getTextContent());
NodeList valuesNodeList=sourceElement.getElementsByTagName("value");
Source source=new Source();
for(int j=0;j<valuesNodeList.getLength();j++) {
Element valueElement=(Element) valuesNodeList.item(j);
source.getValues().add(new Value(Value.Type.valueOf(valueElement.getAttribute("type")),
valueElement.getTextContent()));
}
String targetPath=mapping.getElementsByTagName("targetPath").item(0).getTextContent();

2
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanItemDescriptor.java
View File

@ -13,10 +13,12 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import net.minidev.json.JSONObject;
@RequiredArgsConstructor
@NoArgsConstructor
@ToString
public class CkanItemDescriptor {
static private ObjectMapper mapper;

17
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanRelatedIdentifier.java
View File

@ -7,7 +7,7 @@ import lombok.RequiredArgsConstructor;
import lombok.ToString;
@Getter
@RequiredArgsConstructor
//@RequiredArgsConstructor
@NoArgsConstructor
@ToString
public class CkanRelatedIdentifier {
@ -16,14 +16,23 @@ public class CkanRelatedIdentifier {
return new CkanRelatedIdentifier("URL", "isReferencedBy", doi);
}
@NonNull
// @NonNull
private String relatedIdentifierType;
@NonNull
// @NonNull
private String relationType;
@NonNull
// @NonNull
private String link;
public Integer getZenodoId() {
return Integer.parseInt(link.substring(link.lastIndexOf(".")+1));
}
public CkanRelatedIdentifier(String relatedIdentifierType, String relationType, String link) {
super();
this.relatedIdentifierType = relatedIdentifierType;
this.relationType = relationType;
this.link = link;
}
}

22
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java
View File

@ -1,5 +1,6 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -28,17 +29,24 @@ public class Mapping {
@Setter
private String replacement;
}
@RequiredArgsConstructor
@Getter
@ToString
public static class Source{
public static enum Type{
jsonPath,constant
@RequiredArgsConstructor
@Getter
@ToString
public static class Value{
public static enum Type{
jsonPath,constant
}
@NonNull
private Type type;
@NonNull
private String value;
}
@NonNull
private Type type;
@NonNull
private String value;
private List<Value> values=new ArrayList<>();
}
@NonNull

39
src/test/java/org/gcube/tests/ParsingTests.java
View File

@ -142,9 +142,46 @@ public class ParsingTests {
String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream("/ResearchObject.json"));
CkanItemDescriptor desc=new CkanItemDescriptor(json);
Assert.assertNull(desc.getZenodoDoi());
desc.setZenodoDoi(new CkanRelatedIdentifier("URL", "isReferencedBy", "http://some.dumb.link.com"));
desc.setZenodoDoi(new CkanRelatedIdentifier("URL", "isReferencedBy", "http://10.5281/zenodo.1000001"));
CkanRelatedIdentifier id=desc.getZenodoDoi();
Assert.assertNotNull(id);
System.out.println(desc.getContent());
}
@Test
public void guiTest() throws JsonParseException, JsonMappingException, IOException {
String toParse="{\n" +
" \"metadata\":\n" +
" {\n" +
" \"title\": \"Crop Parameters\",\n" +
" \"description\": \"Parameter definitions for various crops. Can be used as input for a crop \\r\\nsimulation model, such as WOFOST.\",\n" +
" \"access_right\": \"open\",\n" +
" \"license\": \"AFL-3.0\",\n" +
" \"keywords\": [\"crop parameters\", \"crop simulation\", \"input files\", \"wofost\"],\n" +
" \"related_identifiers\": [\n" +
" {\n" +
" \"identifier\": \"http://data.d4science.org/ctlg/AgroClimaticModelling/crop_parameters\",\n" +
" \"relation\": \"compiles\"\n" +
" }],\n" +
" \"contributors\": [\n" +
" {\n" +
" \"type\": \"Producer\",\n" +
" \"name\": \"Knapen Rob\"\n" +
" },\n" +
" {\n" +
" \"type\": \"DataCurator\",\n" +
" \"name\": \"Knapen Rob\"\n" +
" }],\n" +
" \"version\": \"1\",\n" +
" \"upload_type\": \"dataset\",\n" +
" \"creators\": [\n" +
" {\n" +
" \"name\": \"Knapen Rob\",\n" +
" \"type\": \"Producer\"\n" +
" }]\n" +
" }\n" +
"}";
ZenodoDeposition dep=Parsing.getMapper().readValue(toParse,ZenodoDeposition.class);
System.out.println(dep);
}
}

9
src/test/java/org/gcube/tests/TransformationTests.java
View File

@ -24,16 +24,17 @@ public class TransformationTests {
Translator basic=new Translator();
TestCommons.readAndTransform("/simpleItem.json",basic);
TestCommons.readAndTransform("/FSKXModel.json",basic);
TestCommons.readAndTransform("/ResearchObject.json",basic);
TestCommons.readAndTransform("/ResearchObject.json",basic);
}
@Test
public void transformWithMappings() throws Exception {
GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml"));
TestCommons.readAndTransform("/ResearchObject.json", new Translator(IS.readMappings(res)));
GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml"));
Translator t=new Translator(IS.readMappings(res));
// TestCommons.readAndTransform("/ResearchObject.json", t);
TestCommons.readAndTransform("/crop_parameters.json",t);
}

21
src/test/resources/ResearchObject.xml
View File

@ -17,8 +17,8 @@
<mappings>
<mapping>
<source type="constant">
<value>dataset</value>
<source>
<value type="constant">dataset</value>
</source>
<targetPath>$.metadata</targetPath>
<targetElement>upload_type</targetElement>
@ -26,22 +26,23 @@
<mapping>
<source type="jsonPath">
<value>$.extras[?(@.key=='Author')].value</value>
<source>
<value type="jsonPath">$.extras[?(@.key=='Author')].value</value>
<value type="jsonPath">$.author</value>
</source>
<targetPath>$.metadata.creators[0]</targetPath>
<targetElement>name</targetElement>
<regexp type="extract">
<!-- <regexp type="extract">
<target>([A-Za-z]*, [A-Za-z]*)(?=,)</target>
</regexp>
</regexp> -->
</mapping>
<mapping>
<source type="constant">
<value>Producer</value>
<!-- <mapping>
<source>
<value type="constant">Producer</value>
</source>
<targetPath>$.metadata.creators[0]</targetPath>
<targetElement>type</targetElement>
</mapping>
</mapping> -->
<!-- <mapping>
<source type="jsonPath">
<value>$.extras[?(@.key=='Author')].value</value>