gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

File upload management

This commit is contained in:
Fabio Sinibaldi 2019-12-10 17:06:52 +01:00
parent c52f65b228
commit b3865c791e
19 changed files with 486 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pom.xml
View File

@ -63,7 +63,12 @@
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-multipart</artifactId>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>

66
src/main/java/org/gcube/data/publishing/ckan2zenodo/LocalConfiguration.java Normal file
View File

@ -0,0 +1,66 @@
package org.gcube.data.publishing.ckan2zenodo;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LocalConfiguration {
public static class Configuration{
public static final String THREAD_POOL_SIZE="THREAD_POOL_SIZE";
}
public static final Map<String,Object> defaultConfigurationMap=new HashMap<String,Object>();
static {
defaultConfigurationMap.put(Configuration.THREAD_POOL_SIZE, "5");
}
private static LocalConfiguration instance=null;
@Synchronized
public static final LocalConfiguration get() {
if(instance==null)
instance=new LocalConfiguration();
return instance;
}
public static String getProperty(String property) {
try{
return (String) get().props.getOrDefault(property, defaultConfigurationMap.get(property));
}catch(Throwable t) {
log.warn("Unable to get configuration property "+property,t);
return defaultConfigurationMap.get(property)+"";
}
}
public Map<String,String> asMap(){
HashMap<String,String> toReturn=new HashMap<>();
for(Object key :props.keySet())
toReturn.put(key+"", (String)props.getOrDefault(key, defaultConfigurationMap.get(key)));
return toReturn;
}
//***************** INSTANCE
Properties props;
private LocalConfiguration() {
props=new Properties();
try{
props.load(this.getClass().getResourceAsStream("config.properties"));
}catch(Exception e) {
log.warn("********************** UNABLE TO LOAD PROPERTIES **********************",e);
log.debug("Reverting to defaults : "+defaultConfigurationMap);
}
}
}

11
src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java
View File

@ -1,18 +1,11 @@
package org.gcube.data.publishing.ckan2zenodo;
import java.util.ArrayList;
import java.util.HashMap;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.data.publishing.ckan2zenodo.commons.IS;
import org.gcube.data.publishing.ckan2zenodo.model.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Regexp;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import lombok.Synchronized;
@ -32,10 +25,10 @@ public class TransformerManager {
return builder;
}
public Transformer getByProfile(String profile) throws Exception {
public Translator getByProfile(String profile) throws Exception {
for(GenericResource r: IS.queryForGenericResources("Ckan-Zenodo-Mappings")){
if (r.profile().name().equals(profile))
return new Transformer(IS.readMappings(r));
return new Translator(IS.readMappings(r),IS.readResourceFilters(r));
}
throw new Exception("No transformer found for profile "+profile);
}

59
src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java → src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java
View File

@ -1,15 +1,20 @@
package org.gcube.data.publishing.ckan2zenodo;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.gcube.data.publishing.ckan2zenodo.commons.Parsing;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter.Filter;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Contributor;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.AccessRights;
@ -23,15 +28,26 @@ import com.jayway.jsonpath.JsonPath;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
@Slf4j
@RequiredArgsConstructor
public class Transformer {
public class Translator {
@NonNull
private List<Mapping> mappings;
@NonNull
private ResourceFilter resourceFilter;
public Translator() {
this(Collections.EMPTY_LIST,new ResourceFilter(Collections.EMPTY_LIST));
}
public Translator(List<Mapping> mappings) {
this(mappings,new ResourceFilter(Collections.EMPTY_LIST));
}
public ZenodoDeposition transform(CkanItemDescriptor toTransform, ZenodoDeposition deposition) throws TransformationException {
log.debug("Transforming "+toTransform+". Existing Deposition is : "+deposition);
@ -144,9 +160,17 @@ public class Transformer {
// apply value mappings
resultingValue =mapping.getValueMapping().getOrDefault(sourceValue, resultingValue);
// check if targetPath exists
List<String> targetElementFound=targetCtx.read(mapping.getTargetPath());
if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null) {
// targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing"));
Parsing.addElement(targetCtx, mapping.getTargetPath());
}
// apply resulting value
targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement(),resultingValue);
targetCtx=targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement(),resultingValue);
}
}catch(Throwable t) {
throw new TransformationException("Exception while applying "+mapping,t);
@ -155,9 +179,30 @@ public class Transformer {
return mapper.readValue(targetCtx.jsonString(), ZenodoDeposition.class);
}catch(Throwable t) {
log.error("Unable to transform "+source+" using previous "+target,t);
log.error("Unable to translate "+source+" using previous "+target,t);
throw new TransformationException("Unable to translate "+source.getName(),t);
}
}
public List<CkanResource> filterResources(CkanItemDescriptor source) throws TransformationException{
try {
ObjectMapper mapper=Parsing.getMapper();
DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent());
ArrayList<CkanResource> toReturn=new ArrayList<>();
for(Filter f:resourceFilter.getFilters()) {
JSONArray filtered=sourceCtx.read(f.getConditions().get(0));
for(Object obj:filtered) {
Map<String,String> map=(Map<String, String>) obj;
toReturn.add(mapper.readValue((new JSONObject(map)).toJSONString(), CkanResource.class));
}
}
return toReturn;
}catch(Throwable t) {
log.error("Unable to filter resources. ",t);
throw new TransformationException("Unable to filter "+source.getName()+" resources",t);
}
}
}

32
src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/FileUploaderManager.java Normal file
View File

@ -0,0 +1,32 @@
package org.gcube.data.publishing.ckan2zenodo.clients;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import javax.ws.rs.core.Response;
import org.gcube.data.publishing.ckan2zenodo.LocalConfiguration;
import org.gcube.data.publishing.ckan2zenodo.LocalConfiguration.Configuration;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
public class FileUploaderManager {
static ExecutorService service=null;
static {
service=Executors.newFixedThreadPool(Integer.parseInt(LocalConfiguration.getProperty(Configuration.THREAD_POOL_SIZE)));
}
public FutureTask<ZenodoDeposition> createTask(Callable<ZenodoDeposition> call ){
return new FutureTask<ZenodoDeposition> (call);
}
public static Future<Response> submit(Callable<Response> call){
return service.submit(call);
}
}

115
src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java
View File

@ -1,6 +1,12 @@
package org.gcube.data.publishing.ckan2zenodo.clients;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
@ -12,14 +18,16 @@ import org.gcube.data.publishing.ckan2zenodo.Fixer;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.media.multipart.file.FileDataBodyPart;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
@ -35,6 +43,8 @@ public class Zenodo {
private static final String DEPOSITION_BASE_URL="deposit/depositions";
private static final String PUBLISH_URL_PRE="deposit/depositions";
private static final String PUBLISH_URL_POST="actions/publish";
private static final String ACCESS_TOKEN="access_token";
@ -54,9 +64,11 @@ public class Zenodo {
Client client;
private synchronized Client getWebClient() {
if(client==null)
if(client==null) {
client = ClientBuilder.newClient()
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
client.register(MultiPartFeature.class);
}
return client;
}
@ -83,37 +95,110 @@ public class Zenodo {
}
}
public ZenodoDeposition updateMetadata(Integer depositionId,DepositionMetadata meta) throws ZenodoException, JsonProcessingException {
public ZenodoDeposition updateMetadata(ZenodoDeposition dep) throws ZenodoException, JsonProcessingException {
return updateMetadata(dep.getId(), dep.getMetadata());
}
public FileDeposition uploadFile(ZenodoDeposition dep, String toUploadName,String urlString) throws ZenodoException {
Callable<Response> call=new Callable<Response>() {
@Override
public Response call() throws Exception {
File temp=null;
try {
log.debug("Downloading "+urlString);
//Download locally into temp
URL url=new URL(urlString);
temp=File.createTempFile("zenodo_", ".tmp");
long size=Files.copy(url.openStream(), temp.toPath(),StandardCopyOption.REPLACE_EXISTING);
//upload
FormDataMultiPart multi=new FormDataMultiPart();
FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file",
temp,MediaType.APPLICATION_OCTET_STREAM_TYPE);
multi.field("name", toUploadName);
multi.bodyPart(fileDataBodyPart);
log.debug("Starting transfer of "+toUploadName+" ("+urlString+") into "+dep.getId());
Response toReturn=getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(dep.getId()+"").path("files").
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.post(Entity.entity(multi,multi.getMediaType()));
log.debug("DONE.");
return toReturn;
}catch(Throwable e) {
throw new ZenodoException("Unable to transfer file "+toUploadName+" url : "+urlString,e);
}finally {
//finally delete temp
if(temp!=null) Files.deleteIfExists(temp.toPath());
}
}
};
log.debug("Submitting request to upload "+urlString+" to Manager");
Future<Response> resp=FileUploaderManager.submit(call);
try {
return check(resp.get(),FileDeposition.class);
}catch(ZenodoException z) {
throw z;
}catch(Throwable t) {
throw new ZenodoException(t.getMessage(),t);
}
//return
}
private ZenodoDeposition updateMetadata(Integer depositionId,DepositionMetadata meta) throws ZenodoException, JsonProcessingException {
String serialized="{\"metadata\":"+Fixer.fixIncoming(mapper.writeValueAsString(meta))+"}";
try {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(depositionId+"").
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.put(Entity.json(serialized));
return check(resp,ZenodoDeposition.class);
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(depositionId+"").
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.put(Entity.json(serialized));
return check(resp,ZenodoDeposition.class);
}catch(Throwable t) {
log.debug("Error while tryin to update "+serialized);
throw t;
}
}
public void deleteDeposition(String depositionId) throws ZenodoException {
public void deleteDeposition(Integer depositionId) throws ZenodoException {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(depositionId).
path(DEPOSITION_BASE_URL).path(depositionId+"").
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.delete();
check(resp,null);
}
public ZenodoDeposition createNew() throws ZenodoException {
Response resp = getWebClient().target(credentials.getBaseUrl()).
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.post(Entity.json("{}"));
return check(resp,ZenodoDeposition.class);
}
public ZenodoDeposition publish(ZenodoDeposition dep) throws ZenodoException{
return publish(dep.getId());
}
private ZenodoDeposition publish(Integer depositionId) throws ZenodoException{
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(PUBLISH_URL_PRE).
path(depositionId+"").
path(PUBLISH_URL_POST).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.post(Entity.json("{}"));
return check(resp,ZenodoDeposition.class);
}
}

32
src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java
View File

@ -9,10 +9,11 @@ import java.util.List;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.data.publishing.ckan2zenodo.model.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Source;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.w3c.dom.Element;
@ -102,4 +103,29 @@ public class IS {
throw new ConfigurationException("Invaild mapping resource "+res.id()+" name : "+res.profile().name(),t);
}
}
public static ResourceFilter readResourceFilters(GenericResource res) throws ConfigurationException{
try{
ArrayList<ResourceFilter.Filter> filtersList=new ArrayList<>();
Element root=res.profile().body();
NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter");
for(int i=0;i<filters.getLength();i++) {
Element filter=(Element) filters.item(i);
ArrayList<String> conditions=new ArrayList<>();
NodeList conditionNodes=filter.getElementsByTagName("condition");
for(int j=0;j<conditionNodes.getLength();j++) {
conditions.add(conditionNodes.item(j).getTextContent());
}
if(!conditions.isEmpty())
filtersList.add(new ResourceFilter.Filter(conditions));
}
if(filtersList.isEmpty())
throw new Exception("No Filter actually declared.");
return new ResourceFilter(filtersList);
}catch(Throwable t) {
log.debug("Error while parsing filters from resource "+res.id()+" name : "+res.profile().name(),t);
throw new ConfigurationException("Invalid resource filters declared in "+res.id()+" name : "+res.profile().name(),t);
}
}
}

34
src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/Parsing.java
View File

@ -1,10 +1,19 @@
package org.gcube.data.publishing.ckan2zenodo.commons;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
public class Parsing {
@ -25,4 +34,29 @@ public class Parsing {
return mapper;
}
public static final DocumentContext addElement(DocumentContext ctx,String path) throws ConfigurationException {
JsonPath jPath=JsonPath.compile(path);
if(jPath.isDefinite()) {
String parent=path.substring(0,path.lastIndexOf("."));
List<String> found=ctx.read(parent);
if(found==null || found.size()==0 || found.get(0)==null) {
//missing parent, use recursion
addElement(ctx,parent);
}
// found parent, adding element
String element=path.substring(path.lastIndexOf(".")+1);
Object value=new HashMap<String,String>();
if(element.contains("[")) {
value=new ArrayList<Object>(Collections.singletonList(value));
element=element.substring(0,element.indexOf("["));
}
ctx.put(parent, element, value);
return ctx;
}else throw new ConfigurationException("Unable to initialize non-definite path : "+path);
}
}

32
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanResource.java Normal file
View File

@ -0,0 +1,32 @@
package org.gcube.data.publishing.ckan2zenodo.model;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
public class CkanResource{
private String mimetype;
private String cache_url;
private String hash;
private String description;
private String name;
private String format;
private String url;
private String datastore_active;
private String cache_last_updated;
private String package_id;
private String created;
private String state;
private String mimetype_inner;
private String last_modified;
private String position;
private String revision_id;
private String url_type;
private String id;
private String resource_type;
private String size;
}

2
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java → src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java
View File

@ -1,4 +1,4 @@
package org.gcube.data.publishing.ckan2zenodo.model;
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import java.util.HashMap;
import java.util.List;

26
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java Normal file
View File

@ -0,0 +1,26 @@
package org.gcube.data.publishing.ckan2zenodo.model.parsing;
import java.util.List;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
@RequiredArgsConstructor
@ToString
@Getter
public class ResourceFilter {
@RequiredArgsConstructor
@ToString
@Getter
public static class Filter {
@NonNull
private List<String> conditions;
}
@NonNull
private List<Filter> filters;
}

1
src/main/resources/config.properties Normal file
View File

@ -0,0 +1 @@
THREAD_POOL_SIZE=5

51
src/test/java/org/gcube/tests/ParsingTests.java
View File

@ -1,15 +1,27 @@
package org.gcube.tests;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import org.gcube.data.publishing.ckan2zenodo.Fixer;
import org.gcube.data.publishing.ckan2zenodo.commons.Parsing;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
public class ParsingTests {
@ -18,8 +30,8 @@ public class ParsingTests {
public static void init () {
mapper=TestCommons.getMapper();
}
@Test
public void DateTest() throws Exception {
for(String d:new String[] {"2019-11-29T14:54:42.542142","2016-06-15T16:10:03.319363+00:00","2019-11-30","2019-11-29T17:01:31.000160+0000"}) {
@ -28,7 +40,7 @@ public class ParsingTests {
System.out.println("--");
}
}
@Test
public void fullCircleFromIncoming() throws JsonParseException, JsonMappingException, IOException {
String s="{\n" +
@ -85,9 +97,38 @@ public class ParsingTests {
System.out.println("FIXED JSON ");
String fixed=Fixer.fixSending(json);
System.out.println(fixed);
}
@Test
public void miscTests() throws IOException, ConfigurationException {
// ObjectMapper mapper=Parsing.getMapper();
// mapper.setSerializationInclusion(Include.ALWAYS);
ZenodoDeposition dep=new ZenodoDeposition();
dep.setMetadata(new DepositionMetadata());
DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(mapper.writeValueAsString(dep));
Parsing.addElement(sourceCtx,"$.metadata.creators[0]");
// addElement(sourceCtx,"$.metadata.creators[0].name",true);
sourceCtx.put("$.metadata.creators[0]","name", "myName");
System.out.println("JSON : "+sourceCtx.jsonString());
Assert.assertTrue(sourceCtx.jsonString().contains("creators"));
Assert.assertTrue(sourceCtx.jsonString().contains("myName"));
ZenodoDeposition parsed=mapper.readValue(sourceCtx.jsonString(), ZenodoDeposition.class);
Assert.assertEquals(parsed.getMetadata().getCreators().get(0).getName(), "myName");
}
@Test
public void read() {
DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse("{\"metadata\":{\"creators\":[{\"name\":\"\"}]}}");
List<Object> found=sourceCtx.read("$['metadata']['creators'][0]");
for(Object s: found) {
System.out.println(s);
}
}
}

6
src/test/java/org/gcube/tests/TestCommons.java
View File

@ -5,7 +5,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import org.gcube.data.publishing.ckan2zenodo.Fixer;
import org.gcube.data.publishing.ckan2zenodo.Transformer;
import org.gcube.data.publishing.ckan2zenodo.Translator;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
@ -41,13 +41,13 @@ public class TestCommons {
return mapper;
}
static final ZenodoDeposition readAndTransform(String jsonFile, Transformer transformer,ZenodoDeposition...depositions) throws Exception {
static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,ZenodoDeposition...depositions) throws Exception {
try{
String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile));
CkanItemDescriptor desc=new CkanItemDescriptor(json);
System.out.println("Going to transform : "+desc.getContent());
System.out.println("Result : ");
ZenodoDeposition dep=transformer.transform(desc, depositions!=null?depositions[0]:null);
ZenodoDeposition dep=transformer.transform(desc, (depositions!=null&&depositions.length>0)?depositions[0]:null);
System.out.println(dep);
System.out.println("As JSON : ");
System.out.println(Fixer.fixSending(getMapper().writeValueAsString(dep)));

12
src/test/java/org/gcube/tests/TransformationTests.java
View File

@ -1,15 +1,9 @@
package org.gcube.tests;
import java.io.IOException;
import java.util.Collections;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.Resources;
import org.gcube.data.publishing.ckan2zenodo.Fixer;
import org.gcube.data.publishing.ckan2zenodo.Transformer;
import org.gcube.data.publishing.ckan2zenodo.Translator;
import org.gcube.data.publishing.ckan2zenodo.commons.IS;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.junit.BeforeClass;
import org.junit.Test;
@ -27,7 +21,7 @@ public class TransformationTests {
@Test
public void transform() throws Exception {
Transformer basic=new Transformer(Collections.EMPTY_LIST);
Translator basic=new Translator();
TestCommons.readAndTransform("/simpleItem.json",basic);
TestCommons.readAndTransform("/FSKXModel.json",basic);
TestCommons.readAndTransform("/ResearchObject.json",basic);
@ -38,7 +32,7 @@ public class TransformationTests {
@Test
public void transformWithMappings() throws Exception {
GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml"));
TestCommons.readAndTransform("/ResearchObject.json", new Transformer(IS.readMappings(res)));
TestCommons.readAndTransform("/ResearchObject.json", new Translator(IS.readMappings(res)));
}

26
src/test/java/org/gcube/tests/ZenodoTests.java
View File

@ -1,17 +1,16 @@
package org.gcube.tests;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Date;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.Resources;
import org.gcube.data.publishing.ckan2zenodo.Transformer;
import org.gcube.data.publishing.ckan2zenodo.Translator;
import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
import org.gcube.data.publishing.ckan2zenodo.commons.IS;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
@ -19,6 +18,7 @@ import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Creator;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.AccessRights;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.UploadType;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.junit.BeforeClass;
import org.junit.Test;
@ -60,7 +60,7 @@ public class ZenodoTests {
Arrays.asList(new Creator("simpleMan")),
"Simple description",
AccessRights.open));
System.out.println(z.updateMetadata(dep.getId(),dep.getMetadata()));
System.out.println(z.updateMetadata(dep));
}
@ -70,9 +70,23 @@ public class ZenodoTests {
GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml"));
ZenodoDeposition dep=z.createNew();
dep=TestCommons.readAndTransform("/ResearchObject.json", new Transformer(IS.readMappings(res)),dep);
Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res));
dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep);
dep=z.updateMetadata(dep);
System.out.println("Resulting Deposition with metadata : ");
System.out.println();
String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream("/ResearchObject.json"));
CkanItemDescriptor desc=new CkanItemDescriptor(json);
System.out.println(z.updateMetadata(dep.getId(), dep.getMetadata()));
for(CkanResource cRes:tran.filterResources(desc)) {
FileDeposition file=z.uploadFile(dep, cRes.getName(), cRes.getUrl());
System.out.println("Published "+file);
}
dep=z.publish(dep);
System.out.println();
}
}

22
src/test/resources/ResearchObject.json
View File

@ -38,6 +38,28 @@
"id": "8caa68cb-6880-4683-9849-1aad02d8cc11",
"resource_type": null,
"size": null
},
{
"mimetype": null,
"cache_url": null,
"hash": "",
"description": "Curated by: World Health Organization (WHO)\r\n\r\nAs part of their core goal for better health information worldwide, the World Health Organization makes their data on global health publicly available through the Global Health Observatory (GHO). The GHO acts as a portal with which to access and analyze health situations and important themes.\r\n\r\nThe various data sets are organized according to themes, such as mortality, health systems, communicable and non-communicable diseases, medicines and vaccines, health risks, and so on. The WHO\u2019s health statistics are to go-to source for global health information and is also used in the work of the US Centers for Disease Control and Prevention.",
"name": "Some test csv",
"format": "CSV",
"url": "https://data.d4science.net/7CQd",
"datastore_active": false,
"cache_last_updated": null,
"package_id": "54938132-299e-46f7-a84a-494fcd75ad8f",
"created": "2019-12-03T11:20:02.594436",
"state": "active",
"mimetype_inner": null,
"last_modified": null,
"position": 0,
"revision_id": "9dd5e97c-8c39-4198-a34e-c56ae0631d17",
"url_type": null,
"id": "8caa68cb-6880-4683-9849-1aad02d8cc22",
"resource_type": null,
"size": null
}],
"num_resources": 1,
"tags": [

13
src/test/resources/ResearchObject.xml
View File

@ -7,12 +7,19 @@
<Name>ResearchObject</Name>
<Description>Simple mappings tests</Description>
<Body>
<resourceFilters>
<filter>
<condition>$.resources[?(@.format=='CSV')]</condition>
</filter>
</resourceFilters>
<mappings>
<mapping>
<source type="constant">
<value>dataset</value>
</source>
</source>
<targetPath>$.metadata</targetPath>
<targetElement>upload_type</targetElement>
</mapping>
@ -22,7 +29,7 @@
<source type="jsonPath">
<value>$.extras[?(@.key=='Author')].value</value>
</source>
<targetPath>$.metadata.contributors[0]</targetPath>
<targetPath>$.metadata.creators[0]</targetPath>
<targetElement>name</targetElement>
<regexp type="extract">
<target>([A-Za-z]*, [A-Za-z]*)(?=,)</target>
@ -32,7 +39,7 @@
<source type="constant">
<value>Producer</value>
</source>
<targetPath>$.metadata.contributors[0]</targetPath>
<targetPath>$.metadata.creators[0]</targetPath>
<targetElement>type</targetElement>
</mapping>
<!-- <mapping>

2
src/test/resources/logback.xml
View File

@ -17,7 +17,7 @@
<root level="WARN">
<root level="DEBUG">
<appender-ref ref="STDOUT" />
</root>