From b3865c791e4aba224c3c2a5104f5fe8e9890b5b7 Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Tue, 10 Dec 2019 17:06:52 +0100 Subject: [PATCH] File upload management --- pom.xml | 7 +- .../ckan2zenodo/LocalConfiguration.java | 66 ++++++++++ .../ckan2zenodo/TransformerManager.java | 11 +- .../{Transformer.java => Translator.java} | 59 +++++++-- .../clients/FileUploaderManager.java | 32 +++++ .../ckan2zenodo/clients/Zenodo.java | 115 +++++++++++++++--- .../publishing/ckan2zenodo/commons/IS.java | 32 ++++- .../ckan2zenodo/commons/Parsing.java | 34 ++++++ .../ckan2zenodo/model/CkanResource.java | 32 +++++ .../model/{ => parsing}/Mapping.java | 2 +- .../model/parsing/ResourceFilter.java | 26 ++++ src/main/resources/config.properties | 1 + .../java/org/gcube/tests/ParsingTests.java | 51 +++++++- .../java/org/gcube/tests/TestCommons.java | 6 +- .../org/gcube/tests/TransformationTests.java | 12 +- .../java/org/gcube/tests/ZenodoTests.java | 26 +++- src/test/resources/ResearchObject.json | 22 ++++ src/test/resources/ResearchObject.xml | 13 +- src/test/resources/logback.xml | 2 +- 19 files changed, 486 insertions(+), 63 deletions(-) create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/LocalConfiguration.java rename src/main/java/org/gcube/data/publishing/ckan2zenodo/{Transformer.java => Translator.java} (69%) create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/FileUploaderManager.java create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanResource.java rename src/main/java/org/gcube/data/publishing/ckan2zenodo/model/{ => parsing}/Mapping.java (93%) create mode 100644 src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java create mode 100644 src/main/resources/config.properties diff --git a/pom.xml b/pom.xml index f12b25d..c751ca0 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,12 @@ org.glassfish.jersey.media jersey-media-json-jackson - + + + org.glassfish.jersey.media + jersey-media-multipart + + com.jayway.jsonpath json-path diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/LocalConfiguration.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/LocalConfiguration.java new file mode 100644 index 0000000..8105cb0 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/LocalConfiguration.java @@ -0,0 +1,66 @@ +package org.gcube.data.publishing.ckan2zenodo; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + + +import lombok.Synchronized; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class LocalConfiguration { + + + + public static class Configuration{ + + public static final String THREAD_POOL_SIZE="THREAD_POOL_SIZE"; + } + public static final Map defaultConfigurationMap=new HashMap(); + + static { + defaultConfigurationMap.put(Configuration.THREAD_POOL_SIZE, "5"); + + } + private static LocalConfiguration instance=null; + + @Synchronized + public static final LocalConfiguration get() { + if(instance==null) + instance=new LocalConfiguration(); + return instance; + } + + public static String getProperty(String property) { + try{ + return (String) get().props.getOrDefault(property, defaultConfigurationMap.get(property)); + }catch(Throwable t) { + log.warn("Unable to get configuration property "+property,t); + return defaultConfigurationMap.get(property)+""; + } + } + + public Map asMap(){ + HashMap toReturn=new HashMap<>(); + for(Object key :props.keySet()) + toReturn.put(key+"", (String)props.getOrDefault(key, defaultConfigurationMap.get(key))); + return toReturn; + } + + //***************** INSTANCE + + Properties props; + + private LocalConfiguration() { + props=new Properties(); + try{ + props.load(this.getClass().getResourceAsStream("config.properties")); + }catch(Exception e) { + log.warn("********************** UNABLE TO LOAD PROPERTIES **********************",e); + log.debug("Reverting to defaults : "+defaultConfigurationMap); + } + } + + +} \ No newline at end of file diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java index e5f5e1d..16a9851 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java @@ -1,18 +1,11 @@ package org.gcube.data.publishing.ckan2zenodo; -import java.util.ArrayList; -import java.util.HashMap; - import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.data.publishing.ckan2zenodo.commons.IS; -import org.gcube.data.publishing.ckan2zenodo.model.Mapping; -import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Regexp; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; import lombok.Synchronized; @@ -32,10 +25,10 @@ public class TransformerManager { return builder; } - public Transformer getByProfile(String profile) throws Exception { + public Translator getByProfile(String profile) throws Exception { for(GenericResource r: IS.queryForGenericResources("Ckan-Zenodo-Mappings")){ if (r.profile().name().equals(profile)) - return new Transformer(IS.readMappings(r)); + return new Translator(IS.readMappings(r),IS.readResourceFilters(r)); } throw new Exception("No transformer found for profile "+profile); } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java similarity index 69% rename from src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java rename to src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java index cf0ada5..3ea3c2b 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/Translator.java @@ -1,15 +1,20 @@ package org.gcube.data.publishing.ckan2zenodo; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.gcube.data.publishing.ckan2zenodo.commons.Parsing; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; -import org.gcube.data.publishing.ckan2zenodo.model.Mapping; -import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Regexp; +import org.gcube.data.publishing.ckan2zenodo.model.CkanResource; import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter.Filter; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Contributor; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.AccessRights; @@ -23,15 +28,26 @@ import com.jayway.jsonpath.JsonPath; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import net.minidev.json.JSONArray; +import net.minidev.json.JSONObject; @Slf4j @RequiredArgsConstructor -public class Transformer { +public class Translator { @NonNull private List mappings; + + @NonNull + private ResourceFilter resourceFilter; - + public Translator() { + this(Collections.EMPTY_LIST,new ResourceFilter(Collections.EMPTY_LIST)); + } + + public Translator(List mappings) { + this(mappings,new ResourceFilter(Collections.EMPTY_LIST)); + } public ZenodoDeposition transform(CkanItemDescriptor toTransform, ZenodoDeposition deposition) throws TransformationException { log.debug("Transforming "+toTransform+". Existing Deposition is : "+deposition); @@ -144,9 +160,17 @@ public class Transformer { // apply value mappings resultingValue =mapping.getValueMapping().getOrDefault(sourceValue, resultingValue); - + + // check if targetPath exists + List targetElementFound=targetCtx.read(mapping.getTargetPath()); + if(targetElementFound==null || targetElementFound.size()==0 || targetElementFound.get(0)==null) { +// targetCtx=targetCtx.add(mapping.getTargetPath(),Collections.singletonList("nothing")); + Parsing.addElement(targetCtx, mapping.getTargetPath()); + } + // apply resulting value - targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement(),resultingValue); + + targetCtx=targetCtx.put(mapping.getTargetPath(),mapping.getTargetElement(),resultingValue); } }catch(Throwable t) { throw new TransformationException("Exception while applying "+mapping,t); @@ -155,9 +179,30 @@ public class Transformer { return mapper.readValue(targetCtx.jsonString(), ZenodoDeposition.class); }catch(Throwable t) { - log.error("Unable to transform "+source+" using previous "+target,t); + log.error("Unable to translate "+source+" using previous "+target,t); throw new TransformationException("Unable to translate "+source.getName(),t); } } + + public List filterResources(CkanItemDescriptor source) throws TransformationException{ + try { + ObjectMapper mapper=Parsing.getMapper(); + DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(source.getContent()); + ArrayList toReturn=new ArrayList<>(); + for(Filter f:resourceFilter.getFilters()) { + JSONArray filtered=sourceCtx.read(f.getConditions().get(0)); + for(Object obj:filtered) { + Map map=(Map) obj; + + toReturn.add(mapper.readValue((new JSONObject(map)).toJSONString(), CkanResource.class)); + } + } + + return toReturn; + }catch(Throwable t) { + log.error("Unable to filter resources. ",t); + throw new TransformationException("Unable to filter "+source.getName()+" resources",t); + } + } } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/FileUploaderManager.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/FileUploaderManager.java new file mode 100644 index 0000000..bb0391d --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/FileUploaderManager.java @@ -0,0 +1,32 @@ +package org.gcube.data.publishing.ckan2zenodo.clients; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; + +import javax.ws.rs.core.Response; + +import org.gcube.data.publishing.ckan2zenodo.LocalConfiguration; +import org.gcube.data.publishing.ckan2zenodo.LocalConfiguration.Configuration; +import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; + +public class FileUploaderManager { + + static ExecutorService service=null; + + static { + service=Executors.newFixedThreadPool(Integer.parseInt(LocalConfiguration.getProperty(Configuration.THREAD_POOL_SIZE))); + } + + public FutureTask createTask(Callable call ){ + return new FutureTask (call); + } + + + public static Future submit(Callable call){ + return service.submit(call); + } + +} diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java index 3cbcc26..a2c1da4 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java @@ -1,6 +1,12 @@ package org.gcube.data.publishing.ckan2zenodo.clients; +import java.io.File; import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; @@ -12,14 +18,16 @@ import org.gcube.data.publishing.ckan2zenodo.Fixer; import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials; import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; +import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; @@ -35,6 +43,8 @@ public class Zenodo { private static final String DEPOSITION_BASE_URL="deposit/depositions"; + private static final String PUBLISH_URL_PRE="deposit/depositions"; + private static final String PUBLISH_URL_POST="actions/publish"; private static final String ACCESS_TOKEN="access_token"; @@ -54,9 +64,11 @@ public class Zenodo { Client client; private synchronized Client getWebClient() { - if(client==null) + if(client==null) { client = ClientBuilder.newClient() - .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true); + .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true); + client.register(MultiPartFeature.class); + } return client; } @@ -83,37 +95,110 @@ public class Zenodo { } } - public ZenodoDeposition updateMetadata(Integer depositionId,DepositionMetadata meta) throws ZenodoException, JsonProcessingException { + public ZenodoDeposition updateMetadata(ZenodoDeposition dep) throws ZenodoException, JsonProcessingException { + return updateMetadata(dep.getId(), dep.getMetadata()); + } + + public FileDeposition uploadFile(ZenodoDeposition dep, String toUploadName,String urlString) throws ZenodoException { + + Callable call=new Callable() { + @Override + public Response call() throws Exception { + File temp=null; + try { + log.debug("Downloading "+urlString); + //Download locally into temp + URL url=new URL(urlString); + temp=File.createTempFile("zenodo_", ".tmp"); + + long size=Files.copy(url.openStream(), temp.toPath(),StandardCopyOption.REPLACE_EXISTING); + + //upload + FormDataMultiPart multi=new FormDataMultiPart(); + FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file", + temp,MediaType.APPLICATION_OCTET_STREAM_TYPE); + multi.field("name", toUploadName); + multi.bodyPart(fileDataBodyPart); + + log.debug("Starting transfer of "+toUploadName+" ("+urlString+") into "+dep.getId()); + Response toReturn=getWebClient().target(credentials.getBaseUrl()). + path(DEPOSITION_BASE_URL).path(dep.getId()+"").path("files"). + queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE) + .post(Entity.entity(multi,multi.getMediaType())); + + log.debug("DONE."); + return toReturn; + + }catch(Throwable e) { + throw new ZenodoException("Unable to transfer file "+toUploadName+" url : "+urlString,e); + }finally { + //finally delete temp + if(temp!=null) Files.deleteIfExists(temp.toPath()); + } + + } + }; + + + log.debug("Submitting request to upload "+urlString+" to Manager"); + Future resp=FileUploaderManager.submit(call); + + try { + return check(resp.get(),FileDeposition.class); + }catch(ZenodoException z) { + throw z; + }catch(Throwable t) { + throw new ZenodoException(t.getMessage(),t); + } + + //return + + } + + private ZenodoDeposition updateMetadata(Integer depositionId,DepositionMetadata meta) throws ZenodoException, JsonProcessingException { String serialized="{\"metadata\":"+Fixer.fixIncoming(mapper.writeValueAsString(meta))+"}"; try { - Response resp = getWebClient().target(credentials.getBaseUrl()). - path(DEPOSITION_BASE_URL).path(depositionId+""). - queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE) - .put(Entity.json(serialized)); - return check(resp,ZenodoDeposition.class); + Response resp = getWebClient().target(credentials.getBaseUrl()). + path(DEPOSITION_BASE_URL).path(depositionId+""). + queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE) + .put(Entity.json(serialized)); + return check(resp,ZenodoDeposition.class); }catch(Throwable t) { log.debug("Error while tryin to update "+serialized); throw t; } } - - public void deleteDeposition(String depositionId) throws ZenodoException { + + public void deleteDeposition(Integer depositionId) throws ZenodoException { Response resp = getWebClient().target(credentials.getBaseUrl()). - path(DEPOSITION_BASE_URL).path(depositionId). + path(DEPOSITION_BASE_URL).path(depositionId+""). queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE) .delete(); check(resp,null); } public ZenodoDeposition createNew() throws ZenodoException { - Response resp = getWebClient().target(credentials.getBaseUrl()). + Response resp = getWebClient().target(credentials.getBaseUrl()). path(DEPOSITION_BASE_URL). queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE) .post(Entity.json("{}")); return check(resp,ZenodoDeposition.class); } + public ZenodoDeposition publish(ZenodoDeposition dep) throws ZenodoException{ + return publish(dep.getId()); + } + + private ZenodoDeposition publish(Integer depositionId) throws ZenodoException{ + Response resp = getWebClient().target(credentials.getBaseUrl()). + path(PUBLISH_URL_PRE). + path(depositionId+""). + path(PUBLISH_URL_POST). + queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE) + .post(Entity.json("{}")); + return check(resp,ZenodoDeposition.class); + } + - } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java index 51b94ee..9e1d572 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/IS.java @@ -9,10 +9,11 @@ import java.util.List; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.ServiceEndpoint; -import org.gcube.data.publishing.ckan2zenodo.model.Mapping; -import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Regexp; -import org.gcube.data.publishing.ckan2zenodo.model.Mapping.Source; import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Regexp; +import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping.Source; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.w3c.dom.Element; @@ -102,4 +103,29 @@ public class IS { throw new ConfigurationException("Invaild mapping resource "+res.id()+" name : "+res.profile().name(),t); } } + + + public static ResourceFilter readResourceFilters(GenericResource res) throws ConfigurationException{ + try{ + ArrayList filtersList=new ArrayList<>(); + Element root=res.profile().body(); + NodeList filters=((Element) root.getElementsByTagName("resourceFilters").item(0)).getElementsByTagName("filter"); + for(int i=0;i conditions=new ArrayList<>(); + NodeList conditionNodes=filter.getElementsByTagName("condition"); + for(int j=0;j found=ctx.read(parent); + if(found==null || found.size()==0 || found.get(0)==null) { + //missing parent, use recursion + addElement(ctx,parent); + } + // found parent, adding element + String element=path.substring(path.lastIndexOf(".")+1); + + Object value=new HashMap(); + if(element.contains("[")) { + value=new ArrayList(Collections.singletonList(value)); + + element=element.substring(0,element.indexOf("[")); + } + + ctx.put(parent, element, value); + return ctx; + }else throw new ConfigurationException("Unable to initialize non-definite path : "+path); + } + } diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanResource.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanResource.java new file mode 100644 index 0000000..f5d00fd --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanResource.java @@ -0,0 +1,32 @@ +package org.gcube.data.publishing.ckan2zenodo.model; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@ToString +public class CkanResource{ + + private String mimetype; + private String cache_url; + private String hash; + private String description; + private String name; + private String format; + private String url; + private String datastore_active; + private String cache_last_updated; + private String package_id; + private String created; + private String state; + private String mimetype_inner; + private String last_modified; + private String position; + private String revision_id; + private String url_type; + private String id; + private String resource_type; + private String size; +} diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java similarity index 93% rename from src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java rename to src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java index 91122d7..2f6e083 100644 --- a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/Mapping.java @@ -1,4 +1,4 @@ -package org.gcube.data.publishing.ckan2zenodo.model; +package org.gcube.data.publishing.ckan2zenodo.model.parsing; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java new file mode 100644 index 0000000..139c0e6 --- /dev/null +++ b/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/parsing/ResourceFilter.java @@ -0,0 +1,26 @@ +package org.gcube.data.publishing.ckan2zenodo.model.parsing; + +import java.util.List; + +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +@RequiredArgsConstructor +@ToString +@Getter +public class ResourceFilter { + + @RequiredArgsConstructor + @ToString + @Getter + public static class Filter { + @NonNull + private List conditions; + } + + @NonNull + private List filters; + +} diff --git a/src/main/resources/config.properties b/src/main/resources/config.properties new file mode 100644 index 0000000..1fe9fbf --- /dev/null +++ b/src/main/resources/config.properties @@ -0,0 +1 @@ +THREAD_POOL_SIZE=5 \ No newline at end of file diff --git a/src/test/java/org/gcube/tests/ParsingTests.java b/src/test/java/org/gcube/tests/ParsingTests.java index bedf9e1..7a9efc7 100644 --- a/src/test/java/org/gcube/tests/ParsingTests.java +++ b/src/test/java/org/gcube/tests/ParsingTests.java @@ -1,15 +1,27 @@ package org.gcube.tests; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import org.gcube.data.publishing.ckan2zenodo.Fixer; +import org.gcube.data.publishing.ckan2zenodo.commons.Parsing; +import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; +import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; public class ParsingTests { @@ -18,8 +30,8 @@ public class ParsingTests { public static void init () { mapper=TestCommons.getMapper(); } - - + + @Test public void DateTest() throws Exception { for(String d:new String[] {"2019-11-29T14:54:42.542142","2016-06-15T16:10:03.319363+00:00","2019-11-30","2019-11-29T17:01:31.000160+0000"}) { @@ -28,7 +40,7 @@ public class ParsingTests { System.out.println("--"); } } - + @Test public void fullCircleFromIncoming() throws JsonParseException, JsonMappingException, IOException { String s="{\n" + @@ -85,9 +97,38 @@ public class ParsingTests { System.out.println("FIXED JSON "); String fixed=Fixer.fixSending(json); System.out.println(fixed); - + } + + @Test + public void miscTests() throws IOException, ConfigurationException { +// ObjectMapper mapper=Parsing.getMapper(); +// mapper.setSerializationInclusion(Include.ALWAYS); + ZenodoDeposition dep=new ZenodoDeposition(); + dep.setMetadata(new DepositionMetadata()); + + + DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(mapper.writeValueAsString(dep)); + Parsing.addElement(sourceCtx,"$.metadata.creators[0]"); +// addElement(sourceCtx,"$.metadata.creators[0].name",true); + sourceCtx.put("$.metadata.creators[0]","name", "myName"); + System.out.println("JSON : "+sourceCtx.jsonString()); + Assert.assertTrue(sourceCtx.jsonString().contains("creators")); + Assert.assertTrue(sourceCtx.jsonString().contains("myName")); + ZenodoDeposition parsed=mapper.readValue(sourceCtx.jsonString(), ZenodoDeposition.class); + Assert.assertEquals(parsed.getMetadata().getCreators().get(0).getName(), "myName"); + + } + - + @Test + public void read() { + DocumentContext sourceCtx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse("{\"metadata\":{\"creators\":[{\"name\":\"\"}]}}"); + List found=sourceCtx.read("$['metadata']['creators'][0]"); + for(Object s: found) { + + System.out.println(s); + } + } } diff --git a/src/test/java/org/gcube/tests/TestCommons.java b/src/test/java/org/gcube/tests/TestCommons.java index eefd1d6..080f0c3 100644 --- a/src/test/java/org/gcube/tests/TestCommons.java +++ b/src/test/java/org/gcube/tests/TestCommons.java @@ -5,7 +5,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import org.gcube.data.publishing.ckan2zenodo.Fixer; -import org.gcube.data.publishing.ckan2zenodo.Transformer; +import org.gcube.data.publishing.ckan2zenodo.Translator; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; @@ -41,13 +41,13 @@ public class TestCommons { return mapper; } - static final ZenodoDeposition readAndTransform(String jsonFile, Transformer transformer,ZenodoDeposition...depositions) throws Exception { + static final ZenodoDeposition readAndTransform(String jsonFile, Translator transformer,ZenodoDeposition...depositions) throws Exception { try{ String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile)); CkanItemDescriptor desc=new CkanItemDescriptor(json); System.out.println("Going to transform : "+desc.getContent()); System.out.println("Result : "); - ZenodoDeposition dep=transformer.transform(desc, depositions!=null?depositions[0]:null); + ZenodoDeposition dep=transformer.transform(desc, (depositions!=null&&depositions.length>0)?depositions[0]:null); System.out.println(dep); System.out.println("As JSON : "); System.out.println(Fixer.fixSending(getMapper().writeValueAsString(dep))); diff --git a/src/test/java/org/gcube/tests/TransformationTests.java b/src/test/java/org/gcube/tests/TransformationTests.java index b548749..b34eac6 100644 --- a/src/test/java/org/gcube/tests/TransformationTests.java +++ b/src/test/java/org/gcube/tests/TransformationTests.java @@ -1,15 +1,9 @@ package org.gcube.tests; -import java.io.IOException; -import java.util.Collections; - import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.Resources; -import org.gcube.data.publishing.ckan2zenodo.Fixer; -import org.gcube.data.publishing.ckan2zenodo.Transformer; +import org.gcube.data.publishing.ckan2zenodo.Translator; import org.gcube.data.publishing.ckan2zenodo.commons.IS; -import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; -import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; import org.junit.BeforeClass; import org.junit.Test; @@ -27,7 +21,7 @@ public class TransformationTests { @Test public void transform() throws Exception { - Transformer basic=new Transformer(Collections.EMPTY_LIST); + Translator basic=new Translator(); TestCommons.readAndTransform("/simpleItem.json",basic); TestCommons.readAndTransform("/FSKXModel.json",basic); TestCommons.readAndTransform("/ResearchObject.json",basic); @@ -38,7 +32,7 @@ public class TransformationTests { @Test public void transformWithMappings() throws Exception { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); - TestCommons.readAndTransform("/ResearchObject.json", new Transformer(IS.readMappings(res))); + TestCommons.readAndTransform("/ResearchObject.json", new Translator(IS.readMappings(res))); } diff --git a/src/test/java/org/gcube/tests/ZenodoTests.java b/src/test/java/org/gcube/tests/ZenodoTests.java index a6522a6..2d5a3b7 100644 --- a/src/test/java/org/gcube/tests/ZenodoTests.java +++ b/src/test/java/org/gcube/tests/ZenodoTests.java @@ -1,17 +1,16 @@ package org.gcube.tests; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.Arrays; import java.util.Date; import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.Resources; -import org.gcube.data.publishing.ckan2zenodo.Transformer; +import org.gcube.data.publishing.ckan2zenodo.Translator; import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo; import org.gcube.data.publishing.ckan2zenodo.commons.IS; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; +import org.gcube.data.publishing.ckan2zenodo.model.CkanResource; import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials; import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException; import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException; @@ -19,6 +18,7 @@ import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Creator; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.AccessRights; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.UploadType; +import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; import org.junit.BeforeClass; import org.junit.Test; @@ -60,7 +60,7 @@ public class ZenodoTests { Arrays.asList(new Creator("simpleMan")), "Simple description", AccessRights.open)); - System.out.println(z.updateMetadata(dep.getId(),dep.getMetadata())); + System.out.println(z.updateMetadata(dep)); } @@ -70,9 +70,23 @@ public class ZenodoTests { GenericResource res=Resources.unmarshal(GenericResource.class, TransformationTests.class.getResourceAsStream("/ResearchObject.xml")); ZenodoDeposition dep=z.createNew(); - dep=TestCommons.readAndTransform("/ResearchObject.json", new Transformer(IS.readMappings(res)),dep); + Translator tran=new Translator(IS.readMappings(res),IS.readResourceFilters(res)); + dep=TestCommons.readAndTransform("/ResearchObject.json", tran,dep); + dep=z.updateMetadata(dep); + System.out.println("Resulting Deposition with metadata : "); + System.out.println(); + String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream("/ResearchObject.json")); + CkanItemDescriptor desc=new CkanItemDescriptor(json); - System.out.println(z.updateMetadata(dep.getId(), dep.getMetadata())); + for(CkanResource cRes:tran.filterResources(desc)) { + FileDeposition file=z.uploadFile(dep, cRes.getName(), cRes.getUrl()); + System.out.println("Published "+file); + } + + + dep=z.publish(dep); + + System.out.println(); } } diff --git a/src/test/resources/ResearchObject.json b/src/test/resources/ResearchObject.json index 3dd57c7..6006de5 100644 --- a/src/test/resources/ResearchObject.json +++ b/src/test/resources/ResearchObject.json @@ -38,6 +38,28 @@ "id": "8caa68cb-6880-4683-9849-1aad02d8cc11", "resource_type": null, "size": null + }, + { + "mimetype": null, + "cache_url": null, + "hash": "", + "description": "Curated by: World Health Organization (WHO)\r\n\r\nAs part of their core goal for better health information worldwide, the World Health Organization makes their data on global health publicly available through the Global Health Observatory (GHO). The GHO acts as a portal with which to access and analyze health situations and important themes.\r\n\r\nThe various data sets are organized according to themes, such as mortality, health systems, communicable and non-communicable diseases, medicines and vaccines, health risks, and so on. The WHO\u2019s health statistics are to go-to source for global health information and is also used in the work of the US Centers for Disease Control and Prevention.", + "name": "Some test csv", + "format": "CSV", + "url": "https://data.d4science.net/7CQd", + "datastore_active": false, + "cache_last_updated": null, + "package_id": "54938132-299e-46f7-a84a-494fcd75ad8f", + "created": "2019-12-03T11:20:02.594436", + "state": "active", + "mimetype_inner": null, + "last_modified": null, + "position": 0, + "revision_id": "9dd5e97c-8c39-4198-a34e-c56ae0631d17", + "url_type": null, + "id": "8caa68cb-6880-4683-9849-1aad02d8cc22", + "resource_type": null, + "size": null }], "num_resources": 1, "tags": [ diff --git a/src/test/resources/ResearchObject.xml b/src/test/resources/ResearchObject.xml index 5d1a708..3e1c9d2 100644 --- a/src/test/resources/ResearchObject.xml +++ b/src/test/resources/ResearchObject.xml @@ -7,12 +7,19 @@ ResearchObject Simple mappings tests + + + $.resources[?(@.format=='CSV')] + + + + dataset - + $.metadata upload_type @@ -22,7 +29,7 @@ $.extras[?(@.key=='Author')].value - $.metadata.contributors[0] + $.metadata.creators[0] name ([A-Za-z]*, [A-Za-z]*)(?=,) @@ -32,7 +39,7 @@ Producer - $.metadata.contributors[0] + $.metadata.creators[0] type