gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Date parsing

This commit is contained in:
Fabio Sinibaldi 2019-11-27 18:21:01 +01:00
parent c669ee2971
commit cc8895533d
10 changed files with 245 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
ckan2zenodo-library/pom.xml
View File

@ -50,6 +50,12 @@
<groupId>org.gcube.data-publishing</groupId>
<artifactId>gcat-client</artifactId>
<version>[1.2.0,2.0.0-SNAPSHOT)</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- EXTERNAL -->
@ -60,12 +66,26 @@
<version>1.14.8</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.4.0</version>
</dependency>
<!-- Test log binding -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.13</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

4
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2ZenodoImpl.java
View File

@ -5,7 +5,7 @@ import java.net.MalformedURLException;
import org.gcube.data.publishing.ckan2zenodo.clients.GCat;
import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
public class Ckan2ZenodoImpl {
@ -25,7 +25,7 @@ public class Ckan2ZenodoImpl {
}
private static final String getZenodoToken() {
private static final ZenodoCredentials getZenodoToken() {
throw new RuntimeException();
}
}

64
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Fixer.java Normal file
View File

@ -0,0 +1,64 @@
package org.gcube.data.publishing.ckan2zenodo;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Fixer {
public static final String fixIncoming(String toFix) {
DocumentContext ctx=JsonPath.parse(toFix);
ctx=fixIncomingDate(ctx, "$.created");
ctx=fixIncomingDate(ctx, "$.modified");
return ctx.jsonString();
}
public static String fixSending(String toFix) {
DocumentContext ctx=JsonPath.parse(toFix);
fixOutgoingDate(ctx, "$.created");
fixOutgoingDate(ctx, "$.modified");
return ctx.jsonString();
}
//*** INCOMINGs
private static final DocumentContext fixIncomingDate(DocumentContext ctx,String toFixPath) {
try {
StringBuilder toFix=new StringBuilder(ctx.read(toFixPath));
ctx.put(toFixPath.substring(0,toFixPath.lastIndexOf(".")),
toFixPath.substring(toFixPath.lastIndexOf(".")+1),
toFix.deleteCharAt(toFix.lastIndexOf(":")).toString());
}catch(Throwable e) {
log.warn("Unable to fix "+toFixPath+". Cause : ",e);
}
return ctx;
}
//*** OUTGOINGs
private static final DocumentContext fixOutgoingDate(DocumentContext ctx,String toFixPath) {
try{
String toFix=ctx.read(toFixPath);
ctx.put(toFixPath.substring(0,toFixPath.lastIndexOf(".")),
toFixPath.substring(toFixPath.lastIndexOf(".")+1),
toFix.substring(0,toFix.length()-2)+":"+
toFix.substring(toFix.length()-2, toFix.length()));
}catch(Throwable t) {
log.debug("Unable to fix "+toFixPath+".Cause : ",t);
}
return ctx;
}
}

16
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Transformer.java Normal file
View File

@ -0,0 +1,16 @@
package org.gcube.data.publishing.ckan2zenodo;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
public class Transformer {
public ZenodoDeposition transform(CkanItemDescriptor toTransform) {
ZenodoDeposition dep=new ZenodoDeposition();
// dep.set
throw new RuntimeException("Implment this");
}
}

4
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/TransformerManager.java
View File

@ -2,6 +2,8 @@ package org.gcube.data.publishing.ckan2zenodo;
public class TransformerManager {
public Transformer getByProfile(String profile) {
throw new RuntimeException("Implement this");
}
}

104
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java
View File

@ -1,65 +1,103 @@
package org.gcube.data.publishing.ckan2zenodo.clients;
import java.io.IOException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.gcube.data.publishing.ckan2zenodo.Fixer;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.glassfish.jersey.client.ClientProperties;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class Zenodo {
private static final String BASE_URL="https://zenodo.org/api/";
private static final String CONTENT_TYPE="application/json";
private static final String DEPOSITION_BASE_URL="deposit/depositions";
private static final String AUTH="Authorization";
private static final String BEARER="Bearer ";
private static final String ACCESS_TOKEN="access_token";
private static ObjectMapper mapper = new ObjectMapper();
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,false);
// mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
}
@NonNull
private String tokenValue;
private ZenodoCredentials credentials;
Client client;
private synchronized Client getWebClient() {
if(client==null)
client = ClientBuilder.newClient(
// new ClientConfig().register(new HTTPBasicProvider(username, password))
)
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
client = ClientBuilder.newClient()
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
return client;
}
public ZenodoDeposition readDeposition(String id) throws ZenodoException {
// String respString=null;
return getWebClient().target(BASE_URL).
path(DEPOSITION_BASE_URL).path(id).request(CONTENT_TYPE)
.header(AUTH, BEARER+tokenValue)
.get(ZenodoDeposition.class);
// respString=check(resp);
Response resp= getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(id).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.get();
return check(resp,ZenodoDeposition.class);
}
private static String check(Response resp) throws ZenodoException {
private static <T> T check(Response resp, Class<T> clazz) throws ZenodoException{
if(resp.getStatus()<200||resp.getStatus()>=300) {
throw new ZenodoException("RESP STATUS IS "+resp.getStatus()+". Message : "+resp.readEntity(String.class));
}else
return resp.readEntity(String.class);
}else {
String respString=resp.readEntity(String.class);
try {
return mapper.readValue(Fixer.fixIncoming(respString), clazz);
} catch (IOException e) {
throw new ZenodoException("Unable to parse response from Zenodo. Content was : \n "+respString,e);
}
}
}
public ZenodoDeposition publishOrUpdate(ZenodoDeposition d) {
public ZenodoDeposition Update(ZenodoDeposition d) {
throw new RuntimeException("Implement This");
}
public ZenodoDeposition create(DepositionMetadata meta) throws ZenodoException, JsonParseException, JsonMappingException, IOException {
ZenodoDeposition toSend=new ZenodoDeposition();
toSend.setMetadata(meta);
String serialized=Fixer.fixSending(mapper.writeValueAsString(toSend));
try{
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.post(Entity.json(serialized));
return check(resp,ZenodoDeposition.class);
}catch(Throwable t) {
log.debug("Exception while sending : "+serialized);
throw t;
}
}
}

15
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/ZenodoCredentials.java Normal file
View File

@ -0,0 +1,15 @@
package org.gcube.data.publishing.ckan2zenodo.model;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Getter
@RequiredArgsConstructor
public class ZenodoCredentials {
@NonNull
private String key;
@NonNull
private String baseUrl;
}

12
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/ZenodoDeposition.java
View File

@ -4,20 +4,20 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import com.fasterxml.jackson.annotation.JsonFormat;
@Getter
@Setter
@ToString
import lombok.Data;
@Data
public class ZenodoDeposition {
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ")
private Date created;
private String doi;
private ArrayList<FileDeposition> files;
private Integer id;
private DepositionMetadata metadata;
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ")
private Date modified;
private Integer owner;
private Integer record_id;

36
ckan2zenodo-library/src/test/java/org/gcube/tests/ParsingTests.java Normal file
View File

@ -0,0 +1,36 @@
package org.gcube.tests;
import java.io.IOException;
import java.util.Date;
import org.gcube.data.publishing.ckan2zenodo.Fixer;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.junit.BeforeClass;
import org.junit.Test;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
public class ParsingTests {
static ObjectMapper mapper=new ObjectMapper();
@BeforeClass
public static void init () {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,false);
}
@Test
public void DateTest() throws IOException {
ZenodoDeposition z=new ZenodoDeposition();
z.setCreated(new Date(System.currentTimeMillis()));
System.out.println("Write : ");
System.out.println(Fixer.fixSending(mapper.writeValueAsString(z)));
String s="{\"conceptrecid\":\"422982\",\"created\":\"2019-11-26T15:43:49.879+00:00\",\"files\":[],\"id\":422983,\"links\":{\"bucket\":\"https://sandbox.zenodo.org/api/files/a6b09898-1807-4f03-a7ac-ed06b2d3ba2b\",\"discard\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983/actions/discard\",\"edit\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983/actions/edit\",\"files\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983/files\",\"html\":\"https://sandbox.zenodo.org/deposit/422983\",\"latest_draft\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983\",\"latest_draft_html\":\"https://sandbox.zenodo.org/deposit/422983\",\"newversion\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983/actions/newversion\",\"publish\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983/actions/publish\",\"registerconceptdoi\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983/actions/registerconceptdoi\",\"self\":\"https://sandbox.zenodo.org/api/deposit/depositions/422983\"},\"metadata\":{\"prereserve_doi\":{\"doi\":\"10.5072/zenodo.422983\",\"recid\":422983}},\"modified\":\"2019-11-26T15:43:49.876858+00:00\",\"owner\":31041,\"record_id\":422983,\"state\":\"unsubmitted\",\"submitted\":false,\"title\":\"\"}";
System.out.println("READ : ");
System.out.println(mapper.readValue(Fixer.fixIncoming(s), ZenodoDeposition.class));
}
}

13
ckan2zenodo-library/src/test/java/org/gcube/tests/ZenodoTests.java
View File

@ -5,16 +5,27 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.junit.Test;
public class ZenodoTests {
@Test
public void readFromZenodo() throws IOException, ZenodoException {
// private static final String BASE_URL="https://zenodo.org/api/";
String BASE_URL="https://sandbox.zenodo.org/api/";
String key=Files.readAllLines(Paths.get("/home/fabio/workspaces/zenodo.key")).get(0);
System.out.println(new Zenodo(key).readDeposition("10"));
System.out.println("Using key : "+key);
Zenodo z=new Zenodo(new ZenodoCredentials(key,BASE_URL));
System.out.println(z.readDeposition("422983"));
System.out.println(z.create(new DepositionMetadata()));
}
}