gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Multiple data values and pattern management

This commit is contained in:
Fabio Sinibaldi 2019-11-29 15:52:17 +01:00
parent cc8895533d
commit b3cb4b792c
7 changed files with 190 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/Fixer.java
View File

@ -1,64 +1,115 @@
package org.gcube.data.publishing.ckan2zenodo;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Commons;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import com.jayway.jsonpath.Option;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Fixer {
private static final ArrayList<SimpleDateFormat> INCOMING_DATE_FORMATS=new ArrayList<SimpleDateFormat>();
private static final SimpleDateFormat INTERNAL_DATE_FORMAT=new SimpleDateFormat(Commons.ISO_DATE_PATTERN);
private static Configuration PATH_CONFIGURATION=null;
private static Configuration ALWAYS_LIST_CONFIG=null;
static {
INCOMING_DATE_FORMATS.add(new SimpleDateFormat("YYYY-MM-DD"));
INCOMING_DATE_FORMATS.add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX"));
PATH_CONFIGURATION = Configuration.builder().options(Option.AS_PATH_LIST,Option.SUPPRESS_EXCEPTIONS).build();
ALWAYS_LIST_CONFIG= Configuration.builder().options(Option.ALWAYS_RETURN_LIST,Option.SUPPRESS_EXCEPTIONS).build();
}
public static final String fixIncoming(String toFix) {
DocumentContext ctx=JsonPath.parse(toFix);
DocumentContext ctx=JsonPath.using(ALWAYS_LIST_CONFIG).parse(toFix);
DocumentContext pathCtx=JsonPath.using(PATH_CONFIGURATION).parse(toFix);
ctx=fixIncomingDate(ctx, "$.created");
ctx=fixIncomingDate(ctx, "$.modified");
ctx=fixIncomingDate(ctx,pathCtx, "$.created");
ctx=fixIncomingDate(ctx,pathCtx, "$.modified");
ctx=fixIncomingDate(ctx,pathCtx, "$..publication_date");
ctx=fixIncomingDate(ctx,pathCtx, "$..embargo_date");
return ctx.jsonString();
}
public static String fixSending(String toFix) {
DocumentContext ctx=JsonPath.parse(toFix);
DocumentContext ctx=JsonPath.using(ALWAYS_LIST_CONFIG).parse(toFix);
DocumentContext pathCtx=JsonPath.using(PATH_CONFIGURATION).parse(toFix);
ctx=fixOutgoingDate(ctx,pathCtx, "$.created");
ctx=fixOutgoingDate(ctx,pathCtx, "$.modified");
ctx=fixOutgoingDate(ctx,pathCtx, "$..publication_date");
ctx=fixOutgoingDate(ctx,pathCtx, "$..embargo_date");
fixOutgoingDate(ctx, "$.created");
fixOutgoingDate(ctx, "$.modified");
return ctx.jsonString();
}
//*** INCOMINGs
private static final DocumentContext fixIncomingDate(DocumentContext ctx,String toFixPath) {
try {
StringBuilder toFix=new StringBuilder(ctx.read(toFixPath));
ctx.put(toFixPath.substring(0,toFixPath.lastIndexOf(".")),
toFixPath.substring(toFixPath.lastIndexOf(".")+1),
toFix.deleteCharAt(toFix.lastIndexOf(":")).toString());
private static final DocumentContext fixIncomingDate(DocumentContext valuesCtx,DocumentContext pathCtx,String toFixPath) {
try {
List<String> values=valuesCtx.read(toFixPath);
List<String> paths=pathCtx.read(toFixPath);
for(int i=0;i<values.size();i++) {
String toFix=values.get(i);
JsonPath path=JsonPath.compile(paths.get(i));
valuesCtx.set(path, fixIncomingDateString(toFix));
}
}catch(Throwable e) {
log.warn("Unable to fix "+toFixPath+". Cause : ",e);
log.debug("Unable to fix "+toFixPath+". Cause : ",e);
}
return ctx;
return valuesCtx;
}
private static final String fixIncomingDateString(String toFix) throws Exception {
String fixed=null;
for(SimpleDateFormat format:INCOMING_DATE_FORMATS) {
try {
Date d=format.parse(toFix);
fixed=INTERNAL_DATE_FORMAT.format(d);
break;
}catch(Throwable t) {
//skip
}
}
if(fixed==null) throw new Exception("No format suitable for incoming date "+toFix);
return fixed;
}
//*** OUTGOINGs
private static final DocumentContext fixOutgoingDate(DocumentContext ctx,String toFixPath) {
private static final DocumentContext fixOutgoingDate(DocumentContext valuesCtx,DocumentContext pathCtx,String toFixPath) {
try{
String toFix=ctx.read(toFixPath);
ctx.put(toFixPath.substring(0,toFixPath.lastIndexOf(".")),
toFixPath.substring(toFixPath.lastIndexOf(".")+1),
toFix.substring(0,toFix.length()-2)+":"+
toFix.substring(toFix.length()-2, toFix.length()));
List<String> values=valuesCtx.read(toFixPath);
List<String> paths=pathCtx.read(toFixPath);
for(int i=0;i<values.size();i++) {
String toFix=values.get(i);
JsonPath path=JsonPath.compile(paths.get(i));
valuesCtx.set(path, toFix.substring(0,toFix.length()-2)+":"+
toFix.substring(toFix.length()-2, toFix.length()));
}
}catch(Throwable t) {
log.debug("Unable to fix "+toFixPath+".Cause : ",t);
}
return ctx;
return valuesCtx;
}
}

42
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java
View File

@ -15,10 +15,13 @@ import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.glassfish.jersey.client.ClientProperties;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@ -39,6 +42,8 @@ public class Zenodo {
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,false);
mapper.configure(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, false);
mapper.setSerializationInclusion(Include.NON_NULL);
// mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
}
@ -68,6 +73,7 @@ public class Zenodo {
if(resp.getStatus()<200||resp.getStatus()>=300) {
throw new ZenodoException("RESP STATUS IS "+resp.getStatus()+". Message : "+resp.readEntity(String.class));
}else {
if(clazz==null) return null;
String respString=resp.readEntity(String.class);
try {
return mapper.readValue(Fixer.fixIncoming(respString), clazz);
@ -77,24 +83,34 @@ public class Zenodo {
}
}
public ZenodoDeposition Update(ZenodoDeposition d) {
throw new RuntimeException("Implement This");
public ZenodoDeposition updateMetadata(Integer depositionId,DepositionMetadata meta) throws ZenodoException, JsonProcessingException {
String serialized="{\"metadata\":"+Fixer.fixIncoming(mapper.writeValueAsString(meta))+"}";
try {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(depositionId+"").
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.put(Entity.json(serialized));
return check(resp,ZenodoDeposition.class);
}catch(Throwable t) {
log.debug("Error while tryin to update "+serialized);
throw t;
}
}
public void deleteDeposition(String depositionId) throws ZenodoException {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(depositionId).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.delete();
check(resp,null);
}
public ZenodoDeposition create(DepositionMetadata meta) throws ZenodoException, JsonParseException, JsonMappingException, IOException {
ZenodoDeposition toSend=new ZenodoDeposition();
toSend.setMetadata(meta);
String serialized=Fixer.fixSending(mapper.writeValueAsString(toSend));
try{
public ZenodoDeposition createNew() throws ZenodoException {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.post(Entity.json(serialized));
return check(resp,ZenodoDeposition.class);
}catch(Throwable t) {
log.debug("Exception while sending : "+serialized);
throw t;
}
.post(Entity.json("{}"));
return check(resp,ZenodoDeposition.class);
}

8
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/Commons.java Normal file
View File

@ -0,0 +1,8 @@
package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
public class Commons {
public static final String ISO_DATE_PATTERN="yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ";
}

2
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/Creator.java
View File

@ -3,9 +3,11 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@NoArgsConstructor
@RequiredArgsConstructor
public class Creator {
@NonNull

45
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/DepositionMetadata.java
View File

@ -1,13 +1,18 @@
package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@NoArgsConstructor
@RequiredArgsConstructor
public class DepositionMetadata {
public static enum UploadType{
@ -52,27 +57,41 @@ public class DepositionMetadata {
other
}
public static enum AccessRights{
open,
embargoed,
restricted,
closed
}
@NonNull
private UploadType upload_type;
private PublicationType publication_type;
private ImageType image_type;
@NonNull
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
private Date publication_date;
@NonNull
private String title;
private ArrayList<Creator> creators;
@NonNull
private List<Creator> creators;
@NonNull
private String description; // TODO HTML
private String access_rights;
@NonNull
private AccessRights access_right;
private String license; // TODO ENUM https://licenses.opendefinition.org/licenses/groups/all.json
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
private Date embargo_date;
private String access_conditions; // TODO HTML
private String doi;
private Boolean preserve_doi;
private ArrayList<String> keywords;
private List<String> keywords;
private String notes; // TODO HTML
private ArrayList<RelatedIdentifier> related_identifiers;
private ArrayList<Contributor> contributors;
private ArrayList<String> references;
private ArrayList<Community> communities;
private ArrayList<Grant> grants;
private List<RelatedIdentifier> related_identifiers;
private List<Contributor> contributors;
private List<String> references;
private List<Community> communities;
private List<Grant> grants;
private String journal_title;
private String journal_volume;
private String journal_issue;
@ -89,13 +108,13 @@ public class DepositionMetadata {
private String imprint_place;
private String partof_title;
private String partof_pages;
private ArrayList<Creator> thesis_supervisors;
private List<Creator> thesis_supervisors;
private String thesis_university;
private ArrayList<Subject> subjects;
private List<Subject> subjects;
private String version;
private String language; //https://www.loc.gov/standards/iso639-2/php/code_list.php
private ArrayList<Location> locations;
private ArrayList<DateInterval> dates;
private List<Location> locations;
private List<DateInterval> dates;
private String method; //TODO html
}

4
ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/ZenodoDeposition.java
View File

@ -11,13 +11,13 @@ import lombok.Data;
@Data
public class ZenodoDeposition {
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ")
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
private Date created;
private String doi;
private ArrayList<FileDeposition> files;
private Integer id;
private DepositionMetadata metadata;
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ")
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
private Date modified;
private Integer owner;
private Integer record_id;

45
ckan2zenodo-library/src/test/java/org/gcube/tests/ZenodoTests.java
View File

@ -3,29 +3,62 @@ package org.gcube.tests;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Date;
import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.Creator;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.AccessRights;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata.UploadType;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.junit.BeforeClass;
import org.junit.Test;
import com.fasterxml.jackson.core.JsonProcessingException;
public class ZenodoTests {
@Test
public void readFromZenodo() throws IOException, ZenodoException {
// private static final String BASE_URL="https://zenodo.org/api/";
private static ZenodoCredentials credentials;
@BeforeClass
public static final void init() throws IOException {
String BASE_URL="https://sandbox.zenodo.org/api/";
String key=Files.readAllLines(Paths.get("/home/fabio/workspaces/zenodo.key")).get(0);
System.out.println("Using key : "+key);
Zenodo z=new Zenodo(new ZenodoCredentials(key,BASE_URL));
credentials=new ZenodoCredentials(key,BASE_URL);
}
@Test
public void readFromZenodo() throws IOException, ZenodoException {
// private static final String BASE_URL="https://zenodo.org/api/";
Zenodo z=new Zenodo(credentials);
System.out.println(z.readDeposition("422983"));
System.out.println(z.create(new DepositionMetadata()));
}
@Test
public void createDeposition() throws ZenodoException, JsonProcessingException {
Zenodo z=new Zenodo(credentials);
ZenodoDeposition dep=z.createNew();
dep.setMetadata(new DepositionMetadata(UploadType.dataset,
new Date(System.currentTimeMillis()),
"Test",
Arrays.asList(new Creator("simpleMan")),
"Simple description",
AccessRights.open));
System.out.println(z.updateMetadata(dep.getId(),dep.getMetadata()));
}
}