gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Upload policy and automatic new version creation

This commit is contained in:
Fabio Sinibaldi 2020-01-24 12:46:13 +01:00
parent 8c6886d518
commit 44e14add24
6 changed files with 203 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2Zenodo.java
View File

@ -7,6 +7,7 @@ import java.util.concurrent.Future;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.UploadPolicy;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.GcatException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;
@ -59,7 +60,7 @@ public interface Ckan2Zenodo {
public List<CkanResource> filterResources(CkanItemDescriptor desc) throws ConfigurationException, TransformationException;
/**
* Uploads @param toUpload resources associating them to given @param deposition
* Same as uploadFiles(Set<CkanResource> toUpload,ZenodoDeposition deposition,UploadPolicy policy) with default policy
*
* @param toUpload
* @param deposition
@ -69,6 +70,18 @@ public interface Ckan2Zenodo {
*/
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload,ZenodoDeposition deposition) throws ZenodoException, ConfigurationException;
/**
* Uploads @param toUpload resources associating them to given @param deposition
*
* @param toUpload
* @param deposition
* @param policy
* @return
* @throws ZenodoException
* @throws ConfigurationException
*/
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload,ZenodoDeposition deposition,UploadPolicy policy) throws ZenodoException, ConfigurationException;
/**
* Publishes @param dep, setting/updateing DOI reference into @param toUpdate
*

53
src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2ZenodoImpl.java
View File

@ -3,15 +3,16 @@ package org.gcube.data.publishing.ckan2zenodo;
import java.net.MalformedURLException;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import org.gcube.data.publishing.ckan2zenodo.clients.FileUploaderManager;
import org.gcube.data.publishing.ckan2zenodo.clients.GCat;
import org.gcube.data.publishing.ckan2zenodo.clients.UploadFilesCall;
import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanRelatedIdentifier;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.UploadPolicy;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.GcatException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;
@ -19,22 +20,21 @@ import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationExceptio
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Ckan2ZenodoImpl implements Ckan2Zenodo{
private Zenodo z=null;
private synchronized Zenodo getZenodo() throws ConfigurationException {
if(z==null)
z=Zenodo.get();
return z;
}
@Override
public CkanItemDescriptor read(String itemName) throws GcatException {
try{
@ -68,8 +68,12 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
}
if(toUpdate.getSubmitted()&&toUpdate.getState().equals("done"))
z.unlockPublished(toUpdate.getId());
ZenodoDeposition updated=z.updateMetadata(toUpdate);
if(toUpdate.getSubmitted())
return z.publish(updated);
else return updated;
return z.updateMetadata(toUpdate);
}
@Override
@ -80,26 +84,19 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
@Override
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload, ZenodoDeposition deposition) throws ZenodoException, ConfigurationException {
return uploadFiles(toUpload, deposition, UploadPolicy.DELETE_ALL);
}
@Override
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload, ZenodoDeposition deposition,
UploadPolicy policy) throws ZenodoException, ConfigurationException {
final Zenodo z=getZenodo();
if(deposition.getSubmitted()&&deposition.getState().equals("done"))
z.unlockPublished(deposition.getId());
Callable<ZenodoDeposition> call=new Callable<ZenodoDeposition>() {
@Override
public ZenodoDeposition call() throws Exception {
ZenodoDeposition dep=deposition;
for(CkanResource r:toUpload) {
try {
dep.getFiles().add(z.uploadFile(dep, r.getName(), r.getUrl()));
}catch(Throwable t) {
throw new Exception("Unable to upload "+r.getName()+".",t);
}
}
return z.readDeposition(dep.getId());
}
};
if(deposition.getSubmitted())
deposition=z.newVersion(deposition.getId());
UploadFilesCall call=new UploadFilesCall(policy,toUpload,deposition,z);
return FileUploaderManager.submitForDeposition(call);
}
@ -107,7 +104,7 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
public ZenodoDeposition publish(ZenodoDeposition dep, CkanItemDescriptor toUpdate) throws ZenodoException, ConfigurationException, InvalidItemException, MalformedURLException {
Zenodo z=getZenodo();
if(dep.getSubmitted()&&dep.getState().equals("done"))
z.unlockPublished(dep.getId());
z.unlockPublished(dep.getId());
// Publishing to zenodo
ZenodoDeposition toReturn =z.publish(dep);
// Updateing item
@ -118,6 +115,6 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
return toReturn;
}
}

81
src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/UploadFilesCall.java Normal file
View File

@ -0,0 +1,81 @@
package org.gcube.data.publishing.ckan2zenodo.clients;
import java.util.Collection;
import java.util.concurrent.Callable;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.UploadPolicy;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
public class UploadFilesCall implements Callable<ZenodoDeposition>{
private UploadPolicy policy;
private Collection<CkanResource> toUpload;
private ZenodoDeposition deposition;
private Zenodo z;
public UploadFilesCall(UploadPolicy policy, Collection<CkanResource> toUpload, ZenodoDeposition deposition,
Zenodo z) {
super();
this.policy = policy;
this.toUpload = toUpload;
this.deposition = deposition;
this.z = z;
}
@Override
public ZenodoDeposition call() throws Exception {
ZenodoDeposition dep=deposition;
if(policy.equals(UploadPolicy.DELETE_ALL)) {
for(FileDeposition f:dep.getFiles()) {
try{
z.deleteFile(dep.getId(), f);
}catch(Throwable t) {
throw new Exception("Unable to delete "+f,t);
}
}
}
for(CkanResource r:toUpload) {
try {
switch(policy) {
case SKIP_EXISTING : {
boolean found=false;
for(FileDeposition f:dep.getFiles()) {
if(f.getFilename().equals(r.getName())) {
found=true;
break;
}
}
if(!found) z.uploadFile(dep, r.getName(), r.getUrl());
break;
}
case DELETE_EXISTING : {
for(FileDeposition f:dep.getFiles()) {
if(f.getFilename().equals(r.getName())) {
z.deleteFile(dep.getId(), f);
break;
}
}
// continue with default
}
default : {
z.uploadFile(dep, r.getName(), r.getUrl());
}
}
}catch(Throwable t) {
throw new Exception("Unable to upload "+r.getName()+".",t);
}
}
return z.readDeposition(dep.getId());
}
}

82
src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java
View File

@ -2,6 +2,7 @@ package org.gcube.data.publishing.ckan2zenodo.clients;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
@ -50,12 +51,15 @@ public class Zenodo {
private static final String DEPOSITION_BASE_URL="deposit/depositions";
private static final String PUBLISH_URL_PRE="deposit/depositions";
private static final String PUBLISH_URL_POST="actions/publish";
private static final String UNLOCK_URL_PRE="deposit/depositions";
private static final String PUBLISH_URL_POST="actions/publish";
private static final String NEW_VERSION_URL_POST="actions/newversion";
private static final String UNLOCK_URL_POST="actions/edit";
private static final String ACCESS_TOKEN="access_token";
private static ObjectMapper mapper = new ObjectMapper();
@ -133,9 +137,30 @@ public class Zenodo {
//Download locally into temp
URL url=new URL(urlString);
temp=File.createTempFile("zenodo_", ".tmp");
// Multiple tries
InputStream is=null;
int attempt=0;
Exception lastException=null;
while(is==null&&attempt<5) {
try {
attempt++;
is=url.openStream();
}catch(Exception e) {
lastException=e;
try{
Thread.sleep(500*attempt);
}catch(InterruptedException e1) {}
}
}
if(is==null) throw new Exception("Unable to download "+urlString,lastException);
// Download
long size=Files.copy(is, temp.toPath(),StandardCopyOption.REPLACE_EXISTING);
long size=Files.copy(url.openStream(), temp.toPath(),StandardCopyOption.REPLACE_EXISTING);
//upload
FormDataMultiPart multi=new FormDataMultiPart();
FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file",
@ -178,6 +203,49 @@ public class Zenodo {
}
public void deleteFile(Integer depositionId,FileDeposition toDelete) throws ZenodoException {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).
path(depositionId+"").
path("files").
path(toDelete.getId()).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.delete();
try {
check(resp,null);
}catch(ZenodoException z) {
throw z;
}catch(Throwable t) {
throw new ZenodoException(t.getMessage(),t);
}
}
public ZenodoDeposition newVersion(Integer originalId) throws ZenodoException {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).
path(originalId+"").
path(NEW_VERSION_URL_POST).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.post(Entity.json("{}"));
ZenodoDeposition d=check(resp,ZenodoDeposition.class);
//GET LATEST DRAFT
resp=getWebClient().target(d.getLinks().getLatest_draft()).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE).get();
try {
return check(resp,ZenodoDeposition.class);
}catch(ZenodoException z) {
throw z;
}catch(Throwable t) {
throw new ZenodoException(t.getMessage(),t);
}
}
private ZenodoDeposition updateMetadata(Integer depositionId,DepositionMetadata meta) throws ZenodoException {
try{
String serialized="{\"metadata\":"+Fixer.fixIncoming(mapper.writeValueAsString(meta))+"}";
@ -199,7 +267,7 @@ public class Zenodo {
public ZenodoDeposition unlockPublished(Integer depositionId) throws ZenodoException {
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(UNLOCK_URL_PRE).
path(DEPOSITION_BASE_URL).
path(depositionId+"").
path(UNLOCK_URL_POST).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
@ -229,7 +297,7 @@ public class Zenodo {
private ZenodoDeposition publish(Integer depositionId) throws ZenodoException{
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(PUBLISH_URL_PRE).
path(DEPOSITION_BASE_URL).
path(depositionId+"").
path(PUBLISH_URL_POST).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)

7
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/UploadPolicy.java Normal file
View File

@ -0,0 +1,7 @@
package org.gcube.data.publishing.ckan2zenodo.model;
public enum UploadPolicy {
SKIP_EXISTING,DELETE_EXISTING,DELETE_ALL
}

2
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/DepositionLinks.java
View File

@ -18,5 +18,5 @@ public class DepositionLinks {
private String latest;
private String latest_html;
private String self;
private String latest_draft;
}