gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

MD5 Management

This commit is contained in:
Fabio Sinibaldi 2020-01-31 14:56:51 +01:00
parent 1fd5ff53e9
commit a17af1d1a4
8 changed files with 173 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2Zenodo.java
View File

@ -70,17 +70,6 @@ public interface Ckan2Zenodo {
*/
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload,ZenodoDeposition deposition) throws ZenodoException, ConfigurationException;
/**
* Uploads @param toUpload resources associating them to given @param deposition
*
* @param toUpload
* @param deposition
* @param policy
* @return
* @throws ZenodoException
* @throws ConfigurationException
*/
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload,ZenodoDeposition deposition,UploadPolicy policy) throws ZenodoException, ConfigurationException;
/**
* Publishes @param dep, setting/updateing DOI reference into @param toUpdate

16
src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2ZenodoImpl.java
View File

@ -51,7 +51,7 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
CkanRelatedIdentifier doi=desc.getZenodoDoi();
if(doi!=null) {
Zenodo z=getZenodo();
toUpdate=z.readDeposition(doi.getZenodoId());
toUpdate=z.readDeposition(doi.getZenodoId());
}
Translator tr=new TransformerManager().getByProfile(desc.getProfile());
return tr.transform(desc, toUpdate);
@ -84,22 +84,16 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
@Override
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload, ZenodoDeposition deposition) throws ZenodoException, ConfigurationException {
return uploadFiles(toUpload, deposition, UploadPolicy.DELETE_ALL);
}
@Override
public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload, ZenodoDeposition deposition,
UploadPolicy policy) throws ZenodoException, ConfigurationException {
final Zenodo z=getZenodo();
if(deposition.getSubmitted())
deposition=z.newVersion(deposition.getId());
UploadFilesCall call=new UploadFilesCall(policy,toUpload,deposition,z);
UploadFilesCall call=new UploadFilesCall(toUpload,deposition,z);
return FileUploaderManager.submitForDeposition(call);
}
@Override
public ZenodoDeposition publish(ZenodoDeposition dep, CkanItemDescriptor toUpdate) throws ZenodoException, ConfigurationException, InvalidItemException, MalformedURLException {
Zenodo z=getZenodo();

106
src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/UploadFilesCall.java
View File

@ -1,26 +1,30 @@
package org.gcube.data.publishing.ckan2zenodo.clients;
import java.nio.file.Files;
import java.util.Collection;
import java.util.concurrent.Callable;
import org.gcube.data.publishing.ckan2zenodo.commons.Net;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.UploadPolicy;
import org.gcube.data.publishing.ckan2zenodo.model.DownloadedFile;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class UploadFilesCall implements Callable<ZenodoDeposition>{
private UploadPolicy policy;
private Collection<CkanResource> toUpload;
private ZenodoDeposition deposition;
private Zenodo z;
public UploadFilesCall(UploadPolicy policy, Collection<CkanResource> toUpload, ZenodoDeposition deposition,
public UploadFilesCall(Collection<CkanResource> toUpload, ZenodoDeposition deposition,
Zenodo z) {
super();
this.policy = policy;
this.toUpload = toUpload;
this.deposition = deposition;
this.z = z;
@ -31,51 +35,69 @@ public class UploadFilesCall implements Callable<ZenodoDeposition>{
@Override
public ZenodoDeposition call() throws Exception {
ZenodoDeposition dep=deposition;
if(policy.equals(UploadPolicy.DELETE_ALL)) {
for(FileDeposition f:dep.getFiles()) {
try{
z.deleteFile(dep.getId(), f);
}catch(Throwable t) {
throw new Exception("Unable to delete "+f,t);
}
}
}
log.debug("Starting file transfer for deposition "+deposition.getTitle()+" id : "+deposition.getId());
for(CkanResource r:toUpload) {
try {
switch(policy) {
case SKIP_EXISTING : {
boolean found=false;
for(FileDeposition f:dep.getFiles()) {
if(f.getFilename().equals(r.getName())) {
found=true;
break;
}
}
if(!found) z.uploadFile(dep, r.getName(), r.getUrl());
log.debug("Removing not referenced files..");
for(FileDeposition f:dep.getFiles()) {
CkanResource found=null;
for(CkanResource r:toUpload)
if(r.getName().equals(f.getFilename())) {
found=r;
break;
}
case DELETE_EXISTING : {
for(FileDeposition f:dep.getFiles()) {
if(f.getFilename().equals(r.getName())) {
z.deleteFile(dep.getId(), f);
break;
}
}
// continue with default
if(found==null) // File not present in current toUpload set
try{
log.debug("Remote file "+f+" is not in requested set. Deleting it..");
z.deleteFile(dep, f);
}catch(Throwable t) {
throw new Exception("Unable to delete "+f,t);
}
default : {
z.uploadFile(dep, r.getName(), r.getUrl());
else {
// File present, checking for update
DownloadedFile downloaded=null;
try {
log.debug("Found already existing remote file "+f);
downloaded=Net.download(found);
if(!downloaded.getMD5().equals(f.getChecksum())) {
log.debug("MD5 differ, going to update : "+downloaded+" - "+f);
z.deleteFile(dep, f);
z.uploadFile(dep, found.getName(), downloaded.getF());
}
}catch(Throwable t) {
throw new Exception("Unable to update "+f,t);
}finally {
if(downloaded!=null) Files.deleteIfExists(downloaded.getF().toPath());
}
}
}
}
log.debug("Going to push additional resources for "+deposition.getTitle()+" ID : "+deposition.getId());
for(CkanResource r:toUpload) {
DownloadedFile downloaded=null;
try {
boolean found=false;
for(FileDeposition f:dep.getFiles())
if(f.getFilename().equals(r.getName())) {
found=true;
break;
}
if(!found) {
downloaded=Net.download(r);
z.uploadFile(dep, r.getName(),downloaded.getF());
}
}catch(Throwable t) {
throw new Exception("Unable to upload "+r.getName()+".",t);
}
}
}
return z.readDeposition(dep.getId());
}
}

55
src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java
View File

@ -131,49 +131,23 @@ public class Zenodo {
return updateMetadata(dep.getId(), dep.getMetadata());
}
public FileDeposition uploadFile(ZenodoDeposition dep, String toUploadName,String urlString) throws ZenodoException {
public FileDeposition uploadFile(ZenodoDeposition deposition, String toUploadName,File toUpload) throws ZenodoException {
final ZenodoDeposition dep=(deposition.getSubmitted())?newVersion(deposition.getId()):deposition;
Callable<Response> call=new Callable<Response>() {
@Override
public Response call() throws Exception {
File temp=null;
public Response call() throws Exception {
try {
log.debug("Downloading "+urlString);
//Download locally into temp
URL url=new URL(urlString);
temp=File.createTempFile("zenodo_", ".tmp");
// Multiple tries
InputStream is=null;
int attempt=0;
Exception lastException=null;
while(is==null&&attempt<5) {
try {
attempt++;
is=url.openStream();
}catch(Exception e) {
lastException=e;
try{
Thread.sleep(500*attempt);
}catch(InterruptedException e1) {}
}
}
if(is==null) throw new Exception("Unable to download "+urlString,lastException);
// Download
long size=Files.copy(is, temp.toPath(),StandardCopyOption.REPLACE_EXISTING);
//upload
FormDataMultiPart multi=new FormDataMultiPart();
FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file",
temp,MediaType.APPLICATION_OCTET_STREAM_TYPE);
toUpload,MediaType.APPLICATION_OCTET_STREAM_TYPE);
multi.field("name", toUploadName);
multi.bodyPart(fileDataBodyPart);
log.debug("Starting transfer of "+toUploadName+" ("+urlString+") into "+dep.getId());
log.debug("Starting transfer of "+toUploadName+" into "+dep.getId());
Response toReturn=getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(dep.getId()+"").path("files").
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
@ -183,17 +157,13 @@ public class Zenodo {
return toReturn;
}catch(Throwable e) {
throw new ZenodoException("Unable to transfer file "+toUploadName+" url : "+urlString,e);
}finally {
//finally delete temp
if(temp!=null) Files.deleteIfExists(temp.toPath());
throw new ZenodoException("Unable to transfer file "+toUploadName,e);
}
}
};
log.debug("Submitting request to upload "+urlString+" to Manager");
log.debug("Submitting request to upload "+toUploadName+" to Manager");
Future<Response> resp=FileUploaderManager.submitForResponse(call);
try {
@ -209,10 +179,13 @@ public class Zenodo {
}
public void deleteFile(Integer depositionId,FileDeposition toDelete) throws ZenodoException {
public void deleteFile(ZenodoDeposition dep,FileDeposition toDelete) throws ZenodoException {
if(dep.getSubmitted())
dep=newVersion(dep.getId());
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).
path(depositionId+"").
path(dep.getId()+"").
path("files").
path(toDelete.getId()).
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)

56
src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/Net.java Normal file
View File

@ -0,0 +1,56 @@
package org.gcube.data.publishing.ckan2zenodo.commons;
import java.io.File;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.DownloadedFile;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Net {
public static DownloadedFile download(CkanResource toDownload) throws Exception {
String urlString=toDownload.getUrl();
log.debug("Downloading "+urlString);
//Download locally into temp
URL url=new URL(urlString);
File temp=File.createTempFile("zenodo_", ".tmp");
MessageDigest md = MessageDigest.getInstance("MD5");
// Multiple tries
InputStream is=null;
int attempt=0;
Exception lastException=null;
while(is==null&&attempt<5) {
try {
attempt++;
is=url.openStream();
}catch(Exception e) {
lastException=e;
try{
Thread.sleep(500*attempt);
}catch(InterruptedException e1) {}
}
}
if(is==null) throw new Exception("Unable to download "+urlString,lastException);
DigestInputStream dis = new DigestInputStream(is, md);
// Download
long size=Files.copy(is, temp.toPath(),StandardCopyOption.REPLACE_EXISTING);
return new DownloadedFile(toDownload,temp,dis.getMessageDigest().toString());
}
}

20
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/DownloadedFile.java Normal file
View File

@ -0,0 +1,20 @@
package org.gcube.data.publishing.ckan2zenodo.model;
import java.io.File;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@AllArgsConstructor
@ToString
public class DownloadedFile {
private CkanResource source;
private File f;
private String MD5;
}

5
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/ZenodoDeposition.java
View File

@ -8,8 +8,10 @@ import java.util.Date;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Data
@Slf4j
public class ZenodoDeposition {
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
@ -29,9 +31,10 @@ public class ZenodoDeposition {
private String title;
public URL getDOIUrl() throws MalformedURLException {
try {
try {
return new URL(links.getConceptdoi());
}catch(Throwable t) {
log.warn("Returned concept link is broken, forming it from doi..");
return new URL("https://doi.org/"+doi);
}
}

13
src/test/java/org/gcube/tests/ZenodoTests.java
View File

@ -9,9 +9,11 @@ import org.gcube.common.resources.gcore.Resources;
import org.gcube.data.publishing.ckan2zenodo.Translator;
import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
import org.gcube.data.publishing.ckan2zenodo.commons.IS;
import org.gcube.data.publishing.ckan2zenodo.commons.Net;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanRelatedIdentifier;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.DownloadedFile;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
@ -47,7 +49,7 @@ public class ZenodoTests {
Zenodo z=new Zenodo(credentials);
System.out.println(z.readDeposition(426312));
System.out.println(z.readDeposition(472904));
}
@ -84,7 +86,8 @@ public class ZenodoTests {
CkanItemDescriptor desc=new CkanItemDescriptor(json);
for(CkanResource cRes:tran.filterResources(desc)) {
FileDeposition file=z.uploadFile(dep, cRes.getName(), cRes.getUrl());
DownloadedFile f=Net.download(cRes);
FileDeposition file=z.uploadFile(dep, cRes.getName(), f.getF());
System.out.println("Published "+file);
}
@ -111,10 +114,14 @@ public class ZenodoTests {
CkanItemDescriptor desc=new CkanItemDescriptor(json);
for(CkanResource cRes:tran.filterResources(desc)) {
FileDeposition file=z.uploadFile(dep, cRes.getName(), cRes.getUrl());
DownloadedFile f=Net.download(cRes);
FileDeposition file=z.uploadFile(dep, cRes.getName(), f.getF());
System.out.println("Published "+file);
}
dep=z.publish(dep);
Assert.assertTrue("Invalid deposition state after publishing ",dep.getState().equals("done"));
Assert.assertTrue("Invalid submitted state after publishing", dep.getSubmitted());
desc.setZenodoDoi(CkanRelatedIdentifier.getZenodo(dep.getDOIUrl()));