MD5 Management

src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2Zenodo.java

@@ -70,17 +70,6 @@ public interface Ckan2Zenodo {
 	 */
 	public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload,ZenodoDeposition deposition) throws ZenodoException, ConfigurationException;
 	
-	/**
-	 * Uploads @param toUpload resources associating them to given @param deposition
-	 * 
-	 * @param toUpload
-	 * @param deposition
-	 * @param policy
-	 * @return
-	 * @throws ZenodoException 
-	 * @throws ConfigurationException 
-	 */
-	public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload,ZenodoDeposition deposition,UploadPolicy policy) throws ZenodoException, ConfigurationException;
 	
 	/**
 	 * Publishes @param dep, setting/updateing DOI reference into @param toUpdate

src/main/java/org/gcube/data/publishing/ckan2zenodo/Ckan2ZenodoImpl.java

@@ -51,7 +51,7 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
 		CkanRelatedIdentifier doi=desc.getZenodoDoi();
 		if(doi!=null) {
 			Zenodo z=getZenodo();
-			toUpdate=z.readDeposition(doi.getZenodoId());
+			toUpdate=z.readDeposition(doi.getZenodoId());			
 		}
 		Translator tr=new TransformerManager().getByProfile(desc.getProfile());
 		return tr.transform(desc, toUpdate);
@@ -84,22 +84,16 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
 
 	@Override
 	public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload, ZenodoDeposition deposition) throws ZenodoException, ConfigurationException {
-		return uploadFiles(toUpload, deposition, UploadPolicy.DELETE_ALL);
-	}
-
-	@Override
-	public Future<ZenodoDeposition> uploadFiles(Set<CkanResource> toUpload, ZenodoDeposition deposition,
-			UploadPolicy policy) throws ZenodoException, ConfigurationException {
 		final Zenodo z=getZenodo();
 		 
-		if(deposition.getSubmitted())
+		UploadFilesCall call=new UploadFilesCall(toUpload,deposition,z);
-			deposition=z.newVersion(deposition.getId());
-
-		UploadFilesCall call=new UploadFilesCall(policy,toUpload,deposition,z);
 
 		return FileUploaderManager.submitForDeposition(call);
+		
 	}
 
+	
+
 	@Override
 	public ZenodoDeposition publish(ZenodoDeposition dep, CkanItemDescriptor toUpdate) throws ZenodoException, ConfigurationException, InvalidItemException, MalformedURLException {
 		Zenodo z=getZenodo();

src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/UploadFilesCall.java

@@ -1,26 +1,30 @@
 package org.gcube.data.publishing.ckan2zenodo.clients;
 
+import java.nio.file.Files;
 import java.util.Collection;
 import java.util.concurrent.Callable;
 
+import org.gcube.data.publishing.ckan2zenodo.commons.Net;
 import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
-import org.gcube.data.publishing.ckan2zenodo.model.UploadPolicy;
+import org.gcube.data.publishing.ckan2zenodo.model.DownloadedFile;
 import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition;
 import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
 
+import lombok.extern.slf4j.Slf4j;
+
+
+@Slf4j
 public class UploadFilesCall implements Callable<ZenodoDeposition>{
 
-	private UploadPolicy policy;
 	private Collection<CkanResource> toUpload;
 	private ZenodoDeposition deposition;
 	private Zenodo z;
-	
+
-	
+
-	
+
-	public UploadFilesCall(UploadPolicy policy, Collection<CkanResource> toUpload, ZenodoDeposition deposition,
+	public UploadFilesCall(Collection<CkanResource> toUpload, ZenodoDeposition deposition,
 			Zenodo z) {
 		super();
-		this.policy = policy;
 		this.toUpload = toUpload;
 		this.deposition = deposition;
 		this.z = z;
@@ -31,51 +35,69 @@ public class UploadFilesCall implements Callable<ZenodoDeposition>{
 	@Override
 	public ZenodoDeposition call() throws Exception {
 		ZenodoDeposition dep=deposition;
-		
-			if(policy.equals(UploadPolicy.DELETE_ALL)) {
-				for(FileDeposition f:dep.getFiles()) {
-					try{
-						z.deleteFile(dep.getId(), f);
-					}catch(Throwable t) {
-						throw new Exception("Unable to delete "+f,t);
-					}
-				}
-			}
 
+		log.debug("Starting file transfer for deposition "+deposition.getTitle()+" id : "+deposition.getId());
 
-			for(CkanResource r:toUpload) {
+		log.debug("Removing not referenced files..");
-				try {
+
-				switch(policy) {
+		for(FileDeposition f:dep.getFiles()) {
-				case SKIP_EXISTING : {
+			CkanResource found=null;
-					boolean found=false;
+			for(CkanResource r:toUpload)
-					for(FileDeposition f:dep.getFiles()) {
+				if(r.getName().equals(f.getFilename())) {
-						if(f.getFilename().equals(r.getName())) {
+					found=r;
-							found=true;
-							break;
-						}
-					}						
-					if(!found) z.uploadFile(dep, r.getName(), r.getUrl());
 					break;
 				}
-				case DELETE_EXISTING : {
+			if(found==null) // File not present in current toUpload set
-					for(FileDeposition f:dep.getFiles()) {
+				try{
-						if(f.getFilename().equals(r.getName())) {
+					log.debug("Remote file "+f+" is not in requested set. Deleting it..");
-							z.deleteFile(dep.getId(), f);
+					z.deleteFile(dep, f);
-							break;
+				}catch(Throwable t) {
-						}
+					throw new Exception("Unable to delete "+f,t);
-					}
-					// continue with default
 				}
-				default : {
+			else {
-					z.uploadFile(dep, r.getName(), r.getUrl());
+					 // File present, checking for update
+				DownloadedFile downloaded=null;
+				try {
+					log.debug("Found already existing remote file "+f);
+					downloaded=Net.download(found);
+					if(!downloaded.getMD5().equals(f.getChecksum())) {
+						log.debug("MD5 differ, going to update : "+downloaded+" - "+f);
+						z.deleteFile(dep, f);
+						z.uploadFile(dep, found.getName(), downloaded.getF());
+					}	
+				}catch(Throwable t) {
+					throw new Exception("Unable to update "+f,t);
+				}finally {
+					if(downloaded!=null) Files.deleteIfExists(downloaded.getF().toPath());
 				}
+			}
+		}
 
-				}
+
+		log.debug("Going to push additional resources for "+deposition.getTitle()+" ID : "+deposition.getId());				
+
+		for(CkanResource r:toUpload) {
+			DownloadedFile downloaded=null;
+			try {
+				boolean found=false;				
+
+				for(FileDeposition f:dep.getFiles())
+					if(f.getFilename().equals(r.getName())) {						
+						found=true;													
+						break;
+					}
+				if(!found) {
+					downloaded=Net.download(r);
+					z.uploadFile(dep, r.getName(),downloaded.getF());				
+				}				
 			}catch(Throwable t) {
 				throw new Exception("Unable to upload "+r.getName()+".",t); 
 			}
-			}
+		}
 		return z.readDeposition(dep.getId());
 	}
-	
+
+
+
+
 }

src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/Zenodo.java

@@ -131,49 +131,23 @@ public class Zenodo {
 		return updateMetadata(dep.getId(), dep.getMetadata());
 	}
 
-	public FileDeposition uploadFile(ZenodoDeposition dep, String toUploadName,String urlString) throws ZenodoException {
+	public FileDeposition uploadFile(ZenodoDeposition deposition, String toUploadName,File toUpload) throws ZenodoException {
-
+		final ZenodoDeposition dep=(deposition.getSubmitted())?newVersion(deposition.getId()):deposition;
+				
+		
 		Callable<Response> call=new Callable<Response>() {
 			@Override
-			public Response call() throws Exception {
+			public Response call() throws Exception {				
-				File temp=null;
 				try {
-					log.debug("Downloading "+urlString);
-					//Download locally into temp
-					URL url=new URL(urlString);
-					temp=File.createTempFile("zenodo_", ".tmp");
-					
-					
-					// Multiple tries
-					InputStream is=null;
-					int attempt=0;
-					Exception lastException=null;
-					while(is==null&&attempt<5) {
-						try {
-							attempt++;
-							is=url.openStream();
-						}catch(Exception e) {
-							lastException=e;
-							try{
-								Thread.sleep(500*attempt);
-							}catch(InterruptedException e1) {}
-						}
-					}
-					if(is==null) throw new Exception("Unable to download "+urlString,lastException);
-					
-					// Download
-					long size=Files.copy(is, temp.toPath(),StandardCopyOption.REPLACE_EXISTING);
-
-					
 					
 					//upload
 					FormDataMultiPart multi=new FormDataMultiPart();
 					FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file",
-							temp,MediaType.APPLICATION_OCTET_STREAM_TYPE);
+							toUpload,MediaType.APPLICATION_OCTET_STREAM_TYPE);
 					multi.field("name", toUploadName);
 					multi.bodyPart(fileDataBodyPart);
 
-					log.debug("Starting transfer of "+toUploadName+" ("+urlString+") into "+dep.getId());
+					log.debug("Starting transfer of "+toUploadName+" into "+dep.getId());
 					Response toReturn=getWebClient().target(credentials.getBaseUrl()).
 					path(DEPOSITION_BASE_URL).path(dep.getId()+"").path("files").
 					queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
@@ -183,17 +157,13 @@ public class Zenodo {
 					return toReturn;
 					
 				}catch(Throwable e) {
-					throw new ZenodoException("Unable to transfer file "+toUploadName+" url : "+urlString,e);			
+					throw new ZenodoException("Unable to transfer file "+toUploadName,e);			
-				}finally {
-					//finally delete temp
-					if(temp!=null) Files.deleteIfExists(temp.toPath());						
 				}
-
 			}
 		};
 		
 		
-		log.debug("Submitting request to upload "+urlString+" to Manager");
+		log.debug("Submitting request to upload "+toUploadName+" to Manager");
 		Future<Response> resp=FileUploaderManager.submitForResponse(call);
 		
 		try {
@@ -209,10 +179,13 @@ public class Zenodo {
 	}
 
 	
-	public void deleteFile(Integer depositionId,FileDeposition toDelete) throws ZenodoException {
+	public void deleteFile(ZenodoDeposition dep,FileDeposition toDelete) throws ZenodoException {
+		if(dep.getSubmitted())
+			dep=newVersion(dep.getId());
+		
 		Response resp = getWebClient().target(credentials.getBaseUrl()).
 				path(DEPOSITION_BASE_URL).
-				path(depositionId+"").
+				path(dep.getId()+"").
 				path("files").
 				path(toDelete.getId()).
 				queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)

src/main/java/org/gcube/data/publishing/ckan2zenodo/commons/Net.java

@@ -0,0 +1,56 @@
+package org.gcube.data.publishing.ckan2zenodo.commons;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+
+import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
+import org.gcube.data.publishing.ckan2zenodo.model.DownloadedFile;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class Net {
+
+	public static DownloadedFile download(CkanResource toDownload) throws Exception {
+		String urlString=toDownload.getUrl();
+		log.debug("Downloading "+urlString);
+		//Download locally into temp
+		URL url=new URL(urlString);
+		File temp=File.createTempFile("zenodo_", ".tmp");
+		MessageDigest md = MessageDigest.getInstance("MD5");
+		
+		// Multiple tries
+		InputStream is=null;
+		int attempt=0;
+		Exception lastException=null;
+		while(is==null&&attempt<5) {
+			try {
+				attempt++;
+				is=url.openStream();
+			}catch(Exception e) {
+				lastException=e;
+				try{
+					Thread.sleep(500*attempt);
+				}catch(InterruptedException e1) {}
+			}
+		}
+		if(is==null) throw new Exception("Unable to download "+urlString,lastException);
+		
+		DigestInputStream dis = new DigestInputStream(is, md);
+		
+		
+		// Download
+		long size=Files.copy(is, temp.toPath(),StandardCopyOption.REPLACE_EXISTING);
+		
+		return new DownloadedFile(toDownload,temp,dis.getMessageDigest().toString());
+		
+		
+	}
+	
+	
+}

src/main/java/org/gcube/data/publishing/ckan2zenodo/model/DownloadedFile.java

@@ -0,0 +1,20 @@
+package org.gcube.data.publishing.ckan2zenodo.model;
+
+import java.io.File;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@ToString
+public class DownloadedFile {
+
+	private CkanResource source;
+	private File f;
+	private String MD5;
+	
+}

src/main/java/org/gcube/data/publishing/ckan2zenodo/model/zenodo/ZenodoDeposition.java

@@ -8,8 +8,10 @@ import java.util.Date;
 import com.fasterxml.jackson.annotation.JsonFormat;
 
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 
 @Data
+@Slf4j
 public class ZenodoDeposition {
 
 	@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
@@ -29,9 +31,10 @@ public class ZenodoDeposition {
 	private String title;
 	
 	public URL getDOIUrl() throws MalformedURLException {
-		try {
+		try {			
 			return new URL(links.getConceptdoi());
 		}catch(Throwable t) {
+			log.warn("Returned concept link is broken, forming it from doi..");
 			return new URL("https://doi.org/"+doi);
 		}
 	}

src/test/java/org/gcube/tests/ZenodoTests.java

@@ -9,9 +9,11 @@ import org.gcube.common.resources.gcore.Resources;
 import org.gcube.data.publishing.ckan2zenodo.Translator;
 import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
 import org.gcube.data.publishing.ckan2zenodo.commons.IS;
+import org.gcube.data.publishing.ckan2zenodo.commons.Net;
 import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
 import org.gcube.data.publishing.ckan2zenodo.model.CkanRelatedIdentifier;
 import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
+import org.gcube.data.publishing.ckan2zenodo.model.DownloadedFile;
 import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
 import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
 import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
@@ -47,7 +49,7 @@ public class ZenodoTests {
 		
 		Zenodo z=new Zenodo(credentials);
 		
-		System.out.println(z.readDeposition(426312));
+		System.out.println(z.readDeposition(472904));
 		
 		
 	}
@@ -84,7 +86,8 @@ public class ZenodoTests {
 		CkanItemDescriptor desc=new CkanItemDescriptor(json);
 		
 		for(CkanResource cRes:tran.filterResources(desc)) {
-			FileDeposition file=z.uploadFile(dep, cRes.getName(), cRes.getUrl());
+			DownloadedFile f=Net.download(cRes);
+			FileDeposition file=z.uploadFile(dep, cRes.getName(), f.getF());
 			System.out.println("Published "+file);
 		}
 				
@@ -111,10 +114,14 @@ public class ZenodoTests {
 		CkanItemDescriptor desc=new CkanItemDescriptor(json);
 		
 		for(CkanResource cRes:tran.filterResources(desc)) {
-			FileDeposition file=z.uploadFile(dep, cRes.getName(), cRes.getUrl());
+			DownloadedFile f=Net.download(cRes);
+			
+			FileDeposition file=z.uploadFile(dep, cRes.getName(), f.getF());
 			System.out.println("Published "+file);
 		}		
 		dep=z.publish(dep);
+		Assert.assertTrue("Invalid deposition state after publishing ",dep.getState().equals("done"));
+		Assert.assertTrue("Invalid submitted state after publishing", dep.getSubmitted());
 		
 		desc.setZenodoDoi(CkanRelatedIdentifier.getZenodo(dep.getDOIUrl()));