package org.gcube.data.publishing.ckan2zenodo.clients; import java.nio.file.Files; import java.util.*; import java.util.concurrent.Callable; import org.gcube.data.publishing.ckan2zenodo.model.CkanResource; import org.gcube.data.publishing.ckan2zenodo.model.DownloadedFile; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.FileDeposition; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; import lombok.extern.slf4j.Slf4j; @Slf4j public class UploadFilesCall implements Callable{ private Collection toUpload; private ZenodoDeposition deposition; private Zenodo z; public UploadFilesCall(Collection toUpload, ZenodoDeposition deposition, Zenodo z) { super(); this.toUpload = toUpload; this.deposition = deposition; this.z = z; } @Override public ZenodoDeposition call() throws Exception { ZenodoDeposition dep=deposition; log.debug("Starting file transfer for deposition "+deposition.getTitle()+" id : "+deposition.getId()); log.debug("Removing not referenced files.."); HashMap resourceMap=new HashMap<>(); for(CkanResource r:toUpload) { DownloadedFile d=new DownloadedFile(r); resourceMap.put(d.getToUseFileName(),d); } Set alreadyExistingFiles=new HashSet<>(); for(FileDeposition f:dep.getFiles()) { CkanResource found=null; if(resourceMap.containsKey(f.getFilename())){ alreadyExistingFiles.add(f.getFilename()); try{ // check for update DownloadedFile downloaded = resourceMap.get(f.getFilename()); if(!downloaded.getMD5().equals(f.getChecksum())) { log.debug("MD5 differ, going to update : "+downloaded+" - "+f); z.deleteFile(dep, f); z.uploadFile(dep, found.getName(), downloaded.getFile()); } }catch (Throwable t){ log.warn("Unable to update "+f,t); } }else { try{ // remove File not present in current toUpload set log.debug("Remote file "+f+" is not in requested set. Deleting it.."); z.deleteFile(dep, f); }catch(Throwable t) { log.warn("Unable to delete "+f,t); } } } log.debug("Going to push additional resources for "+deposition.getTitle()+" ID : "+deposition.getId()); for(Map.Entry e : resourceMap.entrySet()){ DownloadedFile downloadedFile=e.getValue(); if(!alreadyExistingFiles.contains(e.getKey())) try{ // Upload new file z.uploadFile(dep,downloadedFile.getToUseFileName(),downloadedFile.getFile()); }catch(Throwable t) { log.warn("Unable to upload "+downloadedFile.getSource().getName(),t); } } return z.readDeposition(dep.getId()); } }