From e308cdd888dbf8d5f049255bbd4a917ca3323076 Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Tue, 1 Sep 2015 12:04:31 +0000 Subject: [PATCH] Data transfer git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@117753 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../executor/util/DataTransferer.java | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java diff --git a/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java b/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java new file mode 100644 index 0000000..c26f6c4 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java @@ -0,0 +1,110 @@ +package org.gcube.dataanalysis.executor.util; + +import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent; + +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.datatransfer.agent.library.AgentLibrary; +import org.gcube.datatransfer.common.agent.Types.storageType; +import org.gcube.datatransfer.common.options.TransferOptions; +import org.gcube.datatransfer.common.outcome.FileTransferOutcome; + +public class DataTransferer { + + public static void main(String[] args) throws Exception{ + + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps"); + String transferGHN = "dewn04.madgik.di.uoa.gr"; + int transferPort = 8080; + AgentLibrary library = transferAgent().at(transferGHN, transferPort).build(); + + ArrayList input = new ArrayList(); + /* + File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif"); + String file = "wind1.tif"; + String localfolder = "C:/Users/coro/Dropbox/Public/"; + String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file); + + System.out.println("URI from storage: "+storagesmpurl); + + String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file; + */ +// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif"; + String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc="; + + System.out.println("URL for storage: "+urlStorage); + +// URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif"); + URI uri = new URI(urlStorage); + + //http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff + +// URI uri = new URI(storageurl); //localfile.toURI(); +// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif"); + input.add(uri); + + String outPath = "/tmp"; + + TransferOptions options = new TransferOptions(); + + options = new TransferOptions(); + options.setOverwriteFile(true); + options.setType(storageType.LocalGHN); + options.setUnzipFile(false); + + ArrayList outcomes = library.startTransferSync(input, outPath, options); + for (FileTransferOutcome outcome:outcomes){ + System.out.println(outcome); + System.out.println(outcome.fileName()); + System.out.println(outcome.isSuccess()); + System.out.println(outcome.getTotal_size()); + System.out.println(outcome.getTransferredBytes()); + System.out.println(outcome.getTransferTime()); + } + } + + //returns the number of transferred bytes + public static Long transferFileToService(String scope, String username, String service,int port, String fileAbsolutePath, String remoteFolder) throws Exception{ + AnalysisLogger.getLogger().debug("Transferring file "+fileAbsolutePath+" to "+service+":"+port); + ScopeProvider.instance.set(scope); + AgentLibrary library = transferAgent().at(service, port).build(); + ArrayList input = new ArrayList(); + String localfolder = new File(fileAbsolutePath).getParent(); + String file = new File(fileAbsolutePath).getName(); + AnalysisLogger.getLogger().debug("Uploading file "+file+" onto storage"); + String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username,localfolder,file); + AnalysisLogger.getLogger().debug("SMP url generated: "+storagesmpurl); + + URI uri = new URI(storagesmpurl); + input.add(uri); + + TransferOptions options = new TransferOptions(); + + options = new TransferOptions(); + options.setOverwriteFile(true); + options.setType(storageType.LocalGHN); + options.setUnzipFile(false); + options.setTransferTimeout(3, TimeUnit.HOURS); + AnalysisLogger.getLogger().debug("Transferring..."); + ArrayList outcomes = library.startTransferSync(input, remoteFolder, options); + AnalysisLogger.getLogger().debug("Transferring complete"); + Long bytes = 0L; + for (FileTransferOutcome outcome:outcomes){ + AnalysisLogger.getLogger().debug("Outcome "+outcome); + AnalysisLogger.getLogger().debug("Transferred file name "+outcome.fileName()); + AnalysisLogger.getLogger().debug("Transferring success "+outcome.isSuccess()); + AnalysisLogger.getLogger().debug("Transferred bytes "+outcome.getTotal_size()); + AnalysisLogger.getLogger().debug("Transfer time "+outcome.getTransferTime()); + bytes = outcome.getTotal_size(); + } + return bytes; + } + + +}