gCubeSystem
/
ecological-engine-smart-executor
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Data transfer 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@117753 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2015-09-01 12:04:31 +00:00
parent f5f43f8af4
commit e308cdd888
1 changed files with 110 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java Normal file
View File

@ -0,0 +1,110 @@
package org.gcube.dataanalysis.executor.util;
import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent;
import java.io.File;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.datatransfer.agent.library.AgentLibrary;
import org.gcube.datatransfer.common.agent.Types.storageType;
import org.gcube.datatransfer.common.options.TransferOptions;
import org.gcube.datatransfer.common.outcome.FileTransferOutcome;
public class DataTransferer {
public static void main(String[] args) throws Exception{
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps");
String transferGHN = "dewn04.madgik.di.uoa.gr";
int transferPort = 8080;
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
ArrayList<URI> input = new ArrayList<URI>();
/*
File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif");
String file = "wind1.tif";
String localfolder = "C:/Users/coro/Dropbox/Public/";
String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
System.out.println("URI from storage: "+storagesmpurl);
String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
*/
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
System.out.println("URL for storage: "+urlStorage);
// URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
URI uri = new URI(urlStorage);
//http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
// URI uri = new URI(storageurl); //localfile.toURI();
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
input.add(uri);
String outPath = "/tmp";
TransferOptions options = new TransferOptions();
options = new TransferOptions();
options.setOverwriteFile(true);
options.setType(storageType.LocalGHN);
options.setUnzipFile(false);
ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, outPath, options);
for (FileTransferOutcome outcome:outcomes){
System.out.println(outcome);
System.out.println(outcome.fileName());
System.out.println(outcome.isSuccess());
System.out.println(outcome.getTotal_size());
System.out.println(outcome.getTransferredBytes());
System.out.println(outcome.getTransferTime());
}
}
//returns the number of transferred bytes
public static Long transferFileToService(String scope, String username, String service,int port, String fileAbsolutePath, String remoteFolder) throws Exception{
AnalysisLogger.getLogger().debug("Transferring file "+fileAbsolutePath+" to "+service+":"+port);
ScopeProvider.instance.set(scope);
AgentLibrary library = transferAgent().at(service, port).build();
ArrayList<URI> input = new ArrayList<URI>();
String localfolder = new File(fileAbsolutePath).getParent();
String file = new File(fileAbsolutePath).getName();
AnalysisLogger.getLogger().debug("Uploading file "+file+" onto storage");
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username,localfolder,file);
AnalysisLogger.getLogger().debug("SMP url generated: "+storagesmpurl);
URI uri = new URI(storagesmpurl);
input.add(uri);
TransferOptions options = new TransferOptions();
options = new TransferOptions();
options.setOverwriteFile(true);
options.setType(storageType.LocalGHN);
options.setUnzipFile(false);
options.setTransferTimeout(3, TimeUnit.HOURS);
AnalysisLogger.getLogger().debug("Transferring...");
ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
AnalysisLogger.getLogger().debug("Transferring complete");
Long bytes = 0L;
for (FileTransferOutcome outcome:outcomes){
AnalysisLogger.getLogger().debug("Outcome "+outcome);
AnalysisLogger.getLogger().debug("Transferred file name "+outcome.fileName());
AnalysisLogger.getLogger().debug("Transferring success "+outcome.isSuccess());
AnalysisLogger.getLogger().debug("Transferred bytes "+outcome.getTotal_size());
AnalysisLogger.getLogger().debug("Transfer time "+outcome.getTransferTime());
bytes = outcome.getTotal_size();
}
return bytes;
}
}