gCubeSystem
/
ecological-engine-smart-executor
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@118991 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2015-09-23 10:04:25 +00:00
parent f35dd3fb44
commit 5c1bd422e8
4 changed files with 155 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java
View File

@ -4,107 +4,132 @@ import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent
import java.io.File;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.datatransfer.agent.library.AgentLibrary;
import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException;
import org.gcube.datatransfer.common.agent.Types.storageType;
import org.gcube.datatransfer.common.options.TransferOptions;
import org.gcube.datatransfer.common.outcome.FileTransferOutcome;
import org.gcube.datatransfer.common.outcome.TransferStatus;
public class DataTransferer {
public static void main(String[] args) throws Exception{
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps");
public static void main(String[] args) throws Exception {
String scope = "/d4science.research-infrastructures.eu/gCubeApps";
ScopeProvider.instance.set(scope);
String transferGHN = "dewn04.madgik.di.uoa.gr";
int transferPort = 8080;
int transferPort = 8080;
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
ArrayList<URI> input = new ArrayList<URI>();
/*
File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif");
String file = "wind1.tif";
String localfolder = "C:/Users/coro/Dropbox/Public/";
String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
System.out.println("URI from storage: "+storagesmpurl);
String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
*/
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
* File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif"); String file = "wind1.tif"; String localfolder = "C:/Users/coro/Dropbox/Public/"; String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
*
* System.out.println("URI from storage: "+storagesmpurl);
*
* String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
*/
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
System.out.println("URL for storage: "+urlStorage);
System.out.println("URL for storage: " + urlStorage);
// URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
URI uri = new URI(urlStorage);
URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
// URI uri = new URI(urlStorage);
//http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
// http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
// URI uri = new URI(storageurl); //localfile.toURI();
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
// URI uri = new URI(storageurl); //localfile.toURI();
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
input.add(uri);
String outPath = "/tmp";
TransferOptions options = new TransferOptions();
options = new TransferOptions();
options.setOverwriteFile(true);
options.setType(storageType.LocalGHN);
options.setUnzipFile(false);
ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, outPath, options);
for (FileTransferOutcome outcome:outcomes){
System.out.println(outcome);
System.out.println(outcome.fileName());
System.out.println(outcome.isSuccess());
System.out.println(outcome.getTotal_size());
System.out.println(outcome.getTransferredBytes());
System.out.println(outcome.getTransferTime());
}
transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort, "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg", outPath);
}
//returns the number of transferred bytes
public static Long transferFileToService(String scope, String username, String service,int port, String fileAbsolutePath, String remoteFolder) throws Exception{
AnalysisLogger.getLogger().debug("Transferring file "+fileAbsolutePath+" to "+service+":"+port);
// returns the number of transferred bytes
public static boolean transferFileToService(String scope, String username, String service, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
AnalysisLogger.getLogger().debug("Transferring file " + fileAbsolutePath + " to " + service + ":" + port);
ScopeProvider.instance.set(scope);
AgentLibrary library = transferAgent().at(service, port).build();
ArrayList<URI> input = new ArrayList<URI>();
String localfolder = new File(fileAbsolutePath).getParent();
String file = new File(fileAbsolutePath).getName();
AnalysisLogger.getLogger().debug("Uploading file "+file+" onto storage");
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username,localfolder,file);
AnalysisLogger.getLogger().debug("SMP url generated: "+storagesmpurl);
File localFile = new File(fileAbsolutePath);
if (!localFile.exists())
throw new Exception("Local file does not exist: " + localFile);
String localfolder = localFile.getParent();
String file = localFile.getName();
AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, file);
//urls for testing
//storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file;
// String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc=";
// String storagesmpurl = "http://goo.gl/r6ggMA";
// String storagesmpurl = "http://dl.dropboxusercontent.com/u/12809149/3_Aquamaps.jpg";
AnalysisLogger.getLogger().debug("SMP url generated: " + storagesmpurl);
URI uri = new URI(storagesmpurl);
input.add(uri);
TransferOptions options = new TransferOptions();
TransferOptions options = new TransferOptions();
options = new TransferOptions();
options.setOverwriteFile(true);
options.setType(storageType.LocalGHN);
options.setOverwriteFile(false);
options.setType(storageType.DataStorage);
options.setUnzipFile(false);
options.setTransferTimeout(3, TimeUnit.HOURS);
AnalysisLogger.getLogger().debug("Transferring...");
ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
//old code for sync transfer
// ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
ArrayList<URI> outputURI = new ArrayList<URI>();
outputURI.add(new URI("file://"+remoteFolder+""+file));
AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0));
String transferId = library.startTransfer(input, outputURI, options);
TransferStatus transferStatus = null;
do {
try {
Thread.sleep(1000);
transferStatus = TransferStatus.valueOf(library.monitorTransfer(transferId));
} catch (MonitorTransferException e) {
e.printStackTrace();
}
} while (!transferStatus.hasCompleted());
ArrayList<FileTransferOutcome> outcomes = library.getTransferOutcomes(transferId, FileTransferOutcome.class);
AnalysisLogger.getLogger().debug("Transferring complete");
Long bytes = 0L;
for (FileTransferOutcome outcome:outcomes){
AnalysisLogger.getLogger().debug("Outcome "+outcome);
AnalysisLogger.getLogger().debug("Transferred file name "+outcome.fileName());
AnalysisLogger.getLogger().debug("Transferring success "+outcome.isSuccess());
AnalysisLogger.getLogger().debug("Transferred bytes "+outcome.getTotal_size());
AnalysisLogger.getLogger().debug("Transfer time "+outcome.getTransferTime());
bytes = outcome.getTotal_size();
boolean success = false;
String outcomeString = "";
for (FileTransferOutcome outcome : outcomes) {
AnalysisLogger.getLogger().debug("Outcome " + outcome);
outcomeString = outcome.toString();
AnalysisLogger.getLogger().debug("Transferred file name " + outcome.fileName());
AnalysisLogger.getLogger().debug("Transferring success " + outcome.isSuccess());
AnalysisLogger.getLogger().debug("Transferred bytes " + outcome.getTotal_size());
AnalysisLogger.getLogger().debug("Transfer time " + outcome.getTransferTime());
success = outcome.isSuccess();
}
return bytes;
if (!success)
throw new Exception("No Bytes were transferred to the Thredds server: "+outcomeString);
return success;
}
}

30
src/main/java/org/gcube/dataanalysis/executor/util/IfraRetrieval.java
View File

@ -1,30 +0,0 @@
package org.gcube.dataanalysis.executor.util;
import java.util.ArrayList;
import java.util.List;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
public class IfraRetrieval {
public static List<String> retrieveAddresses(String Category, String scope, String exclude) {
if (scope == null || scope.length() == 0)
return new ArrayList<String>();
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Setting Scope to " + scope+" and executing query");
ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
DiscoveryClient<String> client = ICFactory.client();
List<String> addresses = client.submit(query);
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Query to IS finished");
return addresses;
}
}

58
src/main/java/org/gcube/dataanalysis/executor/util/InfraRetrieval.java Normal file
View File

@ -0,0 +1,58 @@
package org.gcube.dataanalysis.executor.util;
import java.util.ArrayList;
import java.util.List;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
public class InfraRetrieval {
public static List<String> retrieveAddresses(String Category, String scope, String exclude) {
if (scope == null || scope.length() == 0)
return new ArrayList<String>();
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Setting Scope to " + scope+" and executing query");
ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
DiscoveryClient<String> client = ICFactory.client();
List<String> addresses = client.submit(query);
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Query to IS finished");
return addresses;
}
public static List<String> retrieveServiceAddress(String Category, String Name, String scope, String exclude) {
if (scope == null || scope.length() == 0)
return new ArrayList<String>();
ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile/Name/text() eq '" + Name+ "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
DiscoveryClient<String> client = ICFactory.client();
List<String> addresses = client.submit(query);
return addresses;
}
public static List<String> retrieveService(String service, String scope) {
if (scope == null || scope.length() == 0)
return new ArrayList<String>();
ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class);
query.addCondition("$resource/Profile/ServiceName/text() eq '"+service+"'").setResult("$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint/text()");
DiscoveryClient<String> client = ICFactory.client();
List<String> addresses = client.submit(query);
return addresses;
}
}