git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@118991 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
f35dd3fb44
commit
5c1bd422e8
|
@ -4,107 +4,132 @@ import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URL;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||||
import org.gcube.datatransfer.agent.library.AgentLibrary;
|
import org.gcube.datatransfer.agent.library.AgentLibrary;
|
||||||
|
import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException;
|
||||||
import org.gcube.datatransfer.common.agent.Types.storageType;
|
import org.gcube.datatransfer.common.agent.Types.storageType;
|
||||||
import org.gcube.datatransfer.common.options.TransferOptions;
|
import org.gcube.datatransfer.common.options.TransferOptions;
|
||||||
import org.gcube.datatransfer.common.outcome.FileTransferOutcome;
|
import org.gcube.datatransfer.common.outcome.FileTransferOutcome;
|
||||||
|
import org.gcube.datatransfer.common.outcome.TransferStatus;
|
||||||
|
|
||||||
public class DataTransferer {
|
public class DataTransferer {
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception{
|
public static void main(String[] args) throws Exception {
|
||||||
|
String scope = "/d4science.research-infrastructures.eu/gCubeApps";
|
||||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps");
|
ScopeProvider.instance.set(scope);
|
||||||
String transferGHN = "dewn04.madgik.di.uoa.gr";
|
String transferGHN = "dewn04.madgik.di.uoa.gr";
|
||||||
int transferPort = 8080;
|
int transferPort = 8080;
|
||||||
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
|
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
|
||||||
|
|
||||||
ArrayList<URI> input = new ArrayList<URI>();
|
ArrayList<URI> input = new ArrayList<URI>();
|
||||||
/*
|
/*
|
||||||
File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif");
|
* File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif"); String file = "wind1.tif"; String localfolder = "C:/Users/coro/Dropbox/Public/"; String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
|
||||||
String file = "wind1.tif";
|
*
|
||||||
String localfolder = "C:/Users/coro/Dropbox/Public/";
|
* System.out.println("URI from storage: "+storagesmpurl);
|
||||||
String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
|
*
|
||||||
|
* String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
|
||||||
System.out.println("URI from storage: "+storagesmpurl);
|
*/
|
||||||
|
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
|
||||||
String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
|
|
||||||
*/
|
|
||||||
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
|
|
||||||
String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
|
String urlStorage = "smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
|
||||||
|
|
||||||
System.out.println("URL for storage: "+urlStorage);
|
System.out.println("URL for storage: " + urlStorage);
|
||||||
|
|
||||||
// URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
|
URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
|
||||||
URI uri = new URI(urlStorage);
|
// URI uri = new URI(urlStorage);
|
||||||
|
|
||||||
//http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
|
// http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
|
||||||
|
|
||||||
// URI uri = new URI(storageurl); //localfile.toURI();
|
// URI uri = new URI(storageurl); //localfile.toURI();
|
||||||
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
|
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
|
||||||
input.add(uri);
|
input.add(uri);
|
||||||
|
|
||||||
String outPath = "/tmp";
|
String outPath = "/tmp";
|
||||||
|
|
||||||
TransferOptions options = new TransferOptions();
|
transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort, "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg", outPath);
|
||||||
|
|
||||||
options = new TransferOptions();
|
|
||||||
options.setOverwriteFile(true);
|
|
||||||
options.setType(storageType.LocalGHN);
|
|
||||||
options.setUnzipFile(false);
|
|
||||||
|
|
||||||
ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, outPath, options);
|
|
||||||
for (FileTransferOutcome outcome:outcomes){
|
|
||||||
System.out.println(outcome);
|
|
||||||
System.out.println(outcome.fileName());
|
|
||||||
System.out.println(outcome.isSuccess());
|
|
||||||
System.out.println(outcome.getTotal_size());
|
|
||||||
System.out.println(outcome.getTransferredBytes());
|
|
||||||
System.out.println(outcome.getTransferTime());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//returns the number of transferred bytes
|
// returns the number of transferred bytes
|
||||||
public static Long transferFileToService(String scope, String username, String service,int port, String fileAbsolutePath, String remoteFolder) throws Exception{
|
public static boolean transferFileToService(String scope, String username, String service, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
|
||||||
AnalysisLogger.getLogger().debug("Transferring file "+fileAbsolutePath+" to "+service+":"+port);
|
AnalysisLogger.getLogger().debug("Transferring file " + fileAbsolutePath + " to " + service + ":" + port);
|
||||||
ScopeProvider.instance.set(scope);
|
ScopeProvider.instance.set(scope);
|
||||||
|
|
||||||
AgentLibrary library = transferAgent().at(service, port).build();
|
AgentLibrary library = transferAgent().at(service, port).build();
|
||||||
ArrayList<URI> input = new ArrayList<URI>();
|
ArrayList<URI> input = new ArrayList<URI>();
|
||||||
String localfolder = new File(fileAbsolutePath).getParent();
|
File localFile = new File(fileAbsolutePath);
|
||||||
String file = new File(fileAbsolutePath).getName();
|
if (!localFile.exists())
|
||||||
AnalysisLogger.getLogger().debug("Uploading file "+file+" onto storage");
|
throw new Exception("Local file does not exist: " + localFile);
|
||||||
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username,localfolder,file);
|
|
||||||
AnalysisLogger.getLogger().debug("SMP url generated: "+storagesmpurl);
|
|
||||||
|
|
||||||
|
String localfolder = localFile.getParent();
|
||||||
|
String file = localFile.getName();
|
||||||
|
AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
|
||||||
|
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, file);
|
||||||
|
//urls for testing
|
||||||
|
//storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file;
|
||||||
|
// String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc=";
|
||||||
|
// String storagesmpurl = "http://goo.gl/r6ggMA";
|
||||||
|
// String storagesmpurl = "http://dl.dropboxusercontent.com/u/12809149/3_Aquamaps.jpg";
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("SMP url generated: " + storagesmpurl);
|
||||||
URI uri = new URI(storagesmpurl);
|
URI uri = new URI(storagesmpurl);
|
||||||
input.add(uri);
|
input.add(uri);
|
||||||
|
|
||||||
TransferOptions options = new TransferOptions();
|
TransferOptions options = new TransferOptions();
|
||||||
|
|
||||||
options = new TransferOptions();
|
options = new TransferOptions();
|
||||||
options.setOverwriteFile(true);
|
options.setOverwriteFile(false);
|
||||||
options.setType(storageType.LocalGHN);
|
options.setType(storageType.DataStorage);
|
||||||
options.setUnzipFile(false);
|
options.setUnzipFile(false);
|
||||||
options.setTransferTimeout(3, TimeUnit.HOURS);
|
options.setTransferTimeout(3, TimeUnit.HOURS);
|
||||||
AnalysisLogger.getLogger().debug("Transferring...");
|
AnalysisLogger.getLogger().debug("Transferring...");
|
||||||
ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
|
|
||||||
|
//old code for sync transfer
|
||||||
|
// ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
|
||||||
|
|
||||||
|
|
||||||
|
ArrayList<URI> outputURI = new ArrayList<URI>();
|
||||||
|
outputURI.add(new URI("file://"+remoteFolder+""+file));
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0));
|
||||||
|
|
||||||
|
String transferId = library.startTransfer(input, outputURI, options);
|
||||||
|
|
||||||
|
TransferStatus transferStatus = null;
|
||||||
|
|
||||||
|
do {
|
||||||
|
try {
|
||||||
|
|
||||||
|
Thread.sleep(1000);
|
||||||
|
transferStatus = TransferStatus.valueOf(library.monitorTransfer(transferId));
|
||||||
|
|
||||||
|
} catch (MonitorTransferException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
} while (!transferStatus.hasCompleted());
|
||||||
|
|
||||||
|
ArrayList<FileTransferOutcome> outcomes = library.getTransferOutcomes(transferId, FileTransferOutcome.class);
|
||||||
|
|
||||||
AnalysisLogger.getLogger().debug("Transferring complete");
|
AnalysisLogger.getLogger().debug("Transferring complete");
|
||||||
Long bytes = 0L;
|
boolean success = false;
|
||||||
for (FileTransferOutcome outcome:outcomes){
|
String outcomeString = "";
|
||||||
AnalysisLogger.getLogger().debug("Outcome "+outcome);
|
for (FileTransferOutcome outcome : outcomes) {
|
||||||
AnalysisLogger.getLogger().debug("Transferred file name "+outcome.fileName());
|
AnalysisLogger.getLogger().debug("Outcome " + outcome);
|
||||||
AnalysisLogger.getLogger().debug("Transferring success "+outcome.isSuccess());
|
outcomeString = outcome.toString();
|
||||||
AnalysisLogger.getLogger().debug("Transferred bytes "+outcome.getTotal_size());
|
AnalysisLogger.getLogger().debug("Transferred file name " + outcome.fileName());
|
||||||
AnalysisLogger.getLogger().debug("Transfer time "+outcome.getTransferTime());
|
AnalysisLogger.getLogger().debug("Transferring success " + outcome.isSuccess());
|
||||||
bytes = outcome.getTotal_size();
|
AnalysisLogger.getLogger().debug("Transferred bytes " + outcome.getTotal_size());
|
||||||
|
AnalysisLogger.getLogger().debug("Transfer time " + outcome.getTransferTime());
|
||||||
|
success = outcome.isSuccess();
|
||||||
}
|
}
|
||||||
return bytes;
|
|
||||||
|
if (!success)
|
||||||
|
throw new Exception("No Bytes were transferred to the Thredds server: "+outcomeString);
|
||||||
|
|
||||||
|
return success;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,30 +0,0 @@
|
||||||
package org.gcube.dataanalysis.executor.util;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
|
||||||
import org.gcube.common.scope.api.ScopeProvider;
|
|
||||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
|
||||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
|
||||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|
||||||
import org.gcube.resources.discovery.icclient.ICFactory;
|
|
||||||
|
|
||||||
public class IfraRetrieval {
|
|
||||||
|
|
||||||
public static List<String> retrieveAddresses(String Category, String scope, String exclude) {
|
|
||||||
if (scope == null || scope.length() == 0)
|
|
||||||
return new ArrayList<String>();
|
|
||||||
|
|
||||||
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Setting Scope to " + scope+" and executing query");
|
|
||||||
ScopeProvider.instance.set(scope);
|
|
||||||
|
|
||||||
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
|
|
||||||
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
|
|
||||||
DiscoveryClient<String> client = ICFactory.client();
|
|
||||||
List<String> addresses = client.submit(query);
|
|
||||||
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Query to IS finished");
|
|
||||||
return addresses;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -0,0 +1,58 @@
|
||||||
|
package org.gcube.dataanalysis.executor.util;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.gcube.common.resources.gcore.GCoreEndpoint;
|
||||||
|
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||||
|
import org.gcube.common.scope.api.ScopeProvider;
|
||||||
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||||
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||||
|
import org.gcube.resources.discovery.icclient.ICFactory;
|
||||||
|
|
||||||
|
public class InfraRetrieval {
|
||||||
|
|
||||||
|
public static List<String> retrieveAddresses(String Category, String scope, String exclude) {
|
||||||
|
if (scope == null || scope.length() == 0)
|
||||||
|
return new ArrayList<String>();
|
||||||
|
|
||||||
|
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Setting Scope to " + scope+" and executing query");
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
|
||||||
|
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
|
||||||
|
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
|
||||||
|
DiscoveryClient<String> client = ICFactory.client();
|
||||||
|
List<String> addresses = client.submit(query);
|
||||||
|
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Query to IS finished");
|
||||||
|
return addresses;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<String> retrieveServiceAddress(String Category, String Name, String scope, String exclude) {
|
||||||
|
if (scope == null || scope.length() == 0)
|
||||||
|
return new ArrayList<String>();
|
||||||
|
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
|
||||||
|
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
|
||||||
|
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile/Name/text() eq '" + Name+ "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
|
||||||
|
DiscoveryClient<String> client = ICFactory.client();
|
||||||
|
List<String> addresses = client.submit(query);
|
||||||
|
|
||||||
|
return addresses;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<String> retrieveService(String service, String scope) {
|
||||||
|
if (scope == null || scope.length() == 0)
|
||||||
|
return new ArrayList<String>();
|
||||||
|
|
||||||
|
ScopeProvider.instance.set(scope);
|
||||||
|
|
||||||
|
SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class);
|
||||||
|
query.addCondition("$resource/Profile/ServiceName/text() eq '"+service+"'").setResult("$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint/text()");
|
||||||
|
DiscoveryClient<String> client = ICFactory.client();
|
||||||
|
List<String> addresses = client.submit(query);
|
||||||
|
|
||||||
|
return addresses;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue