git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@148375 82a268e6-3cf1-43bd-a215-b396298e98cf
parent
3d03f95946
commit
8a05e571d4
@ -1,9 +1,31 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
|
||||
@ -1,13 +1,12 @@
|
||||
#Fri Jun 22 18:05:41 CEST 2012
|
||||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
||||
org.eclipse.jdt.core.compiler.compliance=1.6
|
||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
||||
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
||||
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||
org.eclipse.jdt.core.compiler.source=1.6
|
||||
org.eclipse.jdt.core.compiler.source=1.7
|
||||
|
||||
@ -1,155 +1,49 @@
|
||||
package org.gcube.dataanalysis.executor.util;
|
||||
|
||||
import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.contentmanagement.blobstorage.service.IClient;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
|
||||
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.datatransfer.agent.library.AgentLibrary;
|
||||
import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException;
|
||||
import org.gcube.datatransfer.common.agent.Types.storageType;
|
||||
import org.gcube.datatransfer.common.options.TransferOptions;
|
||||
import org.gcube.datatransfer.common.outcome.FileTransferOutcome;
|
||||
import org.gcube.datatransfer.common.outcome.TransferStatus;
|
||||
|
||||
public class DataTransferer {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
//String scope = "/d4science.research-infrastructures.eu/gCubeApps";
|
||||
|
||||
String scope = "/gcube/devsec/devVRE";
|
||||
//test only ScopeProvider.instance.set(scope);
|
||||
//String transferGHN = "dewn04.madgik.di.uoa.gr";
|
||||
String transferGHN = "access.d4science.org";
|
||||
int transferPort = 8080;
|
||||
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
|
||||
|
||||
ArrayList<URI> input = new ArrayList<URI>();
|
||||
/*
|
||||
* File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif"); String file = "wind1.tif"; String localfolder = "C:/Users/coro/Dropbox/Public/"; String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
|
||||
*
|
||||
* System.out.println("URI from storage: "+storagesmpurl);
|
||||
*
|
||||
* String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
|
||||
*/
|
||||
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
|
||||
//String urlStorage = "http://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
|
||||
String urlStorage ="http://goo.gl/Vq8QVY";
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
System.out.println("URL for storage: " + urlStorage);
|
||||
import org.gcube.data.transfer.library.DataTransferClient;
|
||||
import org.gcube.data.transfer.library.TransferResult;
|
||||
import org.gcube.data.transfer.model.Destination;
|
||||
import org.gcube.data.transfer.model.PluginInvocation;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
|
||||
// URI uri = new URI(urlStorage);
|
||||
|
||||
// http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
|
||||
|
||||
// URI uri = new URI(storageurl); //localfile.toURI();
|
||||
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
|
||||
input.add(uri);
|
||||
public class DataTransferer {
|
||||
|
||||
//String outPath = "/tmp";
|
||||
String outPath = "/var/www/html/test/";
|
||||
String fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
|
||||
//fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
|
||||
// fileToTransfer = "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/gcube/images/Resource Model.png";
|
||||
transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort,fileToTransfer , outPath);
|
||||
|
||||
|
||||
}
|
||||
private static final Logger logger = LoggerFactory.getLogger(DataTransferer.class);
|
||||
|
||||
// returns the number of transferred bytes
|
||||
public static boolean transferFileToService(String scope, String username, String service, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
|
||||
AnalysisLogger.getLogger().debug("Transferring file " + fileAbsolutePath + " to " + service + ":" + port);
|
||||
// ScopeProvider.instance.set(scope);
|
||||
public static long transferFileToService(String scope, String username, String host, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
|
||||
logger.debug("Transferring file {} to {}:{} " , fileAbsolutePath, host, port );
|
||||
|
||||
DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+host+":"+port);
|
||||
|
||||
AgentLibrary library = transferAgent().at(service, port).build();
|
||||
ArrayList<URI> input = new ArrayList<URI>();
|
||||
File localFile = new File(fileAbsolutePath);
|
||||
if (!localFile.exists())
|
||||
throw new Exception("Local file does not exist: " + localFile);
|
||||
|
||||
String localfolder = localFile.getParent();
|
||||
//String localfolder = localFile.getParent();
|
||||
String file = localFile.getName();
|
||||
AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
|
||||
// ScopeProvider.instance.set(scope);
|
||||
AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
|
||||
|
||||
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true);
|
||||
//urls for testing
|
||||
//storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file;
|
||||
// String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc=";
|
||||
// String storagesmpurl = "http://goo.gl/r6ggMA";
|
||||
// String storagesmpurl = "http://dl.dropboxusercontent.com/u/12809149/3_Aquamaps.jpg";
|
||||
|
||||
AnalysisLogger.getLogger().debug("SMP url generated: " + storagesmpurl);
|
||||
URI uri = new URI(storagesmpurl);
|
||||
input.add(uri);
|
||||
|
||||
TransferOptions options = new TransferOptions();
|
||||
|
||||
options = new TransferOptions();
|
||||
options.setOverwriteFile(false);
|
||||
options.setType(storageType.DataStorage);
|
||||
options.setUnzipFile(false);
|
||||
options.setTransferTimeout(3, TimeUnit.HOURS);
|
||||
AnalysisLogger.getLogger().debug("Transferring...");
|
||||
|
||||
//old code for sync transfer
|
||||
// ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
|
||||
|
||||
Map<String,String> params=new HashMap<>();
|
||||
params.put("DESTINATION", remoteFolder);
|
||||
params.put("SOURCE_PARAMETER", PluginInvocation.DESTINATION_FILE_PATH);
|
||||
|
||||
Destination dest=new Destination(file);
|
||||
|
||||
ArrayList<URI> outputURI = new ArrayList<URI>();
|
||||
// outputURI.add(new URI("file://"+remoteFolder.replace(" ", "_")+file.replace(" ", "_")));
|
||||
outputURI.add(new URI("file://"+remoteFolder.replace(" ", "%20")+file.replace(" ", "%20")));
|
||||
TransferResult tranferResult = client.localFile(localFile,dest,Collections.singleton(new PluginInvocation("DECOMPRESS",params)));
|
||||
|
||||
AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0));
|
||||
|
||||
String transferId = library.startTransfer(input, outputURI, options);
|
||||
|
||||
TransferStatus transferStatus = null;
|
||||
|
||||
do {
|
||||
try {
|
||||
logger.debug("Transferring...");
|
||||
|
||||
Thread.sleep(1000);
|
||||
transferStatus = TransferStatus.valueOf(library.monitorTransfer(transferId));
|
||||
|
||||
} catch (MonitorTransferException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
} while (!transferStatus.hasCompleted());
|
||||
|
||||
ArrayList<FileTransferOutcome> outcomes = library.getTransferOutcomes(transferId, FileTransferOutcome.class);
|
||||
|
||||
AnalysisLogger.getLogger().debug("Transferring complete");
|
||||
boolean success = false;
|
||||
String outcomeString = "";
|
||||
for (FileTransferOutcome outcome : outcomes) {
|
||||
AnalysisLogger.getLogger().debug("Outcome " + outcome);
|
||||
outcomeString = outcome.toString();
|
||||
AnalysisLogger.getLogger().debug("Transferred file name " + outcome.fileName());
|
||||
AnalysisLogger.getLogger().debug("Transferring success " + outcome.isSuccess());
|
||||
AnalysisLogger.getLogger().debug("Transferred bytes " + outcome.getTotal_size());
|
||||
AnalysisLogger.getLogger().debug("Transfer time " + outcome.getTransferTime());
|
||||
success = outcome.isSuccess();
|
||||
}
|
||||
|
||||
if (!success)
|
||||
throw new Exception("No Bytes were transferred to the Thredds server: "+outcomeString);
|
||||
|
||||
|
||||
return success;
|
||||
return tranferResult.getTransferedBytes();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,19 +1,14 @@
|
||||
package org.gcube.dataanalysis.executor.tests;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.Transformations;
|
||||
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
|
||||
import org.gcube.dataanalysis.executor.nodes.transducers.OccurrenceMergingNode;
|
||||
|
||||
public class TestD4ScienceQueueMaps2_1 {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String scope = "/gcube";
|
||||
|
||||
AlgorithmConfiguration config = Regressor.getConfig();
|
||||
AlgorithmConfiguration config = CustomRegressor.getConfig();
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
|
||||
Binary file not shown.
@ -1,5 +0,0 @@
|
||||
#Generated by Maven
|
||||
#Wed Jun 24 16:37:14 CEST 2015
|
||||
version=1.1.0-SNAPSHOT
|
||||
groupId=org.gcube.dataanalysis
|
||||
artifactId=ecological-engine-smart-executor
|
||||
@ -1,25 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||
<ID></ID>
|
||||
<Type>Library</Type>
|
||||
<Profile>
|
||||
<Description>Ecological Engine Executor Library</Description>
|
||||
<Class>EcologicalEngineExecutor</Class>
|
||||
<Name>ecological-engine-smart-executor</Name>
|
||||
<Version>1.0.0</Version>
|
||||
<Packages>
|
||||
<Software>
|
||||
<Name>ecological-engine-smart-executor</Name>
|
||||
<Version>1.1.0-SNAPSHOT</Version>
|
||||
<MavenCoordinates>
|
||||
<groupId>org.gcube.dataanalysis</groupId>
|
||||
<artifactId>ecological-engine-smart-executor</artifactId>
|
||||
<version>1.1.0-SNAPSHOT</version>
|
||||
</MavenCoordinates>
|
||||
<Files>
|
||||
<File>ecological-engine-smart-executor-1.1.0-SNAPSHOT.jar</File>
|
||||
</Files>
|
||||
</Software>
|
||||
</Packages>
|
||||
</Profile>
|
||||
</Resource>
|
||||
Loading…
Reference in New Issue