gCubeSystem
/
ecological-engine-smart-executor
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@148375 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2017-05-08 16:18:18 +00:00
parent 3d03f95946
commit 8a05e571d4
46 changed files with 120 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
.classpath
View File

@ -1,9 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

7
.settings/org.eclipse.jdt.core.prefs
View File

@ -1,13 +1,12 @@
#Fri Jun 22 18:05:41 CEST 2012
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.6
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.6
org.eclipse.jdt.core.compiler.source=1.7

68
pom.xml
View File

@ -9,12 +9,25 @@
</parent>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>ecological-engine-smart-executor</artifactId>
<version>1.4.0-SNAPSHOT</version>
<version>1.5.0-SNAPSHOT</version>
<name>Smart Ecological Engine Executor</name>
<description>Smart Ecological Engine Executor library</description>
<scm>
<url>https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor</url>
</scm>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>gcube-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<developers>
<developer>
<name>Gianpaolo Coro</name>
@ -33,16 +46,16 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>data-transfer-library</artifactId>
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>ecological-engine</artifactId>
<version>[1.8.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>agent-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-core</artifactId>
@ -56,49 +69,23 @@
<dependency>
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-core</artifactId>
<version>5.6.0</version>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>common-authorization</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>common-utils-encryption</artifactId>
<groupId>org.gcube.core</groupId>
</exclusion>
</exclusions>
<version>5.7.0</version>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-scope</artifactId>
<version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<!--<exclusion> <artifactId>common-scope-maps</artifactId> <groupId>org.gcube.core</groupId>
</exclusion> -->
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
@ -109,13 +96,12 @@
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2</version>
<configuration>
<descriptors>
<descriptor>${distroDirectory}/descriptor.xml</descriptor>

7
src/main/java/org/gcube/dataanalysis/executor/job/management/DistributedProcessingAgentWPS.java
View File

@ -1,22 +1,17 @@
package org.gcube.dataanalysis.executor.job.management;
import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Logger;
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools;
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE;
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad;
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources;
import org.gcube.dataanalysis.ecoengine.utils.Operations;
import com.thoughtworks.xstream.XStream;
import org.slf4j.Logger;
public class DistributedProcessingAgentWPS {

2
src/main/java/org/gcube/dataanalysis/executor/job/management/GenericWorker.java
View File

@ -13,11 +13,11 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import org.apache.log4j.Logger;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanager.storageclient.model.protocol.smp.Handler;
import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode;
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm;
import org.slf4j.Logger;
public class GenericWorker extends StandardLocalInfraAlgorithm{

3
src/main/java/org/gcube/dataanalysis/executor/job/management/WPSJobManager.java
View File

@ -171,8 +171,7 @@ public class WPSJobManager {
} catch (Exception e) {
e.printStackTrace();
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("WPSJobManager->Task Number "+taskNumber+" - Process exception: " + e.getLocalizedMessage());
AnalysisLogger.getLogger().error("WPSJobManager->Task Number {} - Process exception ", taskNumber,e);
exitstatus = GenericWorker.TASK_FAILURE;
}finally{

8
src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/WebApplicationPublisher.java
View File

@ -92,8 +92,8 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
String remotePath = remoteFolder+subpath+"/";
AnalysisLogger.getLogger().debug("Uploading "+key+" -> "+remotePath);
boolean transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath);
if (!transferout){
long transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath);
if (transferout<=0){
throw new Exception("Error transferring files to the infrastructure ");
}
i++;
@ -107,9 +107,7 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
//get URL
addOutputString("Generated Website - Main URL", producedPage);
}catch(Exception e){
e.printStackTrace();
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("An error occurred!");
AnalysisLogger.getLogger().error("an error occurred!",e);
throw e;
}finally{
//clean everything

2
src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymLocalTransducer.java
View File

@ -2,6 +2,7 @@ package org.gcube.dataanalysis.executor.nodes.transducers.bionym;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@ -25,7 +26,6 @@ import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Match
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
import org.gcube.dataanalysis.executor.scripts.OSCommand;
import scala.actors.threadpool.Arrays;
public class BionymLocalTransducer implements Transducerer {
// BionymFlexibleWorkflowTransducer bionymwf = new BionymFlexibleWorkflowTransducer();

3
src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/YasmeenMatcher.java
View File

@ -5,7 +5,6 @@ import java.security.Permission;
import java.util.HashMap;
import java.util.UUID;
import org.apache.tools.ant.ExitException;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.LevensteinMatcher;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher;
@ -158,7 +157,7 @@ public abstract class YasmeenMatcher implements Matcher {
@Override
public void checkExit(int status) {
super.checkExit(status);
throw new ExitException(status);
throw new RuntimeException("exit status is "+status);
}
}
}

7
src/main/java/org/gcube/dataanalysis/executor/rscripts/generic/GenericRScript.java
View File

@ -153,8 +153,7 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
}
} catch (Exception e) {
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("Could not delete sandbox folder " + folder.getAbsolutePath());
AnalysisLogger.getLogger().warn("Could not delete sandbox folder {} ",folder.getAbsolutePath(),e);
}
if (Rlog != null) {
@ -236,9 +235,7 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
org.gcube.dataanalysis.executor.rscripts.generic.FileUtils.moveFileToDirectory(rowFile, new File(config.getConfigPath()), false);
files.add(preparedFile);
}catch(Exception e){
e.printStackTrace();
AnalysisLogger.getLogger().debug("error in moving file " + rowFile.getAbsolutePath() + " to " + preparedFile);
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().error("error in moving file {} to {}",rowFile.getAbsolutePath(),preparedFile,e);
throw e;
}
break;

156
src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java
View File

@ -1,155 +1,49 @@
package org.gcube.dataanalysis.executor.util;
import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent;
import java.io.File;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.PluginInvocation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.datatransfer.agent.library.AgentLibrary;
import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException;
import org.gcube.datatransfer.common.agent.Types.storageType;
import org.gcube.datatransfer.common.options.TransferOptions;
import org.gcube.datatransfer.common.outcome.FileTransferOutcome;
import org.gcube.datatransfer.common.outcome.TransferStatus;
public class DataTransferer {
public static void main(String[] args) throws Exception {
//String scope = "/d4science.research-infrastructures.eu/gCubeApps";
String scope = "/gcube/devsec/devVRE";
//test only ScopeProvider.instance.set(scope);
//String transferGHN = "dewn04.madgik.di.uoa.gr";
String transferGHN = "access.d4science.org";
int transferPort = 8080;
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
ArrayList<URI> input = new ArrayList<URI>();
/*
* File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif"); String file = "wind1.tif"; String localfolder = "C:/Users/coro/Dropbox/Public/"; String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
*
* System.out.println("URI from storage: "+storagesmpurl);
*
* String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
*/
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
//String urlStorage = "http://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
String urlStorage ="http://goo.gl/Vq8QVY";
System.out.println("URL for storage: " + urlStorage);
URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
// URI uri = new URI(urlStorage);
// http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
// URI uri = new URI(storageurl); //localfile.toURI();
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
input.add(uri);
//String outPath = "/tmp";
String outPath = "/var/www/html/test/";
String fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
//fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
// fileToTransfer = "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/gcube/images/Resource Model.png";
transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort,fileToTransfer , outPath);
}
private static final Logger logger = LoggerFactory.getLogger(DataTransferer.class);
// returns the number of transferred bytes
public static boolean transferFileToService(String scope, String username, String service, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
AnalysisLogger.getLogger().debug("Transferring file " + fileAbsolutePath + " to " + service + ":" + port);
// ScopeProvider.instance.set(scope);
public static long transferFileToService(String scope, String username, String host, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
logger.debug("Transferring file {} to {}:{} " , fileAbsolutePath, host, port );
DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+host+":"+port);
AgentLibrary library = transferAgent().at(service, port).build();
ArrayList<URI> input = new ArrayList<URI>();
File localFile = new File(fileAbsolutePath);
if (!localFile.exists())
throw new Exception("Local file does not exist: " + localFile);
String localfolder = localFile.getParent();
//String localfolder = localFile.getParent();
String file = localFile.getName();
AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
// ScopeProvider.instance.set(scope);
AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true);
//urls for testing
//storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file;
// String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc=";
// String storagesmpurl = "http://goo.gl/r6ggMA";
// String storagesmpurl = "http://dl.dropboxusercontent.com/u/12809149/3_Aquamaps.jpg";
AnalysisLogger.getLogger().debug("SMP url generated: " + storagesmpurl);
URI uri = new URI(storagesmpurl);
input.add(uri);
TransferOptions options = new TransferOptions();
options = new TransferOptions();
options.setOverwriteFile(false);
options.setType(storageType.DataStorage);
options.setUnzipFile(false);
options.setTransferTimeout(3, TimeUnit.HOURS);
AnalysisLogger.getLogger().debug("Transferring...");
//old code for sync transfer
// ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
ArrayList<URI> outputURI = new ArrayList<URI>();
// outputURI.add(new URI("file://"+remoteFolder.replace(" ", "_")+file.replace(" ", "_")));
outputURI.add(new URI("file://"+remoteFolder.replace(" ", "%20")+file.replace(" ", "%20")));
Map<String,String> params=new HashMap<>();
params.put("DESTINATION", remoteFolder);
params.put("SOURCE_PARAMETER", PluginInvocation.DESTINATION_FILE_PATH);
Destination dest=new Destination(file);
AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0));
TransferResult tranferResult = client.localFile(localFile,dest,Collections.singleton(new PluginInvocation("DECOMPRESS",params)));
String transferId = library.startTransfer(input, outputURI, options);
logger.debug("Transferring...");
TransferStatus transferStatus = null;
do {
try {
Thread.sleep(1000);
transferStatus = TransferStatus.valueOf(library.monitorTransfer(transferId));
} catch (MonitorTransferException e) {
e.printStackTrace();
}
} while (!transferStatus.hasCompleted());
ArrayList<FileTransferOutcome> outcomes = library.getTransferOutcomes(transferId, FileTransferOutcome.class);
AnalysisLogger.getLogger().debug("Transferring complete");
boolean success = false;
String outcomeString = "";
for (FileTransferOutcome outcome : outcomes) {
AnalysisLogger.getLogger().debug("Outcome " + outcome);
outcomeString = outcome.toString();
AnalysisLogger.getLogger().debug("Transferred file name " + outcome.fileName());
AnalysisLogger.getLogger().debug("Transferring success " + outcome.isSuccess());
AnalysisLogger.getLogger().debug("Transferred bytes " + outcome.getTotal_size());
AnalysisLogger.getLogger().debug("Transfer time " + outcome.getTransferTime());
success = outcome.isSuccess();
}
if (!success)
throw new Exception("No Bytes were transferred to the Thredds server: "+outcomeString);
return success;
return tranferResult.getTransferedBytes();
}
}

10
src/main/java/org/gcube/dataanalysis/executor/util/LocalRScriptsManager.java
View File

@ -17,7 +17,6 @@ import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.executor.scripts.OSCommand;
import org.junit.runner.notification.RunListener;
public class LocalRScriptsManager {
@ -91,7 +90,7 @@ public class LocalRScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
uberbuffer.append(sb.toString());
AnalysisLogger.getLogger().debug("---ERRORS---");
br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
@ -102,7 +101,7 @@ public class LocalRScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
uberbuffer.append(sb.toString());
AnalysisLogger.getLogger().debug("---END OF ERRORS---");
} catch (Exception e) {
@ -284,7 +283,8 @@ public class LocalRScriptsManager {
}
}
}
/*
public static void main1(String[] args) throws Exception {
String url = StorageUtils.uploadFilesOnStorage("/gcube/devsec/devVRE", "ecological.engine", "C:\\Users\\coro\\Documents\\", "tacsatmini.csv");
System.out.println(url);
@ -321,5 +321,5 @@ public class LocalRScriptsManager {
scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true, "./");
}
*/
}

6
src/main/java/org/gcube/dataanalysis/executor/util/RScriptsManager.java
View File

@ -12,8 +12,6 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.UUID;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
@ -78,7 +76,7 @@ public class RScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
AnalysisLogger.getLogger().debug("---ERRORS---");
br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
@ -89,7 +87,7 @@ public class RScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
AnalysisLogger.getLogger().debug("---END OF ERRORS---");
} catch (Exception e) {
AnalysisLogger.getLogger().debug("---END BY PROCESS INTERRUPTION---");

0
src/main/java/org/gcube/dataanalysis/executor/tests/CustomRegressor.java → src/test/java/org/gcube/dataanalysis/executor/tests/CustomRegressor.java
View File

3
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym.java
View File

@ -1,6 +1,5 @@
package org.gcube.dataanalysis.executor.tests;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
@ -13,7 +12,7 @@ public class RegressionTestBiOnym {
public static void executeWF(String[] args) throws Exception {
// Generate
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
config.setConfigPath("./cfg/");
config.setParam("DatabaseUserName","utente");
config.setParam("DatabasePassword","d4science");

0
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym1Matcher.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym1Matcher.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnymProduzione.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnymProduzione.java
View File

1
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestCMSY.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestCMSY.java
View File

@ -6,7 +6,6 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestCMSY {
/**

1
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestFAOMSY.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestFAOMSY.java
View File

@ -7,7 +7,6 @@ import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
public class RegressionTestFAOMSY {

4
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestGenerators.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestGenerators.java
View File

@ -4,9 +4,7 @@ import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestGenerators {
/**
@ -20,7 +18,7 @@ public class RegressionTestGenerators {
List<ComputationalAgent> generators = GeneratorsFactory.getGenerators(testAquamapsSuitable());
generators.get(0).init();
Regressor.process(generators.get(0));
CustomRegressor.process(generators.get(0));
generators = null;
}

2
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestICCATVPA.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestICCATVPA.java
View File

@ -6,8 +6,6 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
public class RegressionTestICCATVPA {
/**

1
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestLWR.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestLWR.java
View File

@ -6,7 +6,6 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestLWR {
/**

18
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestTransducers.java → src/test/java/org/gcube/dataanalysis/executor/tests/RegressionTestTransducers.java
View File

@ -5,9 +5,7 @@ import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.evaluation.bioclimate.InterpolateTables.INTERPOLATIONFUNCTIONS;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestTransducers {
@ -48,14 +46,14 @@ public static void main(String[] args) throws Exception {
*/
trans = TransducerersFactory.getTransducerers(testConfigLocal5());
trans.get(0).init();
Regressor.process(trans.get(0));
CustomRegressor.process(trans.get(0));
trans = null;
}
private static AlgorithmConfiguration testConfigLocal() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("BIOCLIMATE_HSPEC");
config.setParam("HSPEC_TABLE_LIST", "hspec_validation"+AlgorithmConfiguration.getListSeparator()+"hspec_validation2");
config.setParam("HSPEC_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test");
@ -66,7 +64,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal2() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("BIOCLIMATE_HCAF");
config.setParam("HCAF_TABLE_LIST","hcaf_d"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01332632269756"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01336062995861"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2050");
config.setParam("HCAF_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test");
@ -76,7 +74,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal3() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("BIOCLIMATE_HSPEN");
config.setParam("HSPEN_TABLE_LIST","hspen"+AlgorithmConfiguration.getListSeparator()+"hspen_2016"+AlgorithmConfiguration.getListSeparator()+"hspen_2020"+AlgorithmConfiguration.getListSeparator()+"hspen_2050");
config.setParam("HSPEN_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test");
@ -87,7 +85,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal4() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("HCAF_INTERPOLATION");
config.setParam("FirstHCAF","hcaf_d");
@ -102,7 +100,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal5() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("OCCURRENCES_MERGER");
config.setParam("longitudeColumn", "decimallongitude");
@ -125,7 +123,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal6() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("OCCURRENCES_INSEAS_ONEARTH");
config.setParam("longitudeColumn", "decimallongitude");
@ -140,7 +138,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal7() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("OCCURRENCES_DUPLICATE_DELETER");
config.setParam("longitudeColumn", "decimallongitude");

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestAquaMapsJobs.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestAquaMapsJobs.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnym1_0_0.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestBiOnym1_0_0.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymEvaluation.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestBiOnymEvaluation.java
View File

6
src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocal.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocal.java
View File

@ -1,15 +1,11 @@
package org.gcube.dataanalysis.executor.tests;
import java.io.File;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymLocalTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
@ -67,7 +63,7 @@ public class TestBiOnymLocal {
List<StatisticalType> types = transducer.getInputParameters();
// System.out.println(types);
Regressor.process(transducer);
CustomRegressor.process(transducer);
}
}

6
src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocalFastVersion.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocalFastVersion.java
View File

@ -1,15 +1,11 @@
package org.gcube.dataanalysis.executor.tests;
import java.io.File;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymLocalTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
@ -48,7 +44,7 @@ public class TestBiOnymLocalFastVersion {
List<StatisticalType> types = transducer.getInputParameters();
Regressor.process(transducer);
CustomRegressor.process(transducer);
}
}

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceMaps.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceMaps.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps2.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps2.java
View File

7
src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps2_1.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps2_1.java
View File

@ -1,19 +1,14 @@
package org.gcube.dataanalysis.executor.tests;
import java.io.File;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.ecoengine.utils.Transformations;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
import org.gcube.dataanalysis.executor.nodes.transducers.OccurrenceMergingNode;
public class TestD4ScienceQueueMaps2_1 {
public static void main(String[] args) throws Exception {
String scope = "/gcube";
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setConfigPath("./cfg/");
config.setPersistencePath("./");

14
src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps2_2.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps2_2.java
View File

@ -1,18 +1,6 @@
package org.gcube.dataanalysis.executor.tests;
import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
public class TestD4ScienceQueueMaps2_2 {
@ -30,7 +18,7 @@ public class TestD4ScienceQueueMaps2_2 {
String finalDistributionTable = "hspec_suitable_executorsingle_t3";
// Generate
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setParam("DatabaseUserName","utente");
config.setParam("DatabasePassword","d4science");

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps3.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps3.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMultipleMaps.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMultipleMaps.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestEmailingSystem.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestEmailingSystem.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestGenerator.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestGenerator.java
View File

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestGetRunningExecutor.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestGetRunningExecutor.java
View File

3
src/main/java/org/gcube/dataanalysis/executor/tests/TestKnitrCompiler.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestKnitrCompiler.java
View File

@ -8,7 +8,6 @@ import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class TestKnitrCompiler {
@ -35,7 +34,7 @@ public class TestKnitrCompiler {
//init the transducerer
transducer.init();
//start the process
Regressor.process(transducer);
CustomRegressor.process(transducer);
//retrieve the output
StatisticalType st = transducer.getOutput();
System.out.println("st:"+((PrimitiveType)st).getContent());

3
src/main/java/org/gcube/dataanalysis/executor/tests/TestSGVMInterpolation.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestSGVMInterpolation.java
View File

@ -7,7 +7,6 @@ import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class TestSGVMInterpolation {
@ -46,7 +45,7 @@ public class TestSGVMInterpolation {
//init the transducerer
transducer.init();
//start the process
Regressor.process(transducer);
CustomRegressor.process(transducer);
//retrieve the output
StatisticalType st = transducer.getOutput();
System.out.println("st:"+((PrimitiveType)st).getContent());

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestWPSJobs.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestWPSJobs.java
View File

3
src/main/java/org/gcube/dataanalysis/executor/tests/TestWebAppPublisher.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestWebAppPublisher.java
View File

@ -7,7 +7,6 @@ import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class TestWebAppPublisher {
@ -35,7 +34,7 @@ public class TestWebAppPublisher {
List<ComputationalAgent> trans = null;
trans = TransducerersFactory.getTransducerers(config);
trans.get(0).init();
Regressor.process(trans.get(0));
CustomRegressor.process(trans.get(0));
StatisticalType st = trans.get(0).getOutput();
AnalysisLogger.getLogger().debug("ST:" + st);
trans = null;

0
src/main/java/org/gcube/dataanalysis/executor/tests/TestsMetaInfo.java → src/test/java/org/gcube/dataanalysis/executor/tests/TestsMetaInfo.java
View File

BIN
target/ecological-engine-smart-executor-1.0.0-SNAPSHOT.jar
View File

Binary file not shown.

5
target/maven-archiver/pom.properties
View File

@ -1,5 +0,0 @@
#Generated by Maven
#Wed Jun 24 16:37:14 CEST 2015
version=1.1.0-SNAPSHOT
groupId=org.gcube.dataanalysis
artifactId=ecological-engine-smart-executor

25
target/profile.xml
View File

@ -1,25 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<ID></ID>
<Type>Library</Type>
<Profile>
<Description>Ecological Engine Executor Library</Description>
<Class>EcologicalEngineExecutor</Class>
<Name>ecological-engine-smart-executor</Name>
<Version>1.0.0</Version>
<Packages>
<Software>
<Name>ecological-engine-smart-executor</Name>
<Version>1.1.0-SNAPSHOT</Version>
<MavenCoordinates>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>ecological-engine-smart-executor</artifactId>
<version>1.1.0-SNAPSHOT</version>
</MavenCoordinates>
<Files>
<File>ecological-engine-smart-executor-1.1.0-SNAPSHOT.jar</File>
</Files>
</Software>
</Packages>
</Profile>
</Resource>