Lucio Lelii 2017-05-08 16:18:18 +00:00
parent 3d03f95946
commit 8a05e571d4
46 changed files with 120 additions and 301 deletions

View File

@ -1,9 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

View File

@ -1,13 +1,12 @@
#Fri Jun 22 18:05:41 CEST 2012
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.6
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.6
org.eclipse.jdt.core.compiler.source=1.7

68
pom.xml
View File

@ -9,12 +9,25 @@
</parent>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>ecological-engine-smart-executor</artifactId>
<version>1.4.0-SNAPSHOT</version>
<version>1.5.0-SNAPSHOT</version>
<name>Smart Ecological Engine Executor</name>
<description>Smart Ecological Engine Executor library</description>
<scm>
<url>https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor</url>
</scm>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>gcube-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<developers>
<developer>
<name>Gianpaolo Coro</name>
@ -33,16 +46,16 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>data-transfer-library</artifactId>
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>ecological-engine</artifactId>
<version>[1.8.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>agent-library</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.gcube.contentmanagement</groupId>
<artifactId>storage-manager-core</artifactId>
@ -56,49 +69,23 @@
<dependency>
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-core</artifactId>
<version>5.6.0</version>
</dependency>
<dependency>
<groupId>org.gcube.common</groupId>
<artifactId>common-authorization</artifactId>
<version>[2.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>common-utils-encryption</artifactId>
<groupId>org.gcube.core</groupId>
</exclusion>
</exclusions>
<version>5.7.0</version>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-scope</artifactId>
<version>[1.2.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<!--<exclusion> <artifactId>common-scope-maps</artifactId> <groupId>org.gcube.core</groupId>
</exclusion> -->
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
@ -109,13 +96,12 @@
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2</version>
<configuration>
<descriptors>
<descriptor>${distroDirectory}/descriptor.xml</descriptor>

View File

@ -1,22 +1,17 @@
package org.gcube.dataanalysis.executor.job.management;
import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Logger;
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools;
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE;
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad;
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources;
import org.gcube.dataanalysis.ecoengine.utils.Operations;
import com.thoughtworks.xstream.XStream;
import org.slf4j.Logger;
public class DistributedProcessingAgentWPS {

View File

@ -13,11 +13,11 @@ import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import org.apache.log4j.Logger;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanager.storageclient.model.protocol.smp.Handler;
import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode;
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm;
import org.slf4j.Logger;
public class GenericWorker extends StandardLocalInfraAlgorithm{

View File

@ -171,8 +171,7 @@ public class WPSJobManager {
} catch (Exception e) {
e.printStackTrace();
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("WPSJobManager->Task Number "+taskNumber+" - Process exception: " + e.getLocalizedMessage());
AnalysisLogger.getLogger().error("WPSJobManager->Task Number {} - Process exception ", taskNumber,e);
exitstatus = GenericWorker.TASK_FAILURE;
}finally{

View File

@ -92,8 +92,8 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
String remotePath = remoteFolder+subpath+"/";
AnalysisLogger.getLogger().debug("Uploading "+key+" -> "+remotePath);
boolean transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath);
if (!transferout){
long transferout = DataTransferer.transferFileToService(scope, username, transferServiceAddress, transferServicePort, key, remotePath);
if (transferout<=0){
throw new Exception("Error transferring files to the infrastructure ");
}
i++;
@ -107,9 +107,7 @@ public class WebApplicationPublisher extends StandardLocalInfraAlgorithm{
//get URL
addOutputString("Generated Website - Main URL", producedPage);
}catch(Exception e){
e.printStackTrace();
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("An error occurred!");
AnalysisLogger.getLogger().error("an error occurred!",e);
throw e;
}finally{
//clean everything

View File

@ -2,6 +2,7 @@ package org.gcube.dataanalysis.executor.nodes.transducers.bionym;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@ -25,7 +26,6 @@ import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Match
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
import org.gcube.dataanalysis.executor.scripts.OSCommand;
import scala.actors.threadpool.Arrays;
public class BionymLocalTransducer implements Transducerer {
// BionymFlexibleWorkflowTransducer bionymwf = new BionymFlexibleWorkflowTransducer();

View File

@ -5,7 +5,6 @@ import java.security.Permission;
import java.util.HashMap;
import java.util.UUID;
import org.apache.tools.ant.ExitException;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.LevensteinMatcher;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher;
@ -158,7 +157,7 @@ public abstract class YasmeenMatcher implements Matcher {
@Override
public void checkExit(int status) {
super.checkExit(status);
throw new ExitException(status);
throw new RuntimeException("exit status is "+status);
}
}
}

View File

@ -153,8 +153,7 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
}
} catch (Exception e) {
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("Could not delete sandbox folder " + folder.getAbsolutePath());
AnalysisLogger.getLogger().warn("Could not delete sandbox folder {} ",folder.getAbsolutePath(),e);
}
if (Rlog != null) {
@ -236,9 +235,7 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
org.gcube.dataanalysis.executor.rscripts.generic.FileUtils.moveFileToDirectory(rowFile, new File(config.getConfigPath()), false);
files.add(preparedFile);
}catch(Exception e){
e.printStackTrace();
AnalysisLogger.getLogger().debug("error in moving file " + rowFile.getAbsolutePath() + " to " + preparedFile);
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().error("error in moving file {} to {}",rowFile.getAbsolutePath(),preparedFile,e);
throw e;
}
break;

View File

@ -1,155 +1,49 @@
package org.gcube.dataanalysis.executor.util;
import static org.gcube.datatransfer.agent.library.proxies.Proxies.transferAgent;
import java.io.File;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.gcube.data.transfer.library.DataTransferClient;
import org.gcube.data.transfer.library.TransferResult;
import org.gcube.data.transfer.model.Destination;
import org.gcube.data.transfer.model.PluginInvocation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.datatransfer.agent.library.AgentLibrary;
import org.gcube.datatransfer.agent.library.exceptions.MonitorTransferException;
import org.gcube.datatransfer.common.agent.Types.storageType;
import org.gcube.datatransfer.common.options.TransferOptions;
import org.gcube.datatransfer.common.outcome.FileTransferOutcome;
import org.gcube.datatransfer.common.outcome.TransferStatus;
public class DataTransferer {
public static void main(String[] args) throws Exception {
//String scope = "/d4science.research-infrastructures.eu/gCubeApps";
String scope = "/gcube/devsec/devVRE";
//test only ScopeProvider.instance.set(scope);
//String transferGHN = "dewn04.madgik.di.uoa.gr";
String transferGHN = "access.d4science.org";
int transferPort = 8080;
AgentLibrary library = transferAgent().at(transferGHN, transferPort).build();
ArrayList<URI> input = new ArrayList<URI>();
/*
* File localfile = new File("C:/Users/coro/Dropbox/Public/wind1.tif"); String file = "wind1.tif"; String localfolder = "C:/Users/coro/Dropbox/Public/"; String storagesmpurl = StorageUtils.uploadFilesOnStorage("/gcube/devsec", "gianpaolo.coro",localfolder,file);
*
* System.out.println("URI from storage: "+storagesmpurl);
*
* String urlStorage = "http://dev.d4science.org/uri-resolver/smp?smp-uri="+storagesmpurl+"&fileName="+file;
*/
// String urlStorage = "http://dev.d4science.org/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif";
//String urlStorage = "http://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=";
String urlStorage ="http://goo.gl/Vq8QVY";
System.out.println("URL for storage: " + urlStorage);
URI uri = new URI("http://dl.dropboxusercontent.com/u/12809149/wind1.tif");
// URI uri = new URI(urlStorage);
// http://dev.d4science.org/uri-resolver/smp?smp-uri=smp://data.gcube.org/gzAv/RparhTHO4yhbF9ItALcRlSJRIiBGmbP5+HKCzc=&fileName=wind1.tif&contentType=tiff
// URI uri = new URI(storageurl); //localfile.toURI();
// URI uri = new URI("file:///C:Users/coro/Dropbox/Public/wind1.tif");
input.add(uri);
//String outPath = "/tmp";
String outPath = "/var/www/html/test/";
String fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
//fileToTransfer = "C:\\Users\\coro\\Dropbox\\Public\\3_Aquamaps.jpg";
// fileToTransfer = "C:/Users/coro/Desktop/DATABASE e NOTE/Experiments/WEB_APP_PUBLISHER/gcube/images/Resource Model.png";
transferFileToService(scope, "gianpaolo.coro", transferGHN, transferPort,fileToTransfer , outPath);
}
private static final Logger logger = LoggerFactory.getLogger(DataTransferer.class);
// returns the number of transferred bytes
public static boolean transferFileToService(String scope, String username, String service, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
AnalysisLogger.getLogger().debug("Transferring file " + fileAbsolutePath + " to " + service + ":" + port);
// ScopeProvider.instance.set(scope);
public static long transferFileToService(String scope, String username, String host, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
logger.debug("Transferring file {} to {}:{} " , fileAbsolutePath, host, port );
DataTransferClient client=DataTransferClient.getInstanceByEndpoint("http://"+host+":"+port);
AgentLibrary library = transferAgent().at(service, port).build();
ArrayList<URI> input = new ArrayList<URI>();
File localFile = new File(fileAbsolutePath);
if (!localFile.exists())
throw new Exception("Local file does not exist: " + localFile);
String localfolder = localFile.getParent();
//String localfolder = localFile.getParent();
String file = localFile.getName();
AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
// ScopeProvider.instance.set(scope);
AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true);
//urls for testing
//storagesmpurl = "http://dev.d4science.org/smp?smp-uri="+storagesmpurl+"&fileName="+file;
// String storagesmpurl = "smp://data.gcube.org/sHtVhK4clGtbcWCliQud+5b4PfGx5BW+GmbP5+HKCzc=";
// String storagesmpurl = "http://goo.gl/r6ggMA";
// String storagesmpurl = "http://dl.dropboxusercontent.com/u/12809149/3_Aquamaps.jpg";
AnalysisLogger.getLogger().debug("SMP url generated: " + storagesmpurl);
URI uri = new URI(storagesmpurl);
input.add(uri);
TransferOptions options = new TransferOptions();
options = new TransferOptions();
options.setOverwriteFile(false);
options.setType(storageType.DataStorage);
options.setUnzipFile(false);
options.setTransferTimeout(3, TimeUnit.HOURS);
AnalysisLogger.getLogger().debug("Transferring...");
//old code for sync transfer
// ArrayList<FileTransferOutcome> outcomes = library.startTransferSync(input, remoteFolder, options);
ArrayList<URI> outputURI = new ArrayList<URI>();
// outputURI.add(new URI("file://"+remoteFolder.replace(" ", "_")+file.replace(" ", "_")));
outputURI.add(new URI("file://"+remoteFolder.replace(" ", "%20")+file.replace(" ", "%20")));
Map<String,String> params=new HashMap<>();
params.put("DESTINATION", remoteFolder);
params.put("SOURCE_PARAMETER", PluginInvocation.DESTINATION_FILE_PATH);
Destination dest=new Destination(file);
AnalysisLogger.getLogger().debug("Remote file name will be: " + outputURI.get(0));
TransferResult tranferResult = client.localFile(localFile,dest,Collections.singleton(new PluginInvocation("DECOMPRESS",params)));
String transferId = library.startTransfer(input, outputURI, options);
logger.debug("Transferring...");
TransferStatus transferStatus = null;
do {
try {
Thread.sleep(1000);
transferStatus = TransferStatus.valueOf(library.monitorTransfer(transferId));
} catch (MonitorTransferException e) {
e.printStackTrace();
}
} while (!transferStatus.hasCompleted());
ArrayList<FileTransferOutcome> outcomes = library.getTransferOutcomes(transferId, FileTransferOutcome.class);
AnalysisLogger.getLogger().debug("Transferring complete");
boolean success = false;
String outcomeString = "";
for (FileTransferOutcome outcome : outcomes) {
AnalysisLogger.getLogger().debug("Outcome " + outcome);
outcomeString = outcome.toString();
AnalysisLogger.getLogger().debug("Transferred file name " + outcome.fileName());
AnalysisLogger.getLogger().debug("Transferring success " + outcome.isSuccess());
AnalysisLogger.getLogger().debug("Transferred bytes " + outcome.getTotal_size());
AnalysisLogger.getLogger().debug("Transfer time " + outcome.getTransferTime());
success = outcome.isSuccess();
}
if (!success)
throw new Exception("No Bytes were transferred to the Thredds server: "+outcomeString);
return success;
return tranferResult.getTransferedBytes();
}
}

View File

@ -17,7 +17,6 @@ import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.executor.scripts.OSCommand;
import org.junit.runner.notification.RunListener;
public class LocalRScriptsManager {
@ -91,7 +90,7 @@ public class LocalRScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
uberbuffer.append(sb.toString());
AnalysisLogger.getLogger().debug("---ERRORS---");
br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
@ -102,7 +101,7 @@ public class LocalRScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
uberbuffer.append(sb.toString());
AnalysisLogger.getLogger().debug("---END OF ERRORS---");
} catch (Exception e) {
@ -284,7 +283,8 @@ public class LocalRScriptsManager {
}
}
}
/*
public static void main1(String[] args) throws Exception {
String url = StorageUtils.uploadFilesOnStorage("/gcube/devsec/devVRE", "ecological.engine", "C:\\Users\\coro\\Documents\\", "tacsatmini.csv");
System.out.println(url);
@ -321,5 +321,5 @@ public class LocalRScriptsManager {
scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true, "./");
}
*/
}

View File

@ -12,8 +12,6 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.UUID;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
@ -78,7 +76,7 @@ public class RScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
AnalysisLogger.getLogger().debug("---ERRORS---");
br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
@ -89,7 +87,7 @@ public class RScriptsManager {
line = br.readLine();
sb.append(line + "\n");
}
AnalysisLogger.getLogger().debug(sb);
AnalysisLogger.getLogger().debug(sb.toString());
AnalysisLogger.getLogger().debug("---END OF ERRORS---");
} catch (Exception e) {
AnalysisLogger.getLogger().debug("---END BY PROCESS INTERRUPTION---");

View File

@ -1,6 +1,5 @@
package org.gcube.dataanalysis.executor.tests;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
@ -13,7 +12,7 @@ public class RegressionTestBiOnym {
public static void executeWF(String[] args) throws Exception {
// Generate
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
config.setConfigPath("./cfg/");
config.setParam("DatabaseUserName","utente");
config.setParam("DatabasePassword","d4science");

View File

@ -6,7 +6,6 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestCMSY {
/**

View File

@ -7,7 +7,6 @@ import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
public class RegressionTestFAOMSY {

View File

@ -4,9 +4,7 @@ import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestGenerators {
/**
@ -20,7 +18,7 @@ public class RegressionTestGenerators {
List<ComputationalAgent> generators = GeneratorsFactory.getGenerators(testAquamapsSuitable());
generators.get(0).init();
Regressor.process(generators.get(0));
CustomRegressor.process(generators.get(0));
generators = null;
}

View File

@ -6,8 +6,6 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
public class RegressionTestICCATVPA {
/**

View File

@ -6,7 +6,6 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestLWR {
/**

View File

@ -5,9 +5,7 @@ import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.evaluation.bioclimate.InterpolateTables.INTERPOLATIONFUNCTIONS;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class RegressionTestTransducers {
@ -48,14 +46,14 @@ public static void main(String[] args) throws Exception {
*/
trans = TransducerersFactory.getTransducerers(testConfigLocal5());
trans.get(0).init();
Regressor.process(trans.get(0));
CustomRegressor.process(trans.get(0));
trans = null;
}
private static AlgorithmConfiguration testConfigLocal() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("BIOCLIMATE_HSPEC");
config.setParam("HSPEC_TABLE_LIST", "hspec_validation"+AlgorithmConfiguration.getListSeparator()+"hspec_validation2");
config.setParam("HSPEC_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test");
@ -66,7 +64,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal2() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("BIOCLIMATE_HCAF");
config.setParam("HCAF_TABLE_LIST","hcaf_d"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01332632269756"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01336062995861"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2050");
config.setParam("HCAF_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test");
@ -76,7 +74,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal3() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("BIOCLIMATE_HSPEN");
config.setParam("HSPEN_TABLE_LIST","hspen"+AlgorithmConfiguration.getListSeparator()+"hspen_2016"+AlgorithmConfiguration.getListSeparator()+"hspen_2020"+AlgorithmConfiguration.getListSeparator()+"hspen_2050");
config.setParam("HSPEN_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test");
@ -87,7 +85,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal4() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("HCAF_INTERPOLATION");
config.setParam("FirstHCAF","hcaf_d");
@ -102,7 +100,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal5() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("OCCURRENCES_MERGER");
config.setParam("longitudeColumn", "decimallongitude");
@ -125,7 +123,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal6() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("OCCURRENCES_INSEAS_ONEARTH");
config.setParam("longitudeColumn", "decimallongitude");
@ -140,7 +138,7 @@ public static void main(String[] args) throws Exception {
private static AlgorithmConfiguration testConfigLocal7() {
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setAgent("OCCURRENCES_DUPLICATE_DELETER");
config.setParam("longitudeColumn", "decimallongitude");

View File

@ -1,15 +1,11 @@
package org.gcube.dataanalysis.executor.tests;
import java.io.File;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymLocalTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
@ -67,7 +63,7 @@ public class TestBiOnymLocal {
List<StatisticalType> types = transducer.getInputParameters();
// System.out.println(types);
Regressor.process(transducer);
CustomRegressor.process(transducer);
}
}

View File

@ -1,15 +1,11 @@
package org.gcube.dataanalysis.executor.tests;
import java.io.File;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymLocalTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
@ -48,7 +44,7 @@ public class TestBiOnymLocalFastVersion {
List<StatisticalType> types = transducer.getInputParameters();
Regressor.process(transducer);
CustomRegressor.process(transducer);
}
}

View File

@ -1,19 +1,14 @@
package org.gcube.dataanalysis.executor.tests;
import java.io.File;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.ecoengine.utils.Transformations;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
import org.gcube.dataanalysis.executor.nodes.transducers.OccurrenceMergingNode;
public class TestD4ScienceQueueMaps2_1 {
public static void main(String[] args) throws Exception {
String scope = "/gcube";
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setConfigPath("./cfg/");
config.setPersistencePath("./");

View File

@ -1,18 +1,6 @@
package org.gcube.dataanalysis.executor.tests;
import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
public class TestD4ScienceQueueMaps2_2 {
@ -30,7 +18,7 @@ public class TestD4ScienceQueueMaps2_2 {
String finalDistributionTable = "hspec_suitable_executorsingle_t3";
// Generate
AlgorithmConfiguration config = Regressor.getConfig();
AlgorithmConfiguration config = CustomRegressor.getConfig();
config.setParam("DatabaseUserName","utente");
config.setParam("DatabasePassword","d4science");

View File

@ -8,7 +8,6 @@ import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class TestKnitrCompiler {
@ -35,7 +34,7 @@ public class TestKnitrCompiler {
//init the transducerer
transducer.init();
//start the process
Regressor.process(transducer);
CustomRegressor.process(transducer);
//retrieve the output
StatisticalType st = transducer.getOutput();
System.out.println("st:"+((PrimitiveType)st).getContent());

View File

@ -7,7 +7,6 @@ import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class TestSGVMInterpolation {
@ -46,7 +45,7 @@ public class TestSGVMInterpolation {
//init the transducerer
transducer.init();
//start the process
Regressor.process(transducer);
CustomRegressor.process(transducer);
//retrieve the output
StatisticalType st = transducer.getOutput();
System.out.println("st:"+((PrimitiveType)st).getContent());

View File

@ -7,7 +7,6 @@ import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class TestWebAppPublisher {
@ -35,7 +34,7 @@ public class TestWebAppPublisher {
List<ComputationalAgent> trans = null;
trans = TransducerersFactory.getTransducerers(config);
trans.get(0).init();
Regressor.process(trans.get(0));
CustomRegressor.process(trans.get(0));
StatisticalType st = trans.get(0).getOutput();
AnalysisLogger.getLogger().debug("ST:" + st);
trans = null;

View File

@ -1,5 +0,0 @@
#Generated by Maven
#Wed Jun 24 16:37:14 CEST 2015
version=1.1.0-SNAPSHOT
groupId=org.gcube.dataanalysis
artifactId=ecological-engine-smart-executor

View File

@ -1,25 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Resource xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<ID></ID>
<Type>Library</Type>
<Profile>
<Description>Ecological Engine Executor Library</Description>
<Class>EcologicalEngineExecutor</Class>
<Name>ecological-engine-smart-executor</Name>
<Version>1.0.0</Version>
<Packages>
<Software>
<Name>ecological-engine-smart-executor</Name>
<Version>1.1.0-SNAPSHOT</Version>
<MavenCoordinates>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>ecological-engine-smart-executor</artifactId>
<version>1.1.0-SNAPSHOT</version>
</MavenCoordinates>
<Files>
<File>ecological-engine-smart-executor-1.1.0-SNAPSHOT.jar</File>
</Files>
</Software>
</Packages>
</Profile>
</Resource>