git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@115643 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
de93d14fbf
commit
af4b42664a
40
.classpath
40
.classpath
|
|
@ -1,31 +1,9 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
|
||||
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
#Wed May 06 16:54:55 CEST 2015
|
||||
eclipse.preferences.version=1
|
||||
encoding//src/main/java=UTF-8
|
||||
encoding//src/main/resources=UTF-8
|
||||
encoding//src/test/java=UTF-8
|
||||
encoding/<project>=UTF-8
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -1,5 +0,0 @@
|
|||
#!/bin/sh
|
||||
# LWR
|
||||
cd $1
|
||||
|
||||
java -Xmx1024M -classpath ./:./c3p0-0.9.1.2.jar:./common-configuration-scanner-1.0.1-SNAPSHOT.jar:./common-encryption-1.0.1-3.5.0.jar:./common-gcore-resources-1.2.0-3.5.0.jar:./common-gcore-stubs-1.2.0-3.5.0.jar:./common-scope-1.2.1-SNAPSHOT.jar:./common-scope-maps-1.0.2-3.5.0.jar:./commons-collections-3.1.jar:./commons-io-1.2.jar:./discovery-client-1.0.1-3.5.0.jar:./dom4j-1.6.1.jar:./ecological-engine-1.8.1-SNAPSHOT.jar:./EcologicalEngineExecutor-1.6.4-SNAPSHOT.jar:./hibernate3.jar:./ic-client-1.0.1-3.5.0.jar:./jaxen-1.1.2.jar:./jta-1.1.jar:./log4j-1.2.16.jar:./mongo-java-driver-2.12.4.jar:./postgresql-8.4-702.jdbc4.jar:./slf4j-api-1.6.0.jar:./slf4j-log4j12-1.6.0.jar:./storage-manager-core-2.1.3-3.6.0.jar:./storage-manager-wrapper-2.1.0-3.5.0.jar:./xalan-2.6.0.jar:./xpp3_min-1.1.4c.jar:./xstream-1.3.1.jar:./YASMEEN-matcher-1.2.0.1.jar:./YASMEEN-parser-1.2.0.jar org.gcube.dataanalysis.executor.nodes.algorithms.LWR $2 execution.output
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,17 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<jardesc>
|
||||
<jar path="EcologicalEngineExecutor/AQUAMAPS_SUITABLE/aquamapsnode.jar"/>
|
||||
<options buildIfNeeded="true" compress="true" descriptionLocation="/EcologicalEngineExecutor/aquamapsjarcreator.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
|
||||
<storedRefactorings deprecationInfo="true" structuralOnly="false"/>
|
||||
<selectedProjects/>
|
||||
<manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
|
||||
<sealing sealJar="false">
|
||||
<packagesToSeal/>
|
||||
<packagesToUnSeal/>
|
||||
</sealing>
|
||||
</manifest>
|
||||
<selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
|
||||
<javaElement handleIdentifier="=EcologicalEngineExecutor/src\/main\/java<org.gcube.dataanalysis.executor.generators"/>
|
||||
<javaElement handleIdentifier="=EcologicalEngineExecutor/src\/main\/java<org.gcube.dataanalysis.executor.nodes.algorithms"/>
|
||||
</selectedElements>
|
||||
</jardesc>
|
||||
|
|
@ -7,4 +7,5 @@ AQUAMAPS_SUITABLE_NEURALNETWORK=org.gcube.dataanalysis.ecoengine.spatialdistribu
|
|||
FEED_FORWARD_A_N_N_DISTRIBUTION=org.gcube.dataanalysis.ecoengine.spatialdistributions.FeedForwardNeuralNetworkDistribution
|
||||
LWR=org.gcube.dataanalysis.executor.nodes.algorithms.LWR
|
||||
CMSY=org.gcube.dataanalysis.executor.nodes.algorithms.CMSY
|
||||
FAOMSY=org.gcube.dataanalysis.executor.nodes.algorithms.FAOMSY
|
||||
FAOMSY=org.gcube.dataanalysis.executor.nodes.algorithms.FAOMSY
|
||||
ICCAT_VPA=org.gcube.dataanalysis.executor.nodes.algorithms.ICCATVPA
|
||||
|
|
@ -9,4 +9,5 @@ LWR=org.gcube.dataanalysis.executor.nodes.algorithms.LWR
|
|||
BIONYM=org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer
|
||||
BIONYM_BIODIV=org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymBiodiv
|
||||
CMSY=org.gcube.dataanalysis.executor.nodes.algorithms.CMSY
|
||||
FAOMSY=org.gcube.dataanalysis.executor.nodes.algorithms.FAOMSY
|
||||
FAOMSY=org.gcube.dataanalysis.executor.nodes.algorithms.FAOMSY
|
||||
ICCAT_VPA=org.gcube.dataanalysis.executor.nodes.algorithms.ICCATVPA
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
<?xml version="1.0" encoding="WINDOWS-1252" standalone="no"?>
|
||||
<jardesc>
|
||||
<jar path="ExecutorScriptFile/org.gcube.dataanalysis.executor.executorscriptplugin.jar"/>
|
||||
<options buildIfNeeded="true" compress="true" descriptionLocation="/ExecutorScriptFile/createscript.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
|
||||
<storedRefactorings deprecationInfo="true" structuralOnly="false"/>
|
||||
<selectedProjects/>
|
||||
<manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
|
||||
<sealing sealJar="false">
|
||||
<packagesToSeal/>
|
||||
<packagesToUnSeal/>
|
||||
</sealing>
|
||||
</manifest>
|
||||
<selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
|
||||
<javaElement handleIdentifier="=ExecutorScriptFile/src<org.gcube.dataanalysis.executor.plugin.dummy"/>
|
||||
<javaElement handleIdentifier="=ExecutorScriptFile/src<org.gcube.dataanalysis.executor.scripts"/>
|
||||
<javaElement handleIdentifier="=ExecutorScriptFile/src<org.gcube.dataanalysis.executor.plugin"/>
|
||||
</selectedElements>
|
||||
</jardesc>
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<jardesc>
|
||||
<jar path="C:/Users/coro/Desktop/EcologicalEngineExecutor-1.4.0-SNAPSHOT.jar"/>
|
||||
<options buildIfNeeded="true" compress="true" descriptionLocation="/EcologicalEngineExecutor/deployDesktop.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
|
||||
<storedRefactorings deprecationInfo="true" structuralOnly="false"/>
|
||||
<selectedProjects/>
|
||||
<manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
|
||||
<sealing sealJar="false">
|
||||
<packagesToSeal/>
|
||||
<packagesToUnSeal/>
|
||||
</sealing>
|
||||
</manifest>
|
||||
<selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
|
||||
<javaElement handleIdentifier="=EcologicalEngineExecutor/src\/main\/java"/>
|
||||
</selectedElements>
|
||||
</jardesc>
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<jardesc>
|
||||
<jar path="EcologicalEngineExecutor/PARALLEL_PROCESSING/EcologicalEngineExecutor-1.2.0-SNAPSHOT.jar"/>
|
||||
<options buildIfNeeded="true" compress="true" descriptionLocation="/EcologicalEngineExecutor/deployParallelProcessing.jardesc" exportErrors="true" exportWarnings="true" includeDirectoryEntries="false" overwrite="true" saveDescription="true" storeRefactorings="false" useSourceFolders="false"/>
|
||||
<storedRefactorings deprecationInfo="true" structuralOnly="false"/>
|
||||
<selectedProjects/>
|
||||
<manifest generateManifest="true" manifestLocation="" manifestVersion="1.0" reuseManifest="false" saveManifest="false" usesManifest="true">
|
||||
<sealing sealJar="false">
|
||||
<packagesToSeal/>
|
||||
<packagesToUnSeal/>
|
||||
</sealing>
|
||||
</manifest>
|
||||
<selectedElements exportClassFiles="true" exportJavaFiles="false" exportOutputFolder="false">
|
||||
<javaElement handleIdentifier="=EcologicalEngineExecutor/src\/main\/java"/>
|
||||
</selectedElements>
|
||||
</jardesc>
|
||||
10
pom.xml
10
pom.xml
|
|
@ -9,7 +9,7 @@
|
|||
</parent>
|
||||
<groupId>org.gcube.dataanalysis</groupId>
|
||||
<artifactId>ecological-engine-smart-executor</artifactId>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<version>1.1.0-SNAPSHOT</version>
|
||||
<name>Smart Ecological Engine Executor</name>
|
||||
<description>Smart Ecological Engine Executor Description</description>
|
||||
<properties>
|
||||
|
|
@ -19,17 +19,17 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.vremanagement</groupId>
|
||||
<artifactId>smart-executor-client</artifactId>
|
||||
<version>[1.0.0,3.0.0)</version>
|
||||
<version>[1.0.0-SNAPSHOT,3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.contentmanagement</groupId>
|
||||
<artifactId>storage-manager-core</artifactId>
|
||||
<version>[2.0.2,3.0.0)</version>
|
||||
<version>[2.0.2-SNAPSHOT,3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.contentmanagement</groupId>
|
||||
<artifactId>storage-manager-wrapper</artifactId>
|
||||
<version>[2.0.2,3.0.0)</version>
|
||||
<version>[2.0.2-SNAPSHOT,3.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.activemq</groupId>
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
<dependency>
|
||||
<groupId>org.gcube.dataanalysis</groupId>
|
||||
<artifactId>ecological-engine</artifactId>
|
||||
<version>[1.8.0,2.0.0)</version>
|
||||
<version>[1.8.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ public class CMSY extends ActorNode {
|
|||
HashMap<String,String> codeinj = new HashMap<String,String>();
|
||||
codeinj.put("HLH_M07",config.getParam(stock));
|
||||
config.setConfigPath("./");
|
||||
scriptmanager.executeRScript(config, scriptName, "", new HashMap<String,String>(), "", "outputfile.txt", codeinj, true,false,false);
|
||||
scriptmanager.executeRScript(config, scriptName, "", new HashMap<String,String>(), "", "outputfile.txt", codeinj, true,false,false,sandboxFolder);
|
||||
|
||||
outputFileName = scriptmanager.getCurrentOutputFileName();
|
||||
String outputFilePath = new File(sandboxFolder,outputFile).getAbsolutePath();
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ public class FAOMSY extends ActorNode {
|
|||
|
||||
HashMap<String,String> codeinj = new HashMap<String,String>();
|
||||
config.setConfigPath("./");
|
||||
scriptmanager.executeRScript(config, scriptName, "", new HashMap<String,String>(), "", "CatchMSY_Output.csv", codeinj, false,false,false);
|
||||
scriptmanager.executeRScript(config, scriptName, "", new HashMap<String,String>(), "", "CatchMSY_Output.csv", codeinj, false,false,false,sandboxFolder);
|
||||
AnalysisLogger.getLogger().info("FAOMSY The script has finished");
|
||||
String outputFileName = "";
|
||||
//manage the fact that the outputfile could even not exist
|
||||
|
|
|
|||
|
|
@ -0,0 +1,277 @@
|
|||
package org.gcube.dataanalysis.executor.nodes.algorithms;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.Transformations;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.ZipTools;
|
||||
import org.gcube.dataanalysis.executor.scripts.OSCommand;
|
||||
import org.gcube.dataanalysis.executor.util.RScriptsManager;
|
||||
import org.gcube.dataanalysis.executor.util.StorageUtils;
|
||||
|
||||
public class ICCATVPA extends ActorNode {
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public ALG_PROPS[] getProperties() {
|
||||
ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON };
|
||||
return p;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "ICCAT_VPA";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "An algorithm for stock assessment of catch statistics published by the International Commission for the Conservation of Atlantic Tunas (ICCAT). " +
|
||||
"Produces summary statistics about a stock, involving assessment of fishing mortality, abundance, catch trend, fecundity and recruitment. Developed by IFREMER and IRD. " +
|
||||
"Contact persons: Sylvain Bonhommeau sylvain.bonhommeau@ifremer.fr, Julien Barde julien.barde@ird.fr.";
|
||||
}
|
||||
|
||||
protected static String YearStartInp = "StartYear";
|
||||
protected static String YearEndInp = "EndYear";
|
||||
protected static String CAAInp = "CAAFile";
|
||||
protected static String PCAAInp = "PCAAFile";
|
||||
protected static String CPUEInp = "CPUEFile";
|
||||
protected static String PwaaInp = "PwaaFile";
|
||||
protected static String waaInp = "waaFile";
|
||||
|
||||
protected String CAAInpURL ;
|
||||
protected String PCAAInpURL;
|
||||
protected String CPUEInpURL;
|
||||
protected String PwaaInpURL;
|
||||
protected String waaInpURL;
|
||||
|
||||
|
||||
protected static String effectInp = "shortComment";
|
||||
protected static String nCPUEInp = "nCPUE";
|
||||
protected static String CPUEcutInp = "CPUE_cut";
|
||||
protected static String nRemoveYearInp = "n_remove_year";
|
||||
protected static String agePlusGroupInp = "age_plus_group";
|
||||
protected static String scriptName = "run_vpa.R";
|
||||
protected static String packageURL = "http://goo.gl/EqFjNZ";
|
||||
|
||||
protected static String processOutputParam= "ProcessOutputParam";
|
||||
protected String processOutput= "ProcessOutput";
|
||||
|
||||
protected AlgorithmConfiguration config;
|
||||
public float status = 0;
|
||||
|
||||
@Override
|
||||
public List<StatisticalType> getInputParameters() {
|
||||
List<StatisticalType> parameters = new ArrayList<StatisticalType>();
|
||||
IOHelper.addIntegerInput(parameters, YearStartInp, "First year of the dataset temporal extent", "1950");
|
||||
IOHelper.addIntegerInput(parameters, YearEndInp, "Last year of the dataset temporal extent", "2013");
|
||||
|
||||
IOHelper.addFileInput(parameters, CAAInp, "Catch at Age Matrix (Number of Fish caught by year and for each age)", "CAA_Age1_25.csv");
|
||||
IOHelper.addFileInput(parameters, PCAAInp, "Partial Catch at Age Matrix (Number of Fish caught by gear and year and for each age)", "PCAA_Age1_25.csv");
|
||||
IOHelper.addFileInput(parameters, CPUEInp, "Table of Catch Per Unit of Effort used in the stock assessment", "CPUE.csv");
|
||||
IOHelper.addFileInput(parameters, PwaaInp, "Partial weight at age (Weight of Fish caught by gear and year and for each age)", "waa.csv");
|
||||
IOHelper.addFileInput(parameters, waaInp, "Fecundity at age (Fecundity of Fish caught by year and for each age)", "fecaa.csv");
|
||||
|
||||
IOHelper.addStringInput(parameters, effectInp, "Free text for users to describe the current simulation", " ");
|
||||
IOHelper.addIntegerInput(parameters, nCPUEInp, "Number of Catch Per Unit of Effort Time series to use", "7");
|
||||
IOHelper.addIntegerInput(parameters, CPUEcutInp, "Identifier of the Catch Per Unit of Effort Time Serie to be shrunk", "1");
|
||||
IOHelper.addIntegerInput(parameters, nRemoveYearInp, "Number of the (last) years to be removed", "1");
|
||||
IOHelper.addIntegerInput(parameters, agePlusGroupInp, "Maximal age class of catches to be taken into account", "10");
|
||||
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setup(AlgorithmConfiguration config) throws Exception {
|
||||
this.config = config;
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA process is initialized in scope "+config.getGcubeScope()+" for user "+config.getParam("ServiceUserName"));
|
||||
String uuid = (UUID.randomUUID()+"").replace("-", "");
|
||||
processOutput = "ICCAT-VPA_"+"output_"+uuid+".zip";
|
||||
|
||||
config.setParam(processOutputParam, processOutput);
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA Uploading input files: "+config.getGeneralProperties());
|
||||
//upload files on the storage manager
|
||||
CAAInpURL = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), new File(config.getParam(CAAInp)).getParent(), new File(config.getParam(CAAInp)).getName());
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA: CAA DONE! "+CAAInpURL);
|
||||
PCAAInpURL = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), new File(config.getParam(PCAAInp)).getParent(), new File(config.getParam(PCAAInp)).getName());
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA: PCAA DONE! "+PCAAInpURL);
|
||||
CPUEInpURL = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), new File(config.getParam(CPUEInp)).getParent(), new File(config.getParam(CPUEInp)).getName());
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA: CPUE DONE! "+CPUEInpURL);
|
||||
PwaaInpURL = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), new File(config.getParam(PwaaInp)).getParent(), new File(config.getParam(PwaaInp)).getName());
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA: Pwaa DONE! "+PwaaInpURL);
|
||||
waaInpURL = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), new File(config.getParam(waaInp)).getParent(), new File(config.getParam(waaInp)).getName());
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA: waa DONE! "+waaInpURL);
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA Input files uploaded!");
|
||||
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA Setting input URLs: "+config.getGeneralProperties());
|
||||
|
||||
config.setParam(CAAInp, CAAInpURL);
|
||||
config.setParam(PCAAInp, PCAAInpURL);
|
||||
config.setParam(CPUEInp, CPUEInpURL);
|
||||
config.setParam(PwaaInp, PwaaInpURL);
|
||||
config.setParam(waaInp, waaInpURL);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public StatisticalType getOutput() {
|
||||
|
||||
File outfile = new File(processOutput);
|
||||
LinkedHashMap<String, StatisticalType> outputmap = new LinkedHashMap<String, StatisticalType>();
|
||||
|
||||
if (outfile.exists())
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA Output: "+outfile.getAbsolutePath()+" : "+outfile.exists());
|
||||
else
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA Output file does not exist - returning a proxy");
|
||||
|
||||
PrimitiveType o = new PrimitiveType(File.class.getName(), outfile, PrimitiveTypes.FILE, "ProcessedSpecies", "Output file with processed species");
|
||||
outputmap.put("Zip file containing the process output", o);
|
||||
|
||||
PrimitiveType output = new PrimitiveType(HashMap.class.getName(), outputmap, PrimitiveTypes.MAP, "Results File", "Results File");
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA Output Managed");
|
||||
return output;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initSingleNode(AlgorithmConfiguration config) {}
|
||||
|
||||
@Override
|
||||
public float getInternalStatus() {return status;}
|
||||
|
||||
@Override
|
||||
public int executeNode(int leftStartIndex, int numberOfLeftElementsToProcess, int rightStartIndex, int numberOfRightElementsToProcess, boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce) {
|
||||
try {
|
||||
status = 0;
|
||||
config = Transformations.restoreConfig(nodeConfigurationFileObject);
|
||||
|
||||
String outputFile = config.getParam(processOutputParam);
|
||||
String localzipFile = "iccat_zip.zip";
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA ranges: "+" Li:"+leftStartIndex+" NLi:"+leftStartIndex+" Ri:"+rightStartIndex+" NRi:"+numberOfRightElementsToProcess);
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA expected output "+outputFile);
|
||||
|
||||
//download the package
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : downloading package URL: "+packageURL);
|
||||
StorageUtils.downloadInputFile(packageURL, localzipFile);
|
||||
|
||||
//unzip the package
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Unzipping file: "+localzipFile+" having size "+new File(localzipFile).length());
|
||||
ZipTools.unZip(localzipFile, sandboxFolder);
|
||||
|
||||
//download input files
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Downloading remote input files "+config.getGeneralProperties());
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Downloading CAA");
|
||||
StorageUtils.downloadInputFile(config.getParam(CAAInp), "CAA_Age1_25.csv");
|
||||
StorageUtils.downloadInputFile(config.getParam(PCAAInp), "PCAA_Age1_25_Run3.csv");
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Downloading PCAA");
|
||||
StorageUtils.downloadInputFile(config.getParam(CPUEInp), "CPUE_Run3.csv");
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Downloading CPUE");
|
||||
StorageUtils.downloadInputFile(config.getParam(PwaaInp), "waa.csv");
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Downloading Pwaa");
|
||||
StorageUtils.downloadInputFile(config.getParam(waaInp), "fecaa.csv");
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Downloading waa");
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : all files downloaded: ");
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : CCA size: "+new File("CAA_Age1_25.csv"));
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : PCAA size: "+new File("PCAA_Age1_25_Run3.csv"));
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : CPUE size: "+new File("CPUE_Run3.csv"));
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Pwaa size: "+new File("waa.csv"));
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : waa size: "+new File("fecaa.csv"));
|
||||
|
||||
String run = "Run_"+rightStartIndex;
|
||||
//create the input file: e.g.
|
||||
//Run_7 1950-2013 CAA_Age1_25.csv PCAA_Age1_25_Run3.csv CPUE_Run3.csv waa.csv fecaa.csv Run_2 split JP_LL NEAST and wihtout last 1 year in ESPMARTrap 8 1 1 10
|
||||
String header = "Run,Year,CAA,PCAA,CPUE,Pwaa,waa,compare to,effect,nCPUE,CPUE_cut,n_remove_year,age_plus_group";
|
||||
String input = run+","+config.getParam(YearStartInp)+"-"+config.getParam(YearEndInp)+","+"CAA_Age1_25.csv"+","+"PCAA_Age1_25_Run3.csv"+","+"CPUE_Run3.csv"+","+"waa.csv"+","+"fecaa.csv"+",,"+config.getParam(effectInp)+","+config.getParam(nCPUEInp)+","+config.getParam(CPUEcutInp)+","+config.getParam(nRemoveYearInp)+","+config.getParam(agePlusGroupInp);
|
||||
FileWriter fw = new FileWriter(new File(sandboxFolder,"run_spec.csv"));
|
||||
fw.write(header+"\n");
|
||||
fw.write(input+"\n");
|
||||
fw.close();
|
||||
|
||||
//run the code as-is after substituting the Run string
|
||||
HashMap<String,String> codeinj = new HashMap<String,String>();
|
||||
codeinj.put("Run_0", run);
|
||||
String pathSand = new File(sandboxFolder).getAbsolutePath();
|
||||
if (!pathSand.endsWith("/"))
|
||||
pathSand=pathSand+"/";
|
||||
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA : substituting path to sandbox folder "+pathSand);
|
||||
codeinj.put("/home/gcube/irdstockassessment/allinone/", pathSand);
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : changing rights to the fortran file");
|
||||
OSCommand.ExecuteGetLine("chmod 777 vpa-2box.out", null);
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : running the script: "+scriptName);
|
||||
RScriptsManager scriptmanager = new RScriptsManager();
|
||||
scriptmanager.executeRScript(config, scriptName, "", new HashMap<String,String>(), "", "output.csv", codeinj, false,false,false, sandboxFolder);
|
||||
|
||||
//zip the output
|
||||
File outputFolder = new File(sandboxFolder,run);
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : checking the output folder: "+outputFolder.getAbsolutePath()+" exists: "+outputFolder.exists());
|
||||
if (!outputFolder.exists())
|
||||
throw new Exception("ICCAT-VPA: output was not produced!");
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : producing zip output: "+outputFile);
|
||||
ZipTools.zipFolder(new File(sandboxFolder,run).getAbsolutePath(), outputFile);
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : zip output exists: "+new File(outputFile).exists());
|
||||
|
||||
//upload the output
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : uploading on storage");
|
||||
StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), sandboxFolder,outputFile);
|
||||
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA : Finished");
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int nExperiments=1;
|
||||
@Override
|
||||
public int getNumberOfRightElements() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumberOfLeftElements() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
AnalysisLogger.getLogger().info("ICCAT-VPA process stopped");
|
||||
}
|
||||
|
||||
boolean haspostprocessed = false;
|
||||
@Override
|
||||
public void postProcess(boolean manageDuplicates, boolean manageFault) {
|
||||
try {
|
||||
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA - Downloading file "+processOutput);
|
||||
StorageUtils.downloadFilefromStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), config.getPersistencePath(), processOutput);
|
||||
AnalysisLogger.getLogger().debug("ICCAT-VPA - Postprocess complete: output ready in "+processOutput);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -77,7 +77,7 @@ public class SGVMS_Interpolation extends StandardLocalExternalAlgorithm {
|
|||
AnalysisLogger.getLogger().debug("SGVM Interpolation-> Executing the script ");
|
||||
status = 10;
|
||||
//execute the script in multi-user mode
|
||||
scriptmanager.executeRScript(config, scriptName, inputFile, inputParameters, defaultInputFileInTheScript, defaultOutputFileInTheScript, codeInjection, scriptMustReturnAFile,uploadScriptOnTheInfrastructureWorkspace);
|
||||
scriptmanager.executeRScript(config, scriptName, inputFile, inputParameters, defaultInputFileInTheScript, defaultOutputFileInTheScript, codeInjection, scriptMustReturnAFile,uploadScriptOnTheInfrastructureWorkspace, config.getConfigPath());
|
||||
//assign the file path to an output variable for the SM
|
||||
outputFile = scriptmanager.currentOutputFileName;
|
||||
AnalysisLogger.getLogger().debug("SGVM Interpolation-> Output File is "+outputFile);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
package org.gcube.dataanalysis.executor.tests;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
|
||||
|
||||
public class RegressionTestICCATVPA {
|
||||
/**
|
||||
* example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species
|
||||
*
|
||||
*/
|
||||
|
||||
public static AlgorithmConfiguration getConfig() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setParam("DatabaseUserName","utente");
|
||||
config.setParam("DatabasePassword","d4science");
|
||||
config.setParam("DatabaseURL","jdbc:postgresql://dbtest.research-infrastructures.eu/testdb");
|
||||
config.setParam("DatabaseDriver","org.postgresql.Driver");
|
||||
AnalysisLogger.setLogger(config.getConfigPath()+AlgorithmConfiguration.defaultLoggerFile);
|
||||
return config;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
System.out.println("TEST 1");
|
||||
|
||||
List<ComputationalAgent> generators = GeneratorsFactory.getGenerators(testICCATVPA());
|
||||
generators.get(0).init();
|
||||
CustomRegressor.process(generators.get(0));
|
||||
generators = null;
|
||||
|
||||
}
|
||||
|
||||
private static AlgorithmConfiguration testICCATVPA() {
|
||||
|
||||
AlgorithmConfiguration config = getConfig();
|
||||
config.setNumberOfResources(5);
|
||||
config.setModel("ICCAT_VPA");
|
||||
|
||||
config.setParam("UserName", "gianpaolo.coro");
|
||||
config.setGcubeScope("/gcube/devsec");
|
||||
config.setParam("ServiceUserName", "gianpaolo.coro");
|
||||
|
||||
config.setParam("StartYear","1950");
|
||||
config.setParam("EndYear","2013");
|
||||
config.setParam("CAAFile","CAA_Age1_25.csv");
|
||||
config.setParam("PCAAFile","PCAA_Age1_25_Run3.csv");
|
||||
config.setParam("CPUEFile","CPUE_Run3.csv");
|
||||
config.setParam("PwaaFile","waa.csv");
|
||||
config.setParam("waaFile","fecaa.csv");
|
||||
|
||||
config.setParam("shortComment","split JP_LL NEAST and wihtout last 1 year in ESPMARTrap");
|
||||
config.setParam("nCPUE","8");
|
||||
config.setParam("CPUE_cut","1");
|
||||
config.setParam("n_remove_year","1");
|
||||
config.setParam("age_plus_group","10");
|
||||
|
||||
return config;
|
||||
}
|
||||
}
|
||||
|
|
@ -95,15 +95,14 @@ public class RScriptsManager {
|
|||
}
|
||||
}
|
||||
|
||||
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage) throws Exception {
|
||||
executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, true);
|
||||
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, String sandboxFolder) throws Exception {
|
||||
executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, true, sandboxFolder);
|
||||
}
|
||||
|
||||
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles) throws Exception {
|
||||
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles, String sandboxFolder) throws Exception {
|
||||
List<String> tempfiles = new ArrayList<String>();
|
||||
try {
|
||||
status = 0;
|
||||
String sandboxFolder = config.getConfigPath();
|
||||
String scriptPath = new File(sandboxFolder, scriptName).getAbsolutePath();
|
||||
// String originalScriptPath = scriptPath;
|
||||
String preparedScriptPath = null;
|
||||
|
|
@ -283,7 +282,7 @@ public class RScriptsManager {
|
|||
inputParameters.put("npoints", "10");
|
||||
inputParameters.put("equalDist", "TRUE");
|
||||
|
||||
scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true);
|
||||
scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true,"./");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1,5 +1,5 @@
|
|||
#Generated by Maven
|
||||
#Thu Apr 16 18:26:30 CEST 2015
|
||||
version=1.0.0-SNAPSHOT
|
||||
#Wed Jun 24 16:37:14 CEST 2015
|
||||
version=1.1.0-SNAPSHOT
|
||||
groupId=org.gcube.dataanalysis
|
||||
artifactId=ecological-engine-smart-executor
|
||||
|
|
|
|||
|
|
@ -10,14 +10,14 @@
|
|||
<Packages>
|
||||
<Software>
|
||||
<Name>ecological-engine-smart-executor</Name>
|
||||
<Version>1.0.0-SNAPSHOT</Version>
|
||||
<Version>1.1.0-SNAPSHOT</Version>
|
||||
<MavenCoordinates>
|
||||
<groupId>org.gcube.dataanalysis</groupId>
|
||||
<artifactId>ecological-engine-smart-executor</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<version>1.1.0-SNAPSHOT</version>
|
||||
</MavenCoordinates>
|
||||
<Files>
|
||||
<File>ecological-engine-smart-executor-1.0.0-SNAPSHOT.jar</File>
|
||||
<File>ecological-engine-smart-executor-1.1.0-SNAPSHOT.jar</File>
|
||||
</Files>
|
||||
</Software>
|
||||
</Packages>
|
||||
|
|
|
|||
Loading…
Reference in New Issue