224 lines
8.9 KiB
Java
224 lines
8.9 KiB
Java
package org.gcube.dataanalysis.executor.nodes.algorithms;
|
|
|
|
import java.io.File;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.UUID;
|
|
|
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
|
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
|
|
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
|
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
|
|
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveTypesList;
|
|
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
|
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
|
|
import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode;
|
|
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
|
|
import org.gcube.dataanalysis.ecoengine.utils.Transformations;
|
|
import org.gcube.dataanalysis.executor.scripts.OSCommand;
|
|
import org.gcube.dataanalysis.executor.util.RScriptsManager;
|
|
import org.gcube.dataanalysis.executor.util.StorageUtils;
|
|
|
|
public class FAOMSY extends ActorNode {
|
|
|
|
public int count;
|
|
|
|
public float status = 0;
|
|
|
|
@Override
|
|
public ALG_PROPS[] getProperties() {
|
|
ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON };
|
|
return p;
|
|
}
|
|
|
|
@Override
|
|
public String getName() {
|
|
return "FAOMSY";
|
|
}
|
|
|
|
@Override
|
|
public String getDescription() {
|
|
return "An algorithm to estimate the Maximum Sustainable Yield from a catch statistic by FAO.";
|
|
}
|
|
|
|
static String stocksFile = "StocksFile";
|
|
static String processOutput= "ProcessOutput";
|
|
static String nonProcessedOutput= "NonProcessedOutput";
|
|
static String scriptName = "CatchMSY_Dec2014.R";
|
|
|
|
@Override
|
|
public List<StatisticalType> getInputParameters() {
|
|
|
|
List<StatisticalType> parameters = new ArrayList<StatisticalType>();
|
|
IOHelper.addStringInput(parameters, stocksFile, "Http link to a file containing catch statistics for a group of species. Example: http://goo.gl/g6YtVx", "");
|
|
return parameters;
|
|
}
|
|
|
|
@Override
|
|
public StatisticalType getOutput() {
|
|
File outfile = new File(config.getPersistencePath(),config.getParam(processOutput));
|
|
File outfile2 = new File(config.getPersistencePath(),config.getParam(nonProcessedOutput));
|
|
PrimitiveTypesList list = new PrimitiveTypesList(File.class.getName(), PrimitiveTypes.FILE, "OutputFiles", "Textual output files - processed and non-processed species",false);
|
|
if (outfile.exists()){
|
|
PrimitiveType o = new PrimitiveType(File.class.getName(), outfile, PrimitiveTypes.FILE, "ProcessedSpecies", "Output file");
|
|
list.add(o);
|
|
}
|
|
if (outfile2.exists()){
|
|
PrimitiveType o2 = new PrimitiveType(File.class.getName(), outfile2, PrimitiveTypes.FILE, "NonProcessedSpecies", "Output file");
|
|
list.add(o2);
|
|
}
|
|
|
|
|
|
return list;
|
|
}
|
|
|
|
@Override
|
|
public void initSingleNode(AlgorithmConfiguration config) {
|
|
|
|
}
|
|
|
|
@Override
|
|
public float getInternalStatus() {
|
|
return status;
|
|
}
|
|
|
|
AlgorithmConfiguration config;
|
|
|
|
@Override
|
|
public int executeNode(int leftStartIndex, int numberOfLeftElementsToProcess, int rightStartIndex, int numberOfRightElementsToProcess, boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce) {
|
|
try {
|
|
status = 0;
|
|
config = Transformations.restoreConfig(nodeConfigurationFileObject);
|
|
String outputFile = config.getParam(processOutput)+"_part"+rightStartIndex;
|
|
String nonprocessedoutputFile = config.getParam(nonProcessedOutput)+"_part"+rightStartIndex;
|
|
AnalysisLogger.getLogger().info("FAOMSY ranges: "+" Li:"+leftStartIndex+" NLi:"+leftStartIndex+" Ri:"+rightStartIndex+" NRi:"+numberOfRightElementsToProcess);
|
|
|
|
AnalysisLogger.getLogger().info("FAOMSY expected output "+outputFile);
|
|
|
|
File filestock=new File(sandboxFolder,"D20_1.csv");
|
|
StorageUtils.downloadInputFile(config.getParam(stocksFile), filestock.getAbsolutePath());
|
|
|
|
AnalysisLogger.getLogger().debug("Check fileStocks: "+filestock.getAbsolutePath()+" "+filestock.exists());
|
|
File filestocksub=new File(sandboxFolder,"D20.csv");
|
|
|
|
StorageUtils.FileSubset(filestock, filestocksub, rightStartIndex, numberOfRightElementsToProcess, true);
|
|
|
|
RScriptsManager scriptmanager = new RScriptsManager();
|
|
|
|
HashMap<String,String> codeinj = new HashMap<String,String>();
|
|
config.setConfigPath("./");
|
|
scriptmanager.executeRScript(config, scriptName, "", new HashMap<String,String>(), "", "CatchMSY_Output.csv", codeinj, false,false,false);
|
|
AnalysisLogger.getLogger().info("FAOMSY The script has finished");
|
|
String outputFileName = "";
|
|
//manage the fact that the outputfile could even not exist
|
|
try{outputFileName = scriptmanager.getCurrentOutputFileName();}catch(Exception e){
|
|
AnalysisLogger.getLogger().info("FAOMSY Could not get curent output file");
|
|
}
|
|
String optionalFileOutputName = "NonProcessedSpecies.csv";
|
|
|
|
if (outputFileName!=null && outputFileName.length()>0 && new File(outputFileName).exists()){
|
|
AnalysisLogger.getLogger().info("FAOMSY Main file exists!");
|
|
outputFileName = scriptmanager.getCurrentOutputFileName();
|
|
String outputFilePath = new File(sandboxFolder,outputFile).getAbsolutePath();
|
|
AnalysisLogger.getLogger().info("FAOMSY writing output file in path "+outputFilePath);
|
|
OSCommand.FileCopy(outputFileName,outputFilePath);
|
|
AnalysisLogger.getLogger().info("FAOMSY uploading output file "+outputFile);
|
|
StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), sandboxFolder,outputFile);
|
|
}
|
|
if (new File(optionalFileOutputName).exists()){
|
|
AnalysisLogger.getLogger().info("FAOMSY Optional file exists!");
|
|
OSCommand.FileCopy(optionalFileOutputName,nonprocessedoutputFile);
|
|
AnalysisLogger.getLogger().info("FAOMSY uploading output file "+nonprocessedoutputFile);
|
|
//check only
|
|
// String file = FileTools.loadString(nonprocessedoutputFile, "UTF-8");
|
|
// AnalysisLogger.getLogger().info("FAOMSY File check"+file);
|
|
StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), sandboxFolder,nonprocessedoutputFile);
|
|
}
|
|
|
|
AnalysisLogger.getLogger().info("FAOMSY Finished");
|
|
}catch(Exception e){
|
|
e.printStackTrace();
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
@Override
|
|
public void setup(AlgorithmConfiguration config) throws Exception {
|
|
this.config = config;
|
|
AnalysisLogger.getLogger().info("FAOMSY process is initialized");
|
|
String uuid = (UUID.randomUUID()+".txt").replace("-", "");
|
|
config.setParam(processOutput, "FAOMSY_"+"output_"+uuid);
|
|
config.setParam(nonProcessedOutput, "FAOMSY_nonprocessed_"+"output_"+uuid);
|
|
File tempfile = new File(config.getPersistencePath(),"FAOMSY_input_"+(UUID.randomUUID()+".csv").replace("-", ""));
|
|
StorageUtils.downloadInputFile(config.getParam(stocksFile), tempfile.getAbsolutePath());
|
|
nstocks = StorageUtils.calcFileRows(tempfile, true);
|
|
AnalysisLogger.getLogger().info("FAOMSY Found "+nstocks+" stocks!");
|
|
if (nstocks==0)
|
|
throw new Exception("Error in FAOMSY: No stocks to process found in the file "+config.getParam(stocksFile));
|
|
|
|
}
|
|
|
|
int nstocks = 0;
|
|
@Override
|
|
public int getNumberOfRightElements() {
|
|
return nstocks;
|
|
}
|
|
|
|
@Override
|
|
public int getNumberOfLeftElements() {
|
|
return 1;
|
|
}
|
|
|
|
@Override
|
|
public void stop() {
|
|
AnalysisLogger.getLogger().info("CMSY process stopped");
|
|
}
|
|
|
|
boolean haspostprocessed = false;
|
|
|
|
public void assembleFiles(String outputFileName) throws Exception{
|
|
|
|
//try downloading all the files
|
|
List<String> fileslist = new ArrayList<String>();
|
|
for (int i=0;i<=nstocks;i++){
|
|
String filename = outputFileName+"_part"+i;
|
|
try{
|
|
StorageUtils.downloadFilefromStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), config.getPersistencePath(), filename);
|
|
AnalysisLogger.getLogger().debug("FAOMSY - Saved from Storage: "+filename);
|
|
fileslist.add(filename);
|
|
}catch(Exception e){
|
|
AnalysisLogger.getLogger().debug("FAOMSY - Did not save file from Storage: "+filename);
|
|
}
|
|
}
|
|
|
|
|
|
AnalysisLogger.getLogger().debug("FAOMSY - Merging files in "+outputFileName);
|
|
if (fileslist.size()>0)
|
|
StorageUtils.mergeFiles(config.getPersistencePath(), fileslist, outputFileName, true);
|
|
|
|
AnalysisLogger.getLogger().debug("FAOMSY - Deleting parts");
|
|
for (String file:fileslist){
|
|
new File(config.getPersistencePath(),file).delete();
|
|
}
|
|
|
|
AnalysisLogger.getLogger().debug("FAOMSY - File assembling complete");
|
|
|
|
}
|
|
|
|
@Override
|
|
public void postProcess(boolean manageDuplicates, boolean manageFault) {
|
|
try {
|
|
String mainOutputfilename = config.getParam(processOutput);
|
|
String optionalOutputfilename = config.getParam(nonProcessedOutput);
|
|
assembleFiles(mainOutputfilename);
|
|
assembleFiles(optionalOutputfilename);
|
|
AnalysisLogger.getLogger().debug("FAOMSY - Postprocess complete");
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
}
|