gCubeSystem
/
ecological-engine-smart-executor
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@122494 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2016-01-25 16:07:09 +00:00
parent 1f510c74a6
commit a3c03e56dd
1 changed files with 124 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

214
src/main/java/org/gcube/dataanalysis/executor/rscripts/generic/GenericRScript.java
View File

@ -16,161 +16,195 @@ import org.gcube.dataanalysis.ecoengine.utils.ZipTools;
import org.gcube.dataanalysis.executor.util.LocalRScriptsManager; import org.gcube.dataanalysis.executor.util.LocalRScriptsManager;
import org.gcube.dataanalysis.executor.util.StorageUtils; import org.gcube.dataanalysis.executor.util.StorageUtils;
public abstract class GenericRScript extends StandardLocalExternalAlgorithm { public abstract class GenericRScript extends StandardLocalExternalAlgorithm {
//FIXED part // FIXED part
protected HashMap<String,String> outputValues=new HashMap<String,String>(); protected HashMap<String, String> outputValues = new HashMap<String, String>();
protected LinkedHashMap<String, StatisticalType> output = new LinkedHashMap<String, StatisticalType> (); protected LinkedHashMap<String, StatisticalType> output = new LinkedHashMap<String, StatisticalType>();
LocalRScriptsManager scriptmanager; LocalRScriptsManager scriptmanager;
@Override @Override
public void shutdown() { public void shutdown() {
//in the case of forced shutdown, stop the R process // in the case of forced shutdown, stop the R process
if (scriptmanager!=null) if (scriptmanager != null)
scriptmanager.stop(); scriptmanager.stop();
System.gc(); System.gc();
} }
@Override @Override
public void init() throws Exception { public void init() throws Exception {
AnalysisLogger.getLogger().debug("Initializing "+this.getClass().getCanonicalName()); AnalysisLogger.getLogger().debug("Initializing " + this.getClass().getCanonicalName());
initVariables(); initVariables();
} }
@Override @Override
protected void process() throws Exception { protected void process() throws Exception {
//init status boolean deletefiles = false; // for test only
// init status
status = 0; status = 0;
//instantiate the R Script executor // instantiate the R Script executor
scriptmanager = new LocalRScriptsManager(); scriptmanager = new LocalRScriptsManager();
//download and unzip the script in a controlled folder // download and unzip the script in a controlled folder
AnalysisLogger.getLogger().debug("Starting script"); AnalysisLogger.getLogger().debug("Starting script");
File localPackage = new File(config.getConfigPath(),"package"+UUID.randomUUID()+".zip"); File localPackage = new File(config.getConfigPath(), "package" + UUID.randomUUID() + ".zip");
AnalysisLogger.getLogger().debug("Downloading package "+packageURL+"as: "+localPackage.getAbsolutePath()); AnalysisLogger.getLogger().debug("Downloading package " + packageURL + "as: " + localPackage.getAbsolutePath());
StorageUtils.downloadInputFile(packageURL, localPackage.getAbsolutePath(),true); StorageUtils.downloadInputFile(packageURL, localPackage.getAbsolutePath(), true);
AnalysisLogger.getLogger().debug("Generating sandbox folder"); AnalysisLogger.getLogger().debug("Generating sandbox folder");
File folder = new File(config.getConfigPath(),"rscr_"+UUID.randomUUID()); File folder = new File(config.getConfigPath(), "rscr_" + UUID.randomUUID());
boolean mkdir = folder.mkdir(); boolean mkdir = folder.mkdir();
AnalysisLogger.getLogger().debug("Sandbox "+folder.getAbsolutePath()+" generated: "+mkdir); AnalysisLogger.getLogger().debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir);
AnalysisLogger.getLogger().debug("Unzipping package into "+folder.getAbsolutePath()); AnalysisLogger.getLogger().debug("Unzipping package into " + folder.getAbsolutePath());
ZipTools.unZip(localPackage.getAbsolutePath(), folder.getAbsolutePath()); ZipTools.unZip(localPackage.getAbsolutePath(), folder.getAbsolutePath());
localPackage.delete(); localPackage.delete();
// File folder = new File("C:/Users/coro/Desktop/WorkFolder/Workspace/EcologicalEngineSmartExecutor/./cfg/rscr_7d329495-b048-4ce0-8bcc-bd74966db56d/");
//File folder = new File("C:/Users/coro/Desktop/WorkFolder/Workspace/EcologicalEngineSmartExecutor/./cfg/rscr_7d329495-b048-4ce0-8bcc-bd74966db56d/");
AnalysisLogger.getLogger().debug("Analysing inputs"); AnalysisLogger.getLogger().debug("Analysing inputs");
//distinguish the environmental variables and the input variables // distinguish the environmental variables and the input variables
HashMap<String,String> inputParameters = new LinkedHashMap<String, String>(); HashMap<String, String> inputParameters = new LinkedHashMap<String, String>();
int i=0; int i = 0;
List<StatisticalType>inputs = getInputParameters(); List<StatisticalType> inputs = getInputParameters();
for (String input:inputvariables){ for (String input : inputvariables) {
String value = config.getParam(input); String value = config.getParam(input);
String defaultValue = inputs.get(i).getDefaultValue(); String defaultValue = inputs.get(i).getDefaultValue();
inputParameters.put(defaultValue, value); //inputParameters.put(defaultValue, value);
String punct = "[ \";]*";
String regexp = punct+input+punct+"(<-|=)"+punct+defaultValue+punct+".*";
boolean string = true;
try{
Double.parseDouble(value);
string = false;
}catch(Exception ee){
}
String line = input+"="+value;
if (string)
line = input+"=\""+value+"\"";
inputParameters.put(regexp, line);
i++; i++;
} }
AnalysisLogger.getLogger().debug("Inputs: "+inputParameters.toString()); AnalysisLogger.getLogger().debug("Inputs: " + inputParameters.toString());
AnalysisLogger.getLogger().debug("Analysing environmental variables"); AnalysisLogger.getLogger().debug("Analysing environmental variables");
HashMap<String,String> environmentalParameters = new LinkedHashMap<String, String>(); HashMap<String, String> environmentalParameters = new LinkedHashMap<String, String>();
for (String environment:environmentalvariables){ for (String environment : environmentalvariables) {
String value = config.getParam(environment); String value = config.getParam(environment);
environmentalParameters.put(environment, value); environmentalParameters.put(environment, value);
} }
AnalysisLogger.getLogger().debug("Environmental: "+environmentalParameters.toString()); AnalysisLogger.getLogger().debug("Environmental: " + environmentalParameters.toString());
AnalysisLogger.getLogger().debug("Analysing output variables"); AnalysisLogger.getLogger().debug("Analysing output variables");
HashMap<String,String> outputParameters = new LinkedHashMap<String, String>(); HashMap<String, String> outputParameters = new LinkedHashMap<String, String>();
for (String output:outputvariables){ for (String output : outputvariables) {
outputParameters.put(output, ""); outputParameters.put(output, "");
} }
AnalysisLogger.getLogger().debug("Output: "+outputParameters.toString()); AnalysisLogger.getLogger().debug("Output: " + outputParameters.toString());
AnalysisLogger.getLogger().debug("Main script: "+mainScriptName); AnalysisLogger.getLogger().debug("Main script: " + mainScriptName);
AnalysisLogger.getLogger().debug("Processing main script name"); AnalysisLogger.getLogger().debug("Processing main script name");
//build the script name // build the script name
String scriptName = mainScriptName; String scriptName = mainScriptName;
String scriptSubfolder = ""; String scriptSubfolder = "";
int lioSlash = scriptName.lastIndexOf("/"); int lioSlash = scriptName.lastIndexOf("/");
if (lioSlash>-1){ if (lioSlash > -1) {
scriptSubfolder = scriptName.substring(0,lioSlash); scriptSubfolder = scriptName.substring(0, lioSlash);
scriptName = scriptName.substring(lioSlash+1); scriptName = scriptName.substring(lioSlash + 1);
} }
scriptSubfolder = new File (folder.getAbsoluteFile(),scriptSubfolder).getAbsolutePath(); scriptSubfolder = new File(folder.getAbsoluteFile(), scriptSubfolder).getAbsolutePath();
AnalysisLogger.getLogger().debug("Main script name: "+scriptName); AnalysisLogger.getLogger().debug("Main script name: " + scriptName);
AnalysisLogger.getLogger().debug("Main script folder: "+scriptSubfolder); AnalysisLogger.getLogger().debug("Main script folder: " + scriptSubfolder);
boolean scriptMustReturnAFile = false; boolean scriptMustReturnAFile = false;
boolean uploadScriptOnTheInfrastructureWorkspace = false; boolean uploadScriptOnTheInfrastructureWorkspace = false;
status = 10; status = 10;
AnalysisLogger.getLogger().debug("Executing the script..."); AnalysisLogger.getLogger().debug("Executing the script...");
//execute the script in multi-user mode // execute the script in multi-user mode
scriptmanager.executeRScript(config, scriptName, null, environmentalParameters, outputParameters, null, null,inputParameters, scriptMustReturnAFile,uploadScriptOnTheInfrastructureWorkspace, true, scriptSubfolder);
scriptmanager.executeRScript(config, scriptName, null, environmentalParameters, outputParameters, null, null, inputParameters, scriptMustReturnAFile, uploadScriptOnTheInfrastructureWorkspace, deletefiles, scriptSubfolder);
AnalysisLogger.getLogger().debug("..execution finished!"); AnalysisLogger.getLogger().debug("..execution finished!");
//get the output: one file should have been produced for each output // get the output: one file should have been produced for each output
AnalysisLogger.getLogger().debug("Getting output"); AnalysisLogger.getLogger().debug("Getting output");
for (String output:outputvariables){ for (String output : outputvariables) {
File outPath = new File(scriptSubfolder,output); File outPath = new File(scriptSubfolder, output);
AnalysisLogger.getLogger().debug("Output "+output +" - loading respective file from "+outPath.getAbsolutePath()); AnalysisLogger.getLogger().debug("Output " + output + " - loading respective file from " + outPath.getAbsolutePath());
AnalysisLogger.getLogger().debug("File exists? "+outPath.exists()); AnalysisLogger.getLogger().debug("File exists? " + outPath.exists());
String fileContent = FileTools.loadString(outPath.getAbsolutePath(), "UTF-8"); String fileContent = FileTools.loadString(outPath.getAbsolutePath(), "UTF-8");
fileContent=analyseContent(fileContent); fileContent = analyseContent(fileContent, scriptSubfolder);
AnalysisLogger.getLogger().debug("Retrieved output content: "+output+": "+fileContent); AnalysisLogger.getLogger().debug("Retrieved output content: " + output + ": " + fileContent);
outputValues.put(output, fileContent); outputValues.put(output, fileContent);
} }
//delete the script folder // delete the script folder
AnalysisLogger.getLogger().debug("Deleting sandbox folder");
try{ try {
FileUtils.cleanDirectory(folder); if (deletefiles) {
FileUtils.deleteDirectory(folder); AnalysisLogger.getLogger().debug("Deleting sandbox folder");
AnalysisLogger.getLogger().debug("Folder "+folder.getAbsolutePath()+ " deleted"); FileUtils.cleanDirectory(folder);
}catch(Exception e){ FileUtils.deleteDirectory(folder);
AnalysisLogger.getLogger().debug("Folder " + folder.getAbsolutePath() + " deleted");
}
} catch (Exception e) {
AnalysisLogger.getLogger().debug(e); AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("Could not delete sandbox folder "+folder.getAbsolutePath()); AnalysisLogger.getLogger().debug("Could not delete sandbox folder " + folder.getAbsolutePath());
} }
AnalysisLogger.getLogger().debug("Computation finished."); AnalysisLogger.getLogger().debug("Computation finished.");
status = 100; status = 100;
} }
protected String analyseContent(String filecontent) throws Exception{ protected String analyseContent(String filecontent, String scriptfolder) throws Exception {
String [] rows = filecontent.split("\n"); AnalysisLogger.getLogger().debug("Analysing file content");
String[] rows = filecontent.split(System.lineSeparator());
List<String> files = new ArrayList<String>(); List<String> files = new ArrayList<String>();
for (String row:rows){ for (String row : rows) {
if (row!=null) AnalysisLogger.getLogger().debug("Analysing -> " + row);
if (row != null) {
row = row.replace("\"", ""); row = row.replace("\"", "");
if ((new File(row)).exists()){ File rowFile = new File(row.trim());
String preparedFile = new File(config.getConfigPath(),new File(row).getName()).getAbsolutePath(); AnalysisLogger.getLogger().debug("Checking row file-> "+rowFile.getAbsolutePath());
AnalysisLogger.getLogger().debug("Copying "+row+" to "+preparedFile); if (!rowFile.exists()){
FileUtils.moveFileToDirectory(new File(row), new File(config.getConfigPath()), false); rowFile = new File(scriptfolder, row.trim());
AnalysisLogger.getLogger().debug("File does not exist - checking complete row file-> "+rowFile.getAbsolutePath());
}
if (rowFile.exists()) {
String preparedFile = new File(config.getConfigPath(), rowFile.getName()).getAbsolutePath();
AnalysisLogger.getLogger().debug("Copying " + rowFile.getAbsolutePath() + " to " + preparedFile);
FileUtils.moveFileToDirectory(rowFile, new File(config.getConfigPath()), false);
files.add(preparedFile); files.add(preparedFile);
break; break;
} }
else
AnalysisLogger.getLogger().debug("Checking row file does not exist - treating as a String");
}
} }
if (files.size()>0) if (files.size() > 0) {
AnalysisLogger.getLogger().debug("A File was recognized as output");
return files.get(0); return files.get(0);
else } else {
AnalysisLogger.getLogger().debug("A String was recognized as output");
return filecontent; return filecontent;
}
} }
//DYNAMIC part
protected String mainScriptName=""; // DYNAMIC part
protected String packageURL=""; protected String mainScriptName = "";
protected String packageURL = "";
protected List<String> environmentalvariables = new ArrayList<String>(); protected List<String> environmentalvariables = new ArrayList<String>();
protected List<String> inputvariables= new ArrayList<String>(); protected List<String> inputvariables = new ArrayList<String>();
protected List<String> outputvariables= new ArrayList<String>(); protected List<String> outputvariables = new ArrayList<String>();
protected void initVariables() {
protected void initVariables(){
} }
@Override @Override
protected void setInputParameters() { protected void setInputParameters() {
} }
@Override @Override
public StatisticalType getOutput() { public StatisticalType getOutput() {
return null; return null;