gCubeSystem
/
ecological-engine-smart-executor
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@122705 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2016-02-01 22:07:53 +00:00
parent d8f7c33be7
commit b3c9a340d2
1 changed files with 61 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
src/main/java/org/gcube/dataanalysis/executor/util/LocalRScriptsManager.java
View File

@ -17,6 +17,7 @@ import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.executor.scripts.OSCommand; import org.gcube.dataanalysis.executor.scripts.OSCommand;
import org.junit.runner.notification.RunListener;
public class LocalRScriptsManager { public class LocalRScriptsManager {
@ -50,16 +51,27 @@ public class LocalRScriptsManager {
this.status = status; this.status = status;
} }
public static void substituteStringInFile(String file, String newFile, String s, String sub) throws Exception {
public static void substituteStringInFile(String file, String newFile, String s, String sub, boolean regex) throws Exception {
BufferedReader br = new BufferedReader(new FileReader(new File(file))); BufferedReader br = new BufferedReader(new FileReader(new File(file)));
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(newFile))); BufferedWriter bw = new BufferedWriter(new FileWriter(new File(newFile)));
String line = br.readLine(); String line = br.readLine();
while (line != null) { while (line != null) {
int idx = line.indexOf(s); if (!regex) {
if (idx >= 0) { int idx = line.indexOf(s);
line = line.replace(s, sub); if (idx >= 0) {
line = line.replace(s, sub);
}
}else{
// AnalysisLogger.getLogger().debug("Matching "+line+" vs "+s);
if (line.matches(s)){
AnalysisLogger.getLogger().debug("Matched Input "+s+" line becomes "+sub);
line = sub;
}
} }
bw.write(line + "\n"); bw.write(line + "\n");
line = br.readLine(); line = br.readLine();
} }
@ -68,7 +80,8 @@ public class LocalRScriptsManager {
bw.close(); bw.close();
} }
public static void printRConsole(Process process) { public String printRConsole(Process process) {
StringBuffer uberbuffer = new StringBuffer();
try { try {
BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line = br.readLine(); String line = br.readLine();
@ -78,7 +91,7 @@ public class LocalRScriptsManager {
sb.append(line + "\n"); sb.append(line + "\n");
} }
AnalysisLogger.getLogger().debug(sb); AnalysisLogger.getLogger().debug(sb);
uberbuffer.append(sb.toString());
AnalysisLogger.getLogger().debug("---ERRORS---"); AnalysisLogger.getLogger().debug("---ERRORS---");
br = new BufferedReader(new InputStreamReader(process.getErrorStream())); br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
line = br.readLine(); line = br.readLine();
@ -89,27 +102,30 @@ public class LocalRScriptsManager {
sb.append(line + "\n"); sb.append(line + "\n");
} }
AnalysisLogger.getLogger().debug(sb); AnalysisLogger.getLogger().debug(sb);
uberbuffer.append(sb.toString());
AnalysisLogger.getLogger().debug("---END OF ERRORS---"); AnalysisLogger.getLogger().debug("---END OF ERRORS---");
} catch (Exception e) { } catch (Exception e) {
AnalysisLogger.getLogger().debug("---END BY PROCESS INTERRUPTION---"); AnalysisLogger.getLogger().debug("---END BY PROCESS INTERRUPTION---");
} }
return uberbuffer.toString();
} }
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, String sandboxFolder) throws Exception { public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, String sandboxFolder) throws Exception {
executeRScript(config, scriptName, inputFileURL, inputParameters, null, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, true, sandboxFolder); executeRScript(config, scriptName, inputFileURL, inputParameters, null, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, true, sandboxFolder);
} }
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, HashMap<String, String> outputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, String sandboxFolder) throws Exception { public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, HashMap<String, String> outputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, String sandboxFolder) throws Exception {
executeRScript(config, scriptName, inputFileURL, inputParameters, outputParameters, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, true, sandboxFolder); executeRScript(config, scriptName, inputFileURL, inputParameters, outputParameters, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, true, sandboxFolder);
} }
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles, String sandboxFolder) throws Exception { public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles, String sandboxFolder) throws Exception {
executeRScript(config, scriptName, inputFileURL, inputParameters, null, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, deletefiles, sandboxFolder); executeRScript(config, scriptName, inputFileURL, inputParameters, null, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, deletefiles, sandboxFolder);
} }
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, HashMap<String, String> outputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles, String sandboxFolder) throws Exception { public String executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, HashMap<String, String> outputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles, String sandboxFolder) throws Exception {
List<String> tempfiles = new ArrayList<String>(); List<String> tempfiles = new ArrayList<String>();
String RLog = "";
try { try {
status = 0; status = 0;
String scriptPath = new File(sandboxFolder, scriptName).getAbsolutePath(); String scriptPath = new File(sandboxFolder, scriptName).getAbsolutePath();
@ -143,81 +159,82 @@ public class LocalRScriptsManager {
String substitution = codeInjections.get(toSubstitute); String substitution = codeInjections.get(toSubstitute);
AnalysisLogger.getLogger().debug("Substituting : " + toSubstitute + " with " + substitution); AnalysisLogger.getLogger().debug("Substituting : " + toSubstitute + " with " + substitution);
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath(); preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
substituteStringInFile(scriptPath, preparedScriptPath, toSubstitute, substitution); substituteStringInFile(scriptPath, preparedScriptPath, toSubstitute, substitution,true);
AnalysisLogger.getLogger().debug("New Script name: " + preparedScriptPath+" exists: "+new File(preparedScriptPath).exists()); AnalysisLogger.getLogger().debug("New Script name: " + preparedScriptPath + " exists: " + new File(preparedScriptPath).exists());
tempfiles.add(preparedScriptPath); tempfiles.add(preparedScriptPath);
scriptPath = preparedScriptPath; scriptPath = preparedScriptPath;
} }
} }
if (defaultInputFile!=null){ if (defaultInputFile != null) {
String newInputFile = new File(sandboxFolder, UUID.randomUUID() + defaultInputFile).getAbsolutePath().replace("\\", "/"); String newInputFile = new File(sandboxFolder, UUID.randomUUID() + defaultInputFile).getAbsolutePath().replace("\\", "/");
tempfiles.add(newInputFile); tempfiles.add(newInputFile);
AnalysisLogger.getLogger().debug("New Input File is " + newInputFile); AnalysisLogger.getLogger().debug("New Input File is " + newInputFile);
if (inputFileURL.trim().length() > 0) { if (inputFileURL.trim().length() > 0) {
AnalysisLogger.getLogger().debug("Substituting standard Input"); AnalysisLogger.getLogger().debug("Substituting standard Input");
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath(); preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
substituteStringInFile(scriptPath, preparedScriptPath, defaultInputFile, newInputFile); substituteStringInFile(scriptPath, preparedScriptPath, defaultInputFile, newInputFile,false);
} else { } else {
if ((preparedScriptPath==null) || (preparedScriptPath.trim().length()==0)) if ((preparedScriptPath == null) || (preparedScriptPath.trim().length() == 0))
preparedScriptPath = new File(sandboxFolder, scriptName).getAbsolutePath(); preparedScriptPath = new File(sandboxFolder, scriptName).getAbsolutePath();
} }
AnalysisLogger.getLogger().debug("Creating local file from remote file"); AnalysisLogger.getLogger().debug("Creating local file from remote file");
if (inputFileURL.startsWith("http:") || inputFileURL.startsWith("smp:")) { if (inputFileURL.startsWith("http:") || inputFileURL.startsWith("smp:")) {
StorageUtils.downloadInputFile(inputFileURL, newInputFile); StorageUtils.downloadInputFile(inputFileURL, newInputFile);
} else if (inputFileURL.trim().length() > 0) { } else if (inputFileURL.trim().length() > 0) {
OSCommand.FileCopy(inputFileURL, newInputFile); OSCommand.FileCopy(inputFileURL, newInputFile);
} }
tempfiles.add(preparedScriptPath); tempfiles.add(preparedScriptPath);
scriptPath = preparedScriptPath; scriptPath = preparedScriptPath;
} }
String newOutputFile = UUID.randomUUID().toString() + defaultOutputFile; String newOutputFile = UUID.randomUUID().toString() + defaultOutputFile;
String newOutputFilePath = new File(sandboxFolder, newOutputFile).getAbsolutePath().replace("\\", "/"); String newOutputFilePath = new File(sandboxFolder, newOutputFile).getAbsolutePath().replace("\\", "/");
if (defaultOutputFile!=null){ if (defaultOutputFile != null) {
int extension = defaultOutputFile.lastIndexOf("."); int extension = defaultOutputFile.lastIndexOf(".");
if (extension > 0) if (extension > 0)
newOutputFile = defaultOutputFile.substring(0, extension) + "_" + UUID.randomUUID().toString() + "." + defaultOutputFile.substring(extension + 1); newOutputFile = defaultOutputFile.substring(0, extension) + "_" + UUID.randomUUID().toString() + "." + defaultOutputFile.substring(extension + 1);
newOutputFilePath = new File(sandboxFolder, newOutputFile).getAbsolutePath().replace("\\", "/"); newOutputFilePath = new File(sandboxFolder, newOutputFile).getAbsolutePath().replace("\\", "/");
AnalysisLogger.getLogger().debug("New Output File is " + newOutputFilePath); AnalysisLogger.getLogger().debug("New Output File is " + newOutputFilePath);
AnalysisLogger.getLogger().debug("Substituting standard Output"); AnalysisLogger.getLogger().debug("Substituting standard Output");
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath(); preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
substituteStringInFile(scriptPath, preparedScriptPath, defaultOutputFile, newOutputFilePath); substituteStringInFile(scriptPath, preparedScriptPath, defaultOutputFile, newOutputFilePath,false);
tempfiles.add(preparedScriptPath); tempfiles.add(preparedScriptPath);
scriptPath = preparedScriptPath; scriptPath = preparedScriptPath;
} }
AnalysisLogger.getLogger().debug("Executing R script " + scriptPath); AnalysisLogger.getLogger().debug("Executing R script " + scriptPath);
status = 10; status = 10;
// run the R code // run the R code
String scriptdir = new File(scriptPath).getParent().replace("\\", "/"); String scriptdir = new File(scriptPath).getParent().replace("\\", "/");
process = Runtime.getRuntime().exec("R --no-save\n",null,new File(scriptdir)); process = Runtime.getRuntime().exec("R --no-save\n", null, new File(scriptdir));
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(process.getOutputStream())); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(process.getOutputStream()));
//set the working directory to the one of the script // set the working directory to the one of the script
bw.write("setwd(\""+scriptdir+"\")\n"); bw.write("setwd(\"" + scriptdir + "\")\n");
for (String inputparam : inputParameters.keySet()) { for (String inputparam : inputParameters.keySet()) {
String value = inputParameters.get(inputparam); String value = inputParameters.get(inputparam);
bw.write(inputparam + "<-" + value + "\n"); bw.write(inputparam + "=" + value + "\n");
} }
bw.write("source('" + scriptPath.replace("\\", "/") + "')\n"); bw.write("source('" + scriptPath.replace("\\", "/") + "')\n");
// bw.write("source('" + originalScriptPath.replace("\\", "/") + "')\n"); // bw.write("source('" + originalScriptPath.replace("\\", "/") + "')\n");
if (outputParameters!=null){ if (outputParameters != null) {
for (String outputparam : outputParameters.keySet()) { for (String outputparam : outputParameters.keySet()) {
bw.write("paste(\"out->\","+outputparam+")"+ "\n"); bw.write("paste(\"out->\"," + outputparam + ")" + "\n");
bw.write("write.csv("+outputparam+",file=\""+new File(sandboxFolder,outputparam).getAbsolutePath().replace("\\", "/")+"\",row.names=FALSE,append=F,fileEncoding=\"UTF-8\",eol = \"\\n\")"+ "\n"); bw.write("write.csv(" + outputparam + ",file=\"" + new File(sandboxFolder, outputparam).getAbsolutePath().replace("\\", "/") + "\",row.names=FALSE,append=F,fileEncoding=\"UTF-8\",eol = \"\\n\")" + "\n");
} }
} }
bw.write("q()\n"); bw.write("q()\n");
bw.close(); bw.close();
printRConsole(process); RLog = printRConsole(process);
process.destroy(); process.destroy();
if ((new File(newOutputFilePath)).exists()) { if ((new File(newOutputFilePath)).exists()) {
if (uploadOutputOnStorage) { if (uploadOutputOnStorage) {
AnalysisLogger.getLogger().debug("Found output file ... saving on the workspace"); AnalysisLogger.getLogger().debug("Found output file ... saving on the workspace");
String outputURL = StorageUtils.uploadFilesOnStorage(scope, owner, sandboxFolder, newOutputFile); String outputURL = StorageUtils.uploadFilesOnStorage(scope, owner, sandboxFolder, newOutputFile);
@ -253,6 +270,8 @@ public class LocalRScriptsManager {
AnalysisLogger.getLogger().debug("Done: Deleted temp files"); AnalysisLogger.getLogger().debug("Done: Deleted temp files");
} }
} }
return RLog;
} }
public void stop() { public void stop() {
@ -299,7 +318,7 @@ public class LocalRScriptsManager {
inputParameters.put("npoints", "10"); inputParameters.put("npoints", "10");
inputParameters.put("equalDist", "TRUE"); inputParameters.put("equalDist", "TRUE");
scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true,"./"); scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true, "./");
} }
} }