git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@122705 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
d8f7c33be7
commit
b3c9a340d2
|
@ -17,6 +17,7 @@ import org.gcube.common.scope.api.ScopeProvider;
|
||||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||||
import org.gcube.dataanalysis.executor.scripts.OSCommand;
|
import org.gcube.dataanalysis.executor.scripts.OSCommand;
|
||||||
|
import org.junit.runner.notification.RunListener;
|
||||||
|
|
||||||
public class LocalRScriptsManager {
|
public class LocalRScriptsManager {
|
||||||
|
|
||||||
|
@ -50,16 +51,27 @@ public class LocalRScriptsManager {
|
||||||
this.status = status;
|
this.status = status;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void substituteStringInFile(String file, String newFile, String s, String sub) throws Exception {
|
|
||||||
|
public static void substituteStringInFile(String file, String newFile, String s, String sub, boolean regex) throws Exception {
|
||||||
BufferedReader br = new BufferedReader(new FileReader(new File(file)));
|
BufferedReader br = new BufferedReader(new FileReader(new File(file)));
|
||||||
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(newFile)));
|
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(newFile)));
|
||||||
String line = br.readLine();
|
String line = br.readLine();
|
||||||
|
|
||||||
while (line != null) {
|
while (line != null) {
|
||||||
|
if (!regex) {
|
||||||
int idx = line.indexOf(s);
|
int idx = line.indexOf(s);
|
||||||
if (idx >= 0) {
|
if (idx >= 0) {
|
||||||
line = line.replace(s, sub);
|
line = line.replace(s, sub);
|
||||||
}
|
}
|
||||||
|
}else{
|
||||||
|
// AnalysisLogger.getLogger().debug("Matching "+line+" vs "+s);
|
||||||
|
if (line.matches(s)){
|
||||||
|
AnalysisLogger.getLogger().debug("Matched Input "+s+" line becomes "+sub);
|
||||||
|
line = sub;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
bw.write(line + "\n");
|
bw.write(line + "\n");
|
||||||
line = br.readLine();
|
line = br.readLine();
|
||||||
}
|
}
|
||||||
|
@ -68,7 +80,8 @@ public class LocalRScriptsManager {
|
||||||
bw.close();
|
bw.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void printRConsole(Process process) {
|
public String printRConsole(Process process) {
|
||||||
|
StringBuffer uberbuffer = new StringBuffer();
|
||||||
try {
|
try {
|
||||||
BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()));
|
BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()));
|
||||||
String line = br.readLine();
|
String line = br.readLine();
|
||||||
|
@ -78,7 +91,7 @@ public class LocalRScriptsManager {
|
||||||
sb.append(line + "\n");
|
sb.append(line + "\n");
|
||||||
}
|
}
|
||||||
AnalysisLogger.getLogger().debug(sb);
|
AnalysisLogger.getLogger().debug(sb);
|
||||||
|
uberbuffer.append(sb.toString());
|
||||||
AnalysisLogger.getLogger().debug("---ERRORS---");
|
AnalysisLogger.getLogger().debug("---ERRORS---");
|
||||||
br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
|
br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
|
||||||
line = br.readLine();
|
line = br.readLine();
|
||||||
|
@ -89,10 +102,13 @@ public class LocalRScriptsManager {
|
||||||
sb.append(line + "\n");
|
sb.append(line + "\n");
|
||||||
}
|
}
|
||||||
AnalysisLogger.getLogger().debug(sb);
|
AnalysisLogger.getLogger().debug(sb);
|
||||||
|
uberbuffer.append(sb.toString());
|
||||||
AnalysisLogger.getLogger().debug("---END OF ERRORS---");
|
AnalysisLogger.getLogger().debug("---END OF ERRORS---");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
AnalysisLogger.getLogger().debug("---END BY PROCESS INTERRUPTION---");
|
AnalysisLogger.getLogger().debug("---END BY PROCESS INTERRUPTION---");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return uberbuffer.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, String sandboxFolder) throws Exception {
|
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, String sandboxFolder) throws Exception {
|
||||||
|
@ -107,9 +123,9 @@ public class LocalRScriptsManager {
|
||||||
executeRScript(config, scriptName, inputFileURL, inputParameters, null, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, deletefiles, sandboxFolder);
|
executeRScript(config, scriptName, inputFileURL, inputParameters, null, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, deletefiles, sandboxFolder);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, HashMap<String, String> outputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles, String sandboxFolder) throws Exception {
|
public String executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap<String, String> inputParameters, HashMap<String, String> outputParameters, String defaultInputFile, String defaultOutputFile, HashMap<String, String> codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles, String sandboxFolder) throws Exception {
|
||||||
List<String> tempfiles = new ArrayList<String>();
|
List<String> tempfiles = new ArrayList<String>();
|
||||||
|
String RLog = "";
|
||||||
try {
|
try {
|
||||||
status = 0;
|
status = 0;
|
||||||
String scriptPath = new File(sandboxFolder, scriptName).getAbsolutePath();
|
String scriptPath = new File(sandboxFolder, scriptName).getAbsolutePath();
|
||||||
|
@ -143,23 +159,23 @@ public class LocalRScriptsManager {
|
||||||
String substitution = codeInjections.get(toSubstitute);
|
String substitution = codeInjections.get(toSubstitute);
|
||||||
AnalysisLogger.getLogger().debug("Substituting : " + toSubstitute + " with " + substitution);
|
AnalysisLogger.getLogger().debug("Substituting : " + toSubstitute + " with " + substitution);
|
||||||
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
|
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
|
||||||
substituteStringInFile(scriptPath, preparedScriptPath, toSubstitute, substitution);
|
substituteStringInFile(scriptPath, preparedScriptPath, toSubstitute, substitution,true);
|
||||||
AnalysisLogger.getLogger().debug("New Script name: " + preparedScriptPath+" exists: "+new File(preparedScriptPath).exists());
|
AnalysisLogger.getLogger().debug("New Script name: " + preparedScriptPath + " exists: " + new File(preparedScriptPath).exists());
|
||||||
tempfiles.add(preparedScriptPath);
|
tempfiles.add(preparedScriptPath);
|
||||||
scriptPath = preparedScriptPath;
|
scriptPath = preparedScriptPath;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (defaultInputFile!=null){
|
if (defaultInputFile != null) {
|
||||||
String newInputFile = new File(sandboxFolder, UUID.randomUUID() + defaultInputFile).getAbsolutePath().replace("\\", "/");
|
String newInputFile = new File(sandboxFolder, UUID.randomUUID() + defaultInputFile).getAbsolutePath().replace("\\", "/");
|
||||||
tempfiles.add(newInputFile);
|
tempfiles.add(newInputFile);
|
||||||
AnalysisLogger.getLogger().debug("New Input File is " + newInputFile);
|
AnalysisLogger.getLogger().debug("New Input File is " + newInputFile);
|
||||||
if (inputFileURL.trim().length() > 0) {
|
if (inputFileURL.trim().length() > 0) {
|
||||||
AnalysisLogger.getLogger().debug("Substituting standard Input");
|
AnalysisLogger.getLogger().debug("Substituting standard Input");
|
||||||
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
|
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
|
||||||
substituteStringInFile(scriptPath, preparedScriptPath, defaultInputFile, newInputFile);
|
substituteStringInFile(scriptPath, preparedScriptPath, defaultInputFile, newInputFile,false);
|
||||||
} else {
|
} else {
|
||||||
if ((preparedScriptPath==null) || (preparedScriptPath.trim().length()==0))
|
if ((preparedScriptPath == null) || (preparedScriptPath.trim().length() == 0))
|
||||||
preparedScriptPath = new File(sandboxFolder, scriptName).getAbsolutePath();
|
preparedScriptPath = new File(sandboxFolder, scriptName).getAbsolutePath();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,7 +192,7 @@ public class LocalRScriptsManager {
|
||||||
String newOutputFile = UUID.randomUUID().toString() + defaultOutputFile;
|
String newOutputFile = UUID.randomUUID().toString() + defaultOutputFile;
|
||||||
String newOutputFilePath = new File(sandboxFolder, newOutputFile).getAbsolutePath().replace("\\", "/");
|
String newOutputFilePath = new File(sandboxFolder, newOutputFile).getAbsolutePath().replace("\\", "/");
|
||||||
|
|
||||||
if (defaultOutputFile!=null){
|
if (defaultOutputFile != null) {
|
||||||
int extension = defaultOutputFile.lastIndexOf(".");
|
int extension = defaultOutputFile.lastIndexOf(".");
|
||||||
if (extension > 0)
|
if (extension > 0)
|
||||||
newOutputFile = defaultOutputFile.substring(0, extension) + "_" + UUID.randomUUID().toString() + "." + defaultOutputFile.substring(extension + 1);
|
newOutputFile = defaultOutputFile.substring(0, extension) + "_" + UUID.randomUUID().toString() + "." + defaultOutputFile.substring(extension + 1);
|
||||||
|
@ -184,7 +200,7 @@ public class LocalRScriptsManager {
|
||||||
AnalysisLogger.getLogger().debug("New Output File is " + newOutputFilePath);
|
AnalysisLogger.getLogger().debug("New Output File is " + newOutputFilePath);
|
||||||
AnalysisLogger.getLogger().debug("Substituting standard Output");
|
AnalysisLogger.getLogger().debug("Substituting standard Output");
|
||||||
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
|
preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath();
|
||||||
substituteStringInFile(scriptPath, preparedScriptPath, defaultOutputFile, newOutputFilePath);
|
substituteStringInFile(scriptPath, preparedScriptPath, defaultOutputFile, newOutputFilePath,false);
|
||||||
tempfiles.add(preparedScriptPath);
|
tempfiles.add(preparedScriptPath);
|
||||||
scriptPath = preparedScriptPath;
|
scriptPath = preparedScriptPath;
|
||||||
}
|
}
|
||||||
|
@ -193,27 +209,28 @@ public class LocalRScriptsManager {
|
||||||
status = 10;
|
status = 10;
|
||||||
// run the R code
|
// run the R code
|
||||||
String scriptdir = new File(scriptPath).getParent().replace("\\", "/");
|
String scriptdir = new File(scriptPath).getParent().replace("\\", "/");
|
||||||
process = Runtime.getRuntime().exec("R --no-save\n",null,new File(scriptdir));
|
process = Runtime.getRuntime().exec("R --no-save\n", null, new File(scriptdir));
|
||||||
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(process.getOutputStream()));
|
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(process.getOutputStream()));
|
||||||
|
|
||||||
//set the working directory to the one of the script
|
// set the working directory to the one of the script
|
||||||
bw.write("setwd(\""+scriptdir+"\")\n");
|
bw.write("setwd(\"" + scriptdir + "\")\n");
|
||||||
for (String inputparam : inputParameters.keySet()) {
|
for (String inputparam : inputParameters.keySet()) {
|
||||||
String value = inputParameters.get(inputparam);
|
String value = inputParameters.get(inputparam);
|
||||||
bw.write(inputparam + "<-" + value + "\n");
|
bw.write(inputparam + "=" + value + "\n");
|
||||||
}
|
}
|
||||||
bw.write("source('" + scriptPath.replace("\\", "/") + "')\n");
|
bw.write("source('" + scriptPath.replace("\\", "/") + "')\n");
|
||||||
// bw.write("source('" + originalScriptPath.replace("\\", "/") + "')\n");
|
// bw.write("source('" + originalScriptPath.replace("\\", "/") + "')\n");
|
||||||
if (outputParameters!=null){
|
if (outputParameters != null) {
|
||||||
for (String outputparam : outputParameters.keySet()) {
|
for (String outputparam : outputParameters.keySet()) {
|
||||||
bw.write("paste(\"out->\","+outputparam+")"+ "\n");
|
bw.write("paste(\"out->\"," + outputparam + ")" + "\n");
|
||||||
bw.write("write.csv("+outputparam+",file=\""+new File(sandboxFolder,outputparam).getAbsolutePath().replace("\\", "/")+"\",row.names=FALSE,append=F,fileEncoding=\"UTF-8\",eol = \"\\n\")"+ "\n");
|
bw.write("write.csv(" + outputparam + ",file=\"" + new File(sandboxFolder, outputparam).getAbsolutePath().replace("\\", "/") + "\",row.names=FALSE,append=F,fileEncoding=\"UTF-8\",eol = \"\\n\")" + "\n");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bw.write("q()\n");
|
bw.write("q()\n");
|
||||||
bw.close();
|
bw.close();
|
||||||
printRConsole(process);
|
RLog = printRConsole(process);
|
||||||
|
|
||||||
process.destroy();
|
process.destroy();
|
||||||
|
|
||||||
if ((new File(newOutputFilePath)).exists()) {
|
if ((new File(newOutputFilePath)).exists()) {
|
||||||
|
@ -253,6 +270,8 @@ public class LocalRScriptsManager {
|
||||||
AnalysisLogger.getLogger().debug("Done: Deleted temp files");
|
AnalysisLogger.getLogger().debug("Done: Deleted temp files");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return RLog;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void stop() {
|
public void stop() {
|
||||||
|
@ -299,7 +318,7 @@ public class LocalRScriptsManager {
|
||||||
inputParameters.put("npoints", "10");
|
inputParameters.put("npoints", "10");
|
||||||
inputParameters.put("equalDist", "TRUE");
|
inputParameters.put("equalDist", "TRUE");
|
||||||
|
|
||||||
scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true,"./");
|
scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true, "./");
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue