From 0eec164e5cdddb3007c4f4433f1bb25904af44ee Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Mon, 1 Feb 2016 22:07:16 +0000 Subject: [PATCH] R log as output in case of error git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@122703 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../rscripts/generic/GenericRScript.java | 256 ++++++++++-------- 1 file changed, 147 insertions(+), 109 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/executor/rscripts/generic/GenericRScript.java b/src/main/java/org/gcube/dataanalysis/executor/rscripts/generic/GenericRScript.java index 1d6f374..07bbf3b 100644 --- a/src/main/java/org/gcube/dataanalysis/executor/rscripts/generic/GenericRScript.java +++ b/src/main/java/org/gcube/dataanalysis/executor/rscripts/generic/GenericRScript.java @@ -1,6 +1,8 @@ package org.gcube.dataanalysis.executor.rscripts.generic; import java.io.File; +import java.io.FileWriter; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -40,116 +42,153 @@ public abstract class GenericRScript extends StandardLocalExternalAlgorithm { @Override protected void process() throws Exception { boolean deletefiles = false; // for test only + String Rlog = null; // init status status = 0; - // instantiate the R Script executor - scriptmanager = new LocalRScriptsManager(); - // download and unzip the script in a controlled folder - AnalysisLogger.getLogger().debug("Starting script"); - - File localPackage = new File(config.getConfigPath(), "package" + UUID.randomUUID() + ".zip"); - AnalysisLogger.getLogger().debug("Downloading package " + packageURL + "as: " + localPackage.getAbsolutePath()); - StorageUtils.downloadInputFile(packageURL, localPackage.getAbsolutePath(), true); - AnalysisLogger.getLogger().debug("Generating sandbox folder"); - File folder = new File(config.getConfigPath(), "rscr_" + UUID.randomUUID()); - boolean mkdir = folder.mkdir(); - AnalysisLogger.getLogger().debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir); - AnalysisLogger.getLogger().debug("Unzipping package into " + folder.getAbsolutePath()); - ZipTools.unZip(localPackage.getAbsolutePath(), folder.getAbsolutePath()); - localPackage.delete(); - - // File folder = new File("C:/Users/coro/Desktop/WorkFolder/Workspace/EcologicalEngineSmartExecutor/./cfg/rscr_7d329495-b048-4ce0-8bcc-bd74966db56d/"); - - AnalysisLogger.getLogger().debug("Analysing inputs"); - // distinguish the environmental variables and the input variables - HashMap inputParameters = new LinkedHashMap(); - int i = 0; - List inputs = getInputParameters(); - for (String input : inputvariables) { - String value = config.getParam(input); - String defaultValue = inputs.get(i).getDefaultValue(); - //inputParameters.put(defaultValue, value); - String punct = "[ \";]*"; - String regexp = punct+input+punct+"(<-|=)"+punct+defaultValue+punct+".*"; - boolean string = true; - try{ - Double.parseDouble(value); - string = false; - }catch(Exception ee){ - - } - String line = input+"="+value; - if (string) - line = input+"=\""+value+"\""; - - inputParameters.put(regexp, line); - i++; - } - AnalysisLogger.getLogger().debug("Inputs: " + inputParameters.toString()); - AnalysisLogger.getLogger().debug("Analysing environmental variables"); - HashMap environmentalParameters = new LinkedHashMap(); - for (String environment : environmentalvariables) { - String value = config.getParam(environment); - environmentalParameters.put(environment, value); - } - AnalysisLogger.getLogger().debug("Environmental: " + environmentalParameters.toString()); - AnalysisLogger.getLogger().debug("Analysing output variables"); - HashMap outputParameters = new LinkedHashMap(); - for (String output : outputvariables) { - outputParameters.put(output, ""); - } - AnalysisLogger.getLogger().debug("Output: " + outputParameters.toString()); - - AnalysisLogger.getLogger().debug("Main script: " + mainScriptName); - AnalysisLogger.getLogger().debug("Processing main script name"); - // build the script name - String scriptName = mainScriptName; - String scriptSubfolder = ""; - int lioSlash = scriptName.lastIndexOf("/"); - if (lioSlash > -1) { - scriptSubfolder = scriptName.substring(0, lioSlash); - scriptName = scriptName.substring(lioSlash + 1); - } - scriptSubfolder = new File(folder.getAbsoluteFile(), scriptSubfolder).getAbsolutePath(); - AnalysisLogger.getLogger().debug("Main script name: " + scriptName); - AnalysisLogger.getLogger().debug("Main script folder: " + scriptSubfolder); - - boolean scriptMustReturnAFile = false; - boolean uploadScriptOnTheInfrastructureWorkspace = false; - status = 10; - AnalysisLogger.getLogger().debug("Executing the script..."); - // execute the script in multi-user mode - - scriptmanager.executeRScript(config, scriptName, null, environmentalParameters, outputParameters, null, null, inputParameters, scriptMustReturnAFile, uploadScriptOnTheInfrastructureWorkspace, deletefiles, scriptSubfolder); - AnalysisLogger.getLogger().debug("..execution finished!"); - // get the output: one file should have been produced for each output - AnalysisLogger.getLogger().debug("Getting output"); - for (String output : outputvariables) { - File outPath = new File(scriptSubfolder, output); - AnalysisLogger.getLogger().debug("Output " + output + " - loading respective file from " + outPath.getAbsolutePath()); - AnalysisLogger.getLogger().debug("File exists? " + outPath.exists()); - String fileContent = FileTools.loadString(outPath.getAbsolutePath(), "UTF-8"); - fileContent = analyseContent(fileContent, scriptSubfolder); - AnalysisLogger.getLogger().debug("Retrieved output content: " + output + ": " + fileContent); - outputValues.put(output, fileContent); - } - - // delete the script folder - try { - if (deletefiles) { - AnalysisLogger.getLogger().debug("Deleting sandbox folder"); - FileUtils.cleanDirectory(folder); - FileUtils.deleteDirectory(folder); - AnalysisLogger.getLogger().debug("Folder " + folder.getAbsolutePath() + " deleted"); + // instantiate the R Script executor + scriptmanager = new LocalRScriptsManager(); + // download and unzip the script in a controlled folder + AnalysisLogger.getLogger().debug("Starting script"); + + File localPackage = new File(config.getConfigPath(), "package" + UUID.randomUUID() + ".zip"); + AnalysisLogger.getLogger().debug("Downloading package " + packageURL + "as: " + localPackage.getAbsolutePath()); + StorageUtils.downloadInputFile(packageURL, localPackage.getAbsolutePath(), true); + AnalysisLogger.getLogger().debug("Generating sandbox folder"); + File folder = new File(config.getConfigPath(), "rscr_" + UUID.randomUUID()); + boolean mkdir = folder.mkdir(); + AnalysisLogger.getLogger().debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir); + AnalysisLogger.getLogger().debug("Unzipping package into " + folder.getAbsolutePath()); + ZipTools.unZip(localPackage.getAbsolutePath(), folder.getAbsolutePath()); + localPackage.delete(); + + // File folder = new File("C:/Users/coro/Desktop/WorkFolder/Workspace/EcologicalEngineSmartExecutor/./cfg/rscr_7d329495-b048-4ce0-8bcc-bd74966db56d/"); + + AnalysisLogger.getLogger().debug("Analysing inputs"); + // distinguish the environmental variables and the input variables + HashMap inputParameters = new LinkedHashMap(); + int i = 0; + List inputs = getInputParameters(); + for (String input : inputvariables) { + String value = config.getParam(input); + String defaultValue = inputs.get(i).getDefaultValue(); + defaultValue = defaultValue.replace("(", "\\(").replace(")", "\\)").replace("[", "\\[").replace("]", "\\]").replace("|", "\\|").replace(".", "\\.").replace("?", "\\?").replace("*", "\\*").replace("+", "\\+").replace("{", "\\{").replace("}", "\\}"); + // inputParameters.put(defaultValue, value); + String punct = "[ \";]*"; + String regexp = punct + input + punct + "(<-|=)" + punct + defaultValue + punct + ".*"; + boolean string = true; + try { + Double.parseDouble(value); + string = false; + } catch (Exception ee) { + + } + String line = input + "=" + value; + if (string && !value.equals("T") && !value.equalsIgnoreCase("true") && !value.equalsIgnoreCase("false") && !value.equalsIgnoreCase("F")) + line = input + "=\"" + value + "\""; + + inputParameters.put(regexp, line); + i++; + } + AnalysisLogger.getLogger().debug("Inputs: " + inputParameters.toString()); + AnalysisLogger.getLogger().debug("Analysing environmental variables"); + HashMap environmentalParameters = new LinkedHashMap(); + for (String environment : environmentalvariables) { + String value = config.getParam(environment); + environmentalParameters.put(environment, value); + } + AnalysisLogger.getLogger().debug("Environmental: " + environmentalParameters.toString()); + AnalysisLogger.getLogger().debug("Analysing output variables"); + HashMap outputParameters = new LinkedHashMap(); + for (String output : outputvariables) { + outputParameters.put(output, ""); + } + AnalysisLogger.getLogger().debug("Output: " + outputParameters.toString()); + + AnalysisLogger.getLogger().debug("Main script: " + mainScriptName); + AnalysisLogger.getLogger().debug("Processing main script name"); + // build the script name + String scriptName = mainScriptName; + String scriptSubfolder = ""; + int lioSlash = scriptName.lastIndexOf("/"); + if (lioSlash > -1) { + scriptSubfolder = scriptName.substring(0, lioSlash); + scriptName = scriptName.substring(lioSlash + 1); + } + scriptSubfolder = new File(folder.getAbsoluteFile(), scriptSubfolder).getAbsolutePath(); + AnalysisLogger.getLogger().debug("Main script name: " + scriptName); + AnalysisLogger.getLogger().debug("Main script folder: " + scriptSubfolder); + + boolean scriptMustReturnAFile = false; + boolean uploadScriptOnTheInfrastructureWorkspace = false; + status = 10; + AnalysisLogger.getLogger().debug("Executing the script..."); + // execute the script in multi-user mode + + Rlog = scriptmanager.executeRScript(config, scriptName, null, environmentalParameters, outputParameters, null, null, inputParameters, scriptMustReturnAFile, uploadScriptOnTheInfrastructureWorkspace, deletefiles, scriptSubfolder); + AnalysisLogger.getLogger().debug("..execution finished!"); + // get the output: one file should have been produced for each output + AnalysisLogger.getLogger().debug("Getting output"); + for (String output : outputvariables) { + File outPath = new File(scriptSubfolder, output); + AnalysisLogger.getLogger().debug("Output " + output + " - loading respective file from " + outPath.getAbsolutePath()); + AnalysisLogger.getLogger().debug("File exists? " + outPath.exists()); + String fileContent = FileTools.loadString(outPath.getAbsolutePath(), "UTF-8"); + fileContent = analyseContent(fileContent, scriptSubfolder); + AnalysisLogger.getLogger().debug("Retrieved output content: " + output + ": " + fileContent); + outputValues.put(output, fileContent); } + // delete the script folder + + try { + if (deletefiles) { + AnalysisLogger.getLogger().debug("Deleting sandbox folder"); + FileUtils.cleanDirectory(folder); + FileUtils.deleteDirectory(folder); + AnalysisLogger.getLogger().debug("Folder " + folder.getAbsolutePath() + " deleted"); + } + + } catch (Exception e) { + AnalysisLogger.getLogger().debug(e); + AnalysisLogger.getLogger().debug("Could not delete sandbox folder " + folder.getAbsolutePath()); + } } catch (Exception e) { - AnalysisLogger.getLogger().debug(e); - AnalysisLogger.getLogger().debug("Could not delete sandbox folder " + folder.getAbsolutePath()); + + if (Rlog != null) { + String httpurl = generateRemoteLogFile(Rlog); + String message = "Logs of the script can be found at "+httpurl; + e = new Exception(message); + } + throw e; + } finally { + AnalysisLogger.getLogger().debug("Computation finished."); + status = 100; } - AnalysisLogger.getLogger().debug("Computation finished."); - status = 100; + } + + protected String generateRemoteLogFile(String Rlog) throws Exception { + String uuid = "" + UUID.randomUUID(); + AnalysisLogger.getLogger().debug("Writing the logs of the execution"); + File logfile = new File(config.getPersistencePath(), "RLOG" + uuid + ".txt"); + + FileWriter fw = new FileWriter(logfile); + fw.write(Rlog); + fw.close(); + AnalysisLogger.getLogger().debug("Written in " + logfile); + String url = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("Username"), logfile.getParent(), "/ScriptLogs/" + uuid + "/", logfile.getName()); + AnalysisLogger.getLogger().debug("Uploaded on storage: " + url); + String httpurl = ""; + // String httpurl = url.replace("smp:", "http:"); + if (config.getGcubeScope().startsWith("/gcube")) + httpurl = "http://data-d.d4science.org/uri-resolver/smp?smp-uri=" + url + "&fileName=" + logfile.getName() ; + else + httpurl = "http://data.d4science.org/uri-resolver/smp?smp-uri=" + url + "&fileName=" + logfile.getName() ; + + AnalysisLogger.getLogger().debug("Deleting log file and returning " + httpurl); + logfile.delete(); + // httpurl = URLEncoder.encode(httpurl, "UTF-8"); + return httpurl; } protected String analyseContent(String filecontent, String scriptfolder) throws Exception { @@ -161,21 +200,20 @@ public abstract class GenericRScript extends StandardLocalExternalAlgorithm { if (row != null) { row = row.replace("\"", ""); File rowFile = new File(row.trim()); - AnalysisLogger.getLogger().debug("Checking row file-> "+rowFile.getAbsolutePath()); - if (!rowFile.exists()){ + AnalysisLogger.getLogger().debug("Checking row file-> " + rowFile.getAbsolutePath()); + if (!rowFile.exists()) { rowFile = new File(scriptfolder, row.trim()); - AnalysisLogger.getLogger().debug("File does not exist - checking complete row file-> "+rowFile.getAbsolutePath()); + AnalysisLogger.getLogger().debug("File does not exist - checking complete row file-> " + rowFile.getAbsolutePath()); } if (rowFile.exists()) { - + String preparedFile = new File(config.getConfigPath(), rowFile.getName()).getAbsolutePath(); AnalysisLogger.getLogger().debug("Copying " + rowFile.getAbsolutePath() + " to " + preparedFile); FileUtils.moveFileToDirectory(rowFile, new File(config.getConfigPath()), false); files.add(preparedFile); break; - } - else + } else AnalysisLogger.getLogger().debug("Checking row file does not exist - treating as a String"); } }