git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@151026 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
bcdad87778
commit
fee78b45fe
|
@ -9,7 +9,6 @@ import java.util.List;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
|
||||||
import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools;
|
import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools;
|
||||||
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
|
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
|
||||||
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
||||||
|
@ -18,9 +17,13 @@ import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm;
|
||||||
import org.gcube.dataanalysis.ecoengine.utils.ZipTools;
|
import org.gcube.dataanalysis.ecoengine.utils.ZipTools;
|
||||||
import org.gcube.dataanalysis.executor.util.LocalRScriptsManager;
|
import org.gcube.dataanalysis.executor.util.LocalRScriptsManager;
|
||||||
import org.gcube.dataanalysis.executor.util.StorageUtils;
|
import org.gcube.dataanalysis.executor.util.StorageUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(GenericRScript.class);
|
||||||
|
|
||||||
// FIXED part
|
// FIXED part
|
||||||
protected HashMap<String, String> outputValues = new HashMap<String, String>();
|
protected HashMap<String, String> outputValues = new HashMap<String, String>();
|
||||||
protected LinkedHashMap<String, StatisticalType> output = new LinkedHashMap<String, StatisticalType>();
|
protected LinkedHashMap<String, StatisticalType> output = new LinkedHashMap<String, StatisticalType>();
|
||||||
|
@ -36,7 +39,7 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void init() throws Exception {
|
public void init() throws Exception {
|
||||||
AnalysisLogger.getLogger().debug("Initializing " + this.getClass().getCanonicalName());
|
LOGGER.debug("Initializing " + this.getClass().getCanonicalName());
|
||||||
initVariables();
|
initVariables();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,22 +53,22 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
// instantiate the R Script executor
|
// instantiate the R Script executor
|
||||||
scriptmanager = new LocalRScriptsManager();
|
scriptmanager = new LocalRScriptsManager();
|
||||||
// download and unzip the script in a controlled folder
|
// download and unzip the script in a controlled folder
|
||||||
AnalysisLogger.getLogger().debug("Starting script");
|
LOGGER.debug("Starting script");
|
||||||
|
|
||||||
File localPackage = new File(config.getConfigPath(), "package" + UUID.randomUUID() + ".zip");
|
File localPackage = new File(config.getConfigPath(), "package" + UUID.randomUUID() + ".zip");
|
||||||
AnalysisLogger.getLogger().debug("Downloading package " + packageURL + "as: " + localPackage.getAbsolutePath());
|
LOGGER.debug("Downloading package " + packageURL + "as: " + localPackage.getAbsolutePath());
|
||||||
StorageUtils.downloadInputFile(packageURL, localPackage.getAbsolutePath(), true);
|
StorageUtils.downloadInputFile(packageURL, localPackage.getAbsolutePath(), true);
|
||||||
AnalysisLogger.getLogger().debug("Generating sandbox folder");
|
LOGGER.debug("Generating sandbox folder");
|
||||||
File folder = new File(config.getConfigPath(), "rscr_" + UUID.randomUUID());
|
File folder = new File(config.getConfigPath(), "rscr_" + UUID.randomUUID());
|
||||||
boolean mkdir = folder.mkdir();
|
boolean mkdir = folder.mkdir();
|
||||||
AnalysisLogger.getLogger().debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir);
|
LOGGER.debug("Sandbox " + folder.getAbsolutePath() + " generated: " + mkdir);
|
||||||
AnalysisLogger.getLogger().debug("Unzipping package into " + folder.getAbsolutePath());
|
LOGGER.debug("Unzipping package into " + folder.getAbsolutePath());
|
||||||
ZipTools.unZip(localPackage.getAbsolutePath(), folder.getAbsolutePath());
|
ZipTools.unZip(localPackage.getAbsolutePath(), folder.getAbsolutePath());
|
||||||
localPackage.delete();
|
localPackage.delete();
|
||||||
|
|
||||||
// File folder = new File("C:/Users/coro/Desktop/WorkFolder/Workspace/EcologicalEngineSmartExecutor/./cfg/rscr_7d329495-b048-4ce0-8bcc-bd74966db56d/");
|
// File folder = new File("C:/Users/coro/Desktop/WorkFolder/Workspace/EcologicalEngineSmartExecutor/./cfg/rscr_7d329495-b048-4ce0-8bcc-bd74966db56d/");
|
||||||
|
|
||||||
AnalysisLogger.getLogger().debug("Analysing inputs");
|
LOGGER.debug("Analysing inputs");
|
||||||
// distinguish the environmental variables and the input variables
|
// distinguish the environmental variables and the input variables
|
||||||
HashMap<String, String> inputParameters = new LinkedHashMap<String, String>();
|
HashMap<String, String> inputParameters = new LinkedHashMap<String, String>();
|
||||||
int i = 0;
|
int i = 0;
|
||||||
|
@ -93,23 +96,23 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
inputParameters.put(regexp, line);
|
inputParameters.put(regexp, line);
|
||||||
i++;
|
i++;
|
||||||
}
|
}
|
||||||
AnalysisLogger.getLogger().debug("Inputs: " + inputParameters.toString());
|
LOGGER.debug("Inputs: " + inputParameters.toString());
|
||||||
AnalysisLogger.getLogger().debug("Analysing environmental variables");
|
LOGGER.debug("Analysing environmental variables");
|
||||||
HashMap<String, String> environmentalParameters = new LinkedHashMap<String, String>();
|
HashMap<String, String> environmentalParameters = new LinkedHashMap<String, String>();
|
||||||
for (String environment : environmentalvariables) {
|
for (String environment : environmentalvariables) {
|
||||||
String value = config.getParam(environment);
|
String value = config.getParam(environment);
|
||||||
environmentalParameters.put(environment, value);
|
environmentalParameters.put(environment, value);
|
||||||
}
|
}
|
||||||
AnalysisLogger.getLogger().debug("Environmental: " + environmentalParameters.toString());
|
LOGGER.debug("Environmental: " + environmentalParameters.toString());
|
||||||
AnalysisLogger.getLogger().debug("Analysing output variables");
|
LOGGER.debug("Analysing output variables");
|
||||||
HashMap<String, String> outputParameters = new LinkedHashMap<String, String>();
|
HashMap<String, String> outputParameters = new LinkedHashMap<String, String>();
|
||||||
for (String output : outputvariables) {
|
for (String output : outputvariables) {
|
||||||
outputParameters.put(output, "");
|
outputParameters.put(output, "");
|
||||||
}
|
}
|
||||||
AnalysisLogger.getLogger().debug("Output: " + outputParameters.toString());
|
LOGGER.debug("Output: " + outputParameters.toString());
|
||||||
|
|
||||||
AnalysisLogger.getLogger().debug("Main script: " + mainScriptName);
|
LOGGER.debug("Main script: " + mainScriptName);
|
||||||
AnalysisLogger.getLogger().debug("Processing main script name");
|
LOGGER.debug("Processing main script name");
|
||||||
// build the script name
|
// build the script name
|
||||||
String scriptName = mainScriptName;
|
String scriptName = mainScriptName;
|
||||||
String scriptSubfolder = "";
|
String scriptSubfolder = "";
|
||||||
|
@ -119,26 +122,26 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
scriptName = scriptName.substring(lioSlash + 1);
|
scriptName = scriptName.substring(lioSlash + 1);
|
||||||
}
|
}
|
||||||
scriptSubfolder = new File(folder.getAbsoluteFile(), scriptSubfolder).getAbsolutePath();
|
scriptSubfolder = new File(folder.getAbsoluteFile(), scriptSubfolder).getAbsolutePath();
|
||||||
AnalysisLogger.getLogger().debug("Main script name: " + scriptName);
|
LOGGER.debug("Main script name: " + scriptName);
|
||||||
AnalysisLogger.getLogger().debug("Main script folder: " + scriptSubfolder);
|
LOGGER.debug("Main script folder: " + scriptSubfolder);
|
||||||
|
|
||||||
boolean scriptMustReturnAFile = false;
|
boolean scriptMustReturnAFile = false;
|
||||||
boolean uploadScriptOnTheInfrastructureWorkspace = false;
|
boolean uploadScriptOnTheInfrastructureWorkspace = false;
|
||||||
status = 10;
|
status = 10;
|
||||||
AnalysisLogger.getLogger().debug("Executing the script...");
|
LOGGER.debug("Executing the script...");
|
||||||
// execute the script in multi-user mode
|
// execute the script in multi-user mode
|
||||||
|
|
||||||
Rlog = scriptmanager.executeRScript(config, scriptName, null, environmentalParameters, outputParameters, null, null, inputParameters, scriptMustReturnAFile, uploadScriptOnTheInfrastructureWorkspace, deletefiles, scriptSubfolder);
|
Rlog = scriptmanager.executeRScript(config, scriptName, null, environmentalParameters, outputParameters, null, null, inputParameters, scriptMustReturnAFile, uploadScriptOnTheInfrastructureWorkspace, deletefiles, scriptSubfolder);
|
||||||
AnalysisLogger.getLogger().debug("..execution finished!");
|
LOGGER.debug("..execution finished!");
|
||||||
// get the output: one file should have been produced for each output
|
// get the output: one file should have been produced for each output
|
||||||
AnalysisLogger.getLogger().debug("Getting output");
|
LOGGER.debug("Getting output");
|
||||||
for (String output : outputvariables) {
|
for (String output : outputvariables) {
|
||||||
File outPath = new File(scriptSubfolder, output);
|
File outPath = new File(scriptSubfolder, output);
|
||||||
AnalysisLogger.getLogger().debug("Output " + output + " - loading respective file from " + outPath.getAbsolutePath());
|
LOGGER.debug("Output " + output + " - loading respective file from " + outPath.getAbsolutePath());
|
||||||
AnalysisLogger.getLogger().debug("File exists? " + outPath.exists());
|
LOGGER.debug("File exists? " + outPath.exists());
|
||||||
String fileContent = FileTools.loadString(outPath.getAbsolutePath(), "UTF-8");
|
String fileContent = FileTools.loadString(outPath.getAbsolutePath(), "UTF-8");
|
||||||
fileContent = analyseContent(fileContent, scriptSubfolder);
|
fileContent = analyseContent(fileContent, scriptSubfolder);
|
||||||
AnalysisLogger.getLogger().debug("Retrieved output content: " + output + ": " + fileContent);
|
LOGGER.debug("Retrieved output content: " + output + ": " + fileContent);
|
||||||
outputValues.put(output, fileContent);
|
outputValues.put(output, fileContent);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,14 +149,14 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (deletefiles) {
|
if (deletefiles) {
|
||||||
AnalysisLogger.getLogger().debug("Deleting sandbox folder");
|
LOGGER.debug("Deleting sandbox folder");
|
||||||
FileUtils.cleanDirectory(folder);
|
FileUtils.cleanDirectory(folder);
|
||||||
FileUtils.deleteDirectory(folder);
|
FileUtils.deleteDirectory(folder);
|
||||||
AnalysisLogger.getLogger().debug("Folder " + folder.getAbsolutePath() + " deleted");
|
LOGGER.debug("Folder " + folder.getAbsolutePath() + " deleted");
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
AnalysisLogger.getLogger().warn("Could not delete sandbox folder "+folder.getAbsolutePath(),e);
|
LOGGER.warn("Could not delete sandbox folder "+folder.getAbsolutePath(),e);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Rlog != null) {
|
if (Rlog != null) {
|
||||||
|
@ -170,34 +173,34 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
}
|
}
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
} finally {
|
||||||
AnalysisLogger.getLogger().debug("Computation finished.");
|
LOGGER.debug("Computation finished.");
|
||||||
status = 100;
|
status = 100;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected File saveLogFile(String Rlog) throws Exception {
|
protected File saveLogFile(String Rlog) throws Exception {
|
||||||
String uuid = "" + UUID.randomUUID();
|
String uuid = UUID.randomUUID().toString();
|
||||||
AnalysisLogger.getLogger().debug("Writing the logs of the execution");
|
LOGGER.debug("Writing the logs of the execution");
|
||||||
File logfile = new File(config.getPersistencePath(), "RLOG" + uuid + ".txt");
|
File logfile = new File(config.getPersistencePath(), "RLOG" + uuid + ".txt");
|
||||||
|
|
||||||
FileWriter fw = new FileWriter(logfile);
|
FileWriter fw = new FileWriter(logfile);
|
||||||
fw.write(Rlog);
|
fw.write(Rlog);
|
||||||
fw.close();
|
fw.close();
|
||||||
AnalysisLogger.getLogger().debug("Written in " + logfile);
|
LOGGER.debug("Written in " + logfile);
|
||||||
return logfile;
|
return logfile;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String generateRemoteLogFile(String Rlog) throws Exception {
|
protected String generateRemoteLogFile(String Rlog) throws Exception {
|
||||||
String uuid = "" + UUID.randomUUID();
|
String uuid = "" + UUID.randomUUID();
|
||||||
AnalysisLogger.getLogger().debug("Writing the logs of the execution");
|
LOGGER.debug("Writing the logs of the execution");
|
||||||
File logfile = new File(config.getPersistencePath(), "RLOG" + uuid + ".txt");
|
File logfile = new File(config.getPersistencePath(), "RLOG" + uuid + ".txt");
|
||||||
|
|
||||||
FileWriter fw = new FileWriter(logfile);
|
FileWriter fw = new FileWriter(logfile);
|
||||||
fw.write(Rlog);
|
fw.write(Rlog);
|
||||||
fw.close();
|
fw.close();
|
||||||
AnalysisLogger.getLogger().debug("Written in " + logfile);
|
LOGGER.debug("Written in " + logfile);
|
||||||
String httpurl = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), logfile.getParent(), "/ScriptLogs/" + uuid + "/", logfile.getName(),true);
|
String httpurl = StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), logfile.getParent(), "/ScriptLogs/" + uuid + "/", logfile.getName(),true);
|
||||||
AnalysisLogger.getLogger().debug("Uploaded on storage: " + httpurl);
|
LOGGER.debug("Uploaded on storage: " + httpurl);
|
||||||
|
|
||||||
// String httpurl = url.replace("smp:", "http:");
|
// String httpurl = url.replace("smp:", "http:");
|
||||||
/*
|
/*
|
||||||
|
@ -207,48 +210,48 @@ public abstract class GenericRScript extends StandardLocalInfraAlgorithm {
|
||||||
httpurl = "http://data.d4science.org/uri-resolver/smp?smp-uri=" + url + "&fileName=" + logfile.getName() ;
|
httpurl = "http://data.d4science.org/uri-resolver/smp?smp-uri=" + url + "&fileName=" + logfile.getName() ;
|
||||||
*/
|
*/
|
||||||
|
|
||||||
AnalysisLogger.getLogger().debug("Deleting log file and returning " + httpurl);
|
LOGGER.debug("Deleting log file and returning " + httpurl);
|
||||||
logfile.delete();
|
logfile.delete();
|
||||||
// httpurl = URLEncoder.encode(httpurl, "UTF-8");
|
// httpurl = URLEncoder.encode(httpurl, "UTF-8");
|
||||||
return httpurl;
|
return httpurl;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String analyseContent(String filecontent, String scriptfolder) throws Exception {
|
protected String analyseContent(String filecontent, String scriptfolder) throws Exception {
|
||||||
AnalysisLogger.getLogger().debug("Analysing file content");
|
LOGGER.debug("Analysing file content");
|
||||||
String[] rows = filecontent.split(System.lineSeparator());
|
String[] rows = filecontent.split(System.lineSeparator());
|
||||||
List<String> files = new ArrayList<String>();
|
List<String> files = new ArrayList<String>();
|
||||||
for (String row : rows) {
|
for (String row : rows) {
|
||||||
AnalysisLogger.getLogger().debug("Analysing -> " + row);
|
LOGGER.debug("Analysing -> " + row);
|
||||||
if (row != null) {
|
if (row != null) {
|
||||||
row = row.replace("\"", "");
|
row = row.replace("\"", "");
|
||||||
File rowFile = new File(row.trim());
|
File rowFile = new File(row.trim());
|
||||||
AnalysisLogger.getLogger().debug("Checking row file-> " + rowFile.getAbsolutePath());
|
LOGGER.debug("Checking row file-> " + rowFile.getAbsolutePath());
|
||||||
if (!rowFile.exists()) {
|
if (!rowFile.exists()) {
|
||||||
rowFile = new File(scriptfolder, row.trim());
|
rowFile = new File(scriptfolder, row.trim());
|
||||||
AnalysisLogger.getLogger().debug("File does not exist - checking complete row file-> " + rowFile.getAbsolutePath());
|
LOGGER.debug("File does not exist - checking complete row file-> " + rowFile.getAbsolutePath());
|
||||||
}
|
}
|
||||||
if (rowFile.exists()) {
|
if (rowFile.exists()) {
|
||||||
|
|
||||||
String preparedFile = new File(config.getConfigPath(), rowFile.getName()).getAbsolutePath();
|
String preparedFile = new File(config.getConfigPath(), rowFile.getName()).getAbsolutePath();
|
||||||
AnalysisLogger.getLogger().debug("Copying " + rowFile.getAbsolutePath() + " to " + preparedFile);
|
LOGGER.debug("Copying " + rowFile.getAbsolutePath() + " to " + preparedFile);
|
||||||
try{
|
try{
|
||||||
org.gcube.dataanalysis.executor.rscripts.generic.FileUtils.moveFileToDirectory(rowFile, new File(config.getConfigPath()), false);
|
org.gcube.dataanalysis.executor.rscripts.generic.FileUtils.moveFileToDirectory(rowFile, new File(config.getConfigPath()), false);
|
||||||
files.add(preparedFile);
|
files.add(preparedFile);
|
||||||
}catch(Exception e){
|
}catch(Exception e){
|
||||||
AnalysisLogger.getLogger().error("error in moving file "+rowFile.getAbsolutePath()+" to "+preparedFile,e);
|
LOGGER.error("error in moving file "+rowFile.getAbsolutePath()+" to "+preparedFile,e);
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
} else
|
} else
|
||||||
AnalysisLogger.getLogger().debug("Checking row file does not exist - treating as a String");
|
LOGGER.debug("Checking row file does not exist - treating as a String");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (files.size() > 0) {
|
if (files.size() > 0) {
|
||||||
AnalysisLogger.getLogger().debug("A File was recognized as output");
|
LOGGER.debug("A File was recognized as output");
|
||||||
return files.get(0);
|
return files.get(0);
|
||||||
} else {
|
} else {
|
||||||
AnalysisLogger.getLogger().debug("A String was recognized as output");
|
LOGGER.debug("A String was recognized as output");
|
||||||
return filecontent;
|
return filecontent;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue