package org.gcube.dataanalysis.executor.rscripts.generic; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.UUID; import org.apache.commons.io.FileUtils; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm; import org.gcube.dataanalysis.ecoengine.utils.ZipTools; import org.gcube.dataanalysis.executor.scripts.OSCommand; import org.gcube.dataanalysis.executor.util.LocalRScriptsManager; import org.gcube.dataanalysis.executor.util.RScriptsManager; import org.gcube.dataanalysis.executor.util.StorageUtils; public abstract class GenericRScript extends StandardLocalExternalAlgorithm { //FIXED part protected HashMap outputValues=new HashMap(); protected LinkedHashMap output = new LinkedHashMap (); LocalRScriptsManager scriptmanager; @Override public void shutdown() { //in the case of forced shutdown, stop the R process if (scriptmanager!=null) scriptmanager.stop(); System.gc(); } @Override public void init() throws Exception { AnalysisLogger.getLogger().debug("Initializing "+this.getClass().getCanonicalName()); initVariables(); } @Override protected void process() throws Exception { //init status status = 0; //instantiate the R Script executor scriptmanager = new LocalRScriptsManager(); //download and unzip the script in a controlled folder AnalysisLogger.getLogger().debug("Starting script"); File localPackage = new File(config.getConfigPath(),"package"+UUID.randomUUID()+".zip"); AnalysisLogger.getLogger().debug("Downloading package "+packageURL+"as: "+localPackage.getAbsolutePath()); StorageUtils.downloadInputFile(packageURL, localPackage.getAbsolutePath(),true); AnalysisLogger.getLogger().debug("Generating sandbox folder"); File folder = new File(config.getConfigPath(),"rscr_"+UUID.randomUUID()); boolean mkdir = folder.mkdir(); AnalysisLogger.getLogger().debug("Sandbox "+folder.getAbsolutePath()+" generated: "+mkdir); AnalysisLogger.getLogger().debug("Unzipping package into "+folder.getAbsolutePath()); ZipTools.unZip(localPackage.getAbsolutePath(), folder.getAbsolutePath()); localPackage.delete(); //File folder = new File("C:/Users/coro/Desktop/WorkFolder/Workspace/EcologicalEngineSmartExecutor/./cfg/rscr_7d329495-b048-4ce0-8bcc-bd74966db56d/"); AnalysisLogger.getLogger().debug("Analysing inputs"); //distinguish the environmental variables and the input variables HashMap inputParameters = new LinkedHashMap(); int i=0; Listinputs = getInputParameters(); for (String input:inputvariables){ String value = config.getParam(input); String defaultValue = inputs.get(i).getDefaultValue(); inputParameters.put(defaultValue, value); i++; } AnalysisLogger.getLogger().debug("Inputs: "+inputParameters.toString()); AnalysisLogger.getLogger().debug("Analysing environmental variables"); HashMap environmentalParameters = new LinkedHashMap(); for (String environment:environmentalvariables){ String value = config.getParam(environment); environmentalParameters.put(environment, value); } AnalysisLogger.getLogger().debug("Environmental: "+environmentalParameters.toString()); AnalysisLogger.getLogger().debug("Analysing output variables"); HashMap outputParameters = new LinkedHashMap(); for (String output:outputvariables){ outputParameters.put(output, ""); } AnalysisLogger.getLogger().debug("Output: "+outputParameters.toString()); AnalysisLogger.getLogger().debug("Main script: "+mainScriptName); AnalysisLogger.getLogger().debug("Processing main script name"); //build the script name String scriptName = mainScriptName; String scriptSubfolder = ""; int lioSlash = scriptName.lastIndexOf("/"); if (lioSlash>-1){ scriptSubfolder = scriptName.substring(0,lioSlash); scriptName = scriptName.substring(lioSlash+1); } scriptSubfolder = new File (folder.getAbsoluteFile(),scriptSubfolder).getAbsolutePath(); AnalysisLogger.getLogger().debug("Main script name: "+scriptName); AnalysisLogger.getLogger().debug("Main script folder: "+scriptSubfolder); boolean scriptMustReturnAFile = false; boolean uploadScriptOnTheInfrastructureWorkspace = false; status = 10; AnalysisLogger.getLogger().debug("Executing the script..."); //execute the script in multi-user mode scriptmanager.executeRScript(config, scriptName, null, environmentalParameters, outputParameters, null, null,inputParameters, scriptMustReturnAFile,uploadScriptOnTheInfrastructureWorkspace, true, scriptSubfolder); AnalysisLogger.getLogger().debug("..execution finished!"); //get the output: one file should have been produced for each output AnalysisLogger.getLogger().debug("Getting output"); for (String output:outputvariables){ File outPath = new File(scriptSubfolder,output); AnalysisLogger.getLogger().debug("Output "+output +" - loading respective file from "+outPath.getAbsolutePath()); AnalysisLogger.getLogger().debug("File exists? "+outPath.exists()); String fileContent = FileTools.loadString(outPath.getAbsolutePath(), "UTF-8"); fileContent=analyseContent(fileContent); AnalysisLogger.getLogger().debug("Retrieved output content: "+output+": "+fileContent); outputValues.put(output, fileContent); } //delete the script folder AnalysisLogger.getLogger().debug("Deleting sandbox folder"); try{ FileUtils.cleanDirectory(folder); FileUtils.deleteDirectory(folder); AnalysisLogger.getLogger().debug("Folder "+folder.getAbsolutePath()+ " deleted"); }catch(Exception e){ AnalysisLogger.getLogger().debug(e); AnalysisLogger.getLogger().debug("Could not delete sandbox folder "+folder.getAbsolutePath()); } AnalysisLogger.getLogger().debug("Computation finished."); status = 100; } protected String analyseContent(String filecontent) throws Exception{ String [] rows = filecontent.split("\n"); List files = new ArrayList(); for (String row:rows){ if (row!=null) row = row.replace("\"", ""); if ((new File(row)).exists()){ String preparedFile = new File(config.getConfigPath(),new File(row).getName()).getAbsolutePath(); AnalysisLogger.getLogger().debug("Copying "+row+" to "+preparedFile); FileUtils.moveFileToDirectory(new File(row), new File(config.getConfigPath()), false); files.add(preparedFile); break; } } if (files.size()>0) return files.get(0); else return filecontent; } //DYNAMIC part protected String mainScriptName=""; protected String packageURL=""; protected List environmentalvariables = new ArrayList(); protected List inputvariables= new ArrayList(); protected List outputvariables= new ArrayList(); protected void initVariables(){ } @Override protected void setInputParameters() { } @Override public StatisticalType getOutput() { return null; } }