gCubeSystem
/
ecological-engine-smart-executor
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@131527 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2016-09-19 17:03:48 +00:00
parent 11c044a7bf
commit 4ac15d4fa1
16 changed files with 931 additions and 383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/org/gcube/dataanalysis/executor/generators/D4ScienceDistributedProcessing.java
View File

@ -3,6 +3,7 @@ package org.gcube.dataanalysis.executor.generators;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.UUID;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
@ -15,6 +16,7 @@ import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode;
import org.gcube.dataanalysis.ecoengine.interfaces.Generator; import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm; import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm;
import org.gcube.dataanalysis.executor.job.management.DistributedProcessingAgent; import org.gcube.dataanalysis.executor.job.management.DistributedProcessingAgent;
import org.gcube.dataanalysis.executor.job.management.DistributedProcessingAgentWPS;
public class D4ScienceDistributedProcessing implements Generator { public class D4ScienceDistributedProcessing implements Generator {
@ -24,7 +26,7 @@ public class D4ScienceDistributedProcessing implements Generator {
protected AlgorithmConfiguration config; protected AlgorithmConfiguration config;
protected ActorNode distributedModel; protected ActorNode distributedModel;
protected String mainclass; protected String mainclass;
DistributedProcessingAgent agent; DistributedProcessingAgentWPS agent;
public D4ScienceDistributedProcessing(){ public D4ScienceDistributedProcessing(){
} }
@ -100,10 +102,13 @@ public class D4ScienceDistributedProcessing implements Generator {
distributedModel.setup(config); distributedModel.setup(config);
String scope = config.getGcubeScope(); String scope = config.getGcubeScope();
AnalysisLogger.getLogger().info("Using the following scope for the computation:"+scope); AnalysisLogger.getLogger().info("Using the following scope for the computation:"+scope);
String owner = config.getParam("ServiceUserName"); String owner = config.getGcubeUserName();
int leftNum = distributedModel.getNumberOfLeftElements(); int leftNum = distributedModel.getNumberOfLeftElements();
int rightNum = distributedModel.getNumberOfRightElements(); int rightNum = distributedModel.getNumberOfRightElements();
agent = new DistributedProcessingAgent(config, scope, owner, mainclass, config.getPersistencePath(), algorithm, defaultContainerFolder, maxMessagesAllowedPerJob, forceUpload, leftNum, rightNum,config.getTaskID()); if (config.getTaskID()==null || config.getTaskID().length()==0)
config.setTaskID(""+UUID.randomUUID());
agent = new DistributedProcessingAgentWPS(config, scope, owner, mainclass, config.getPersistencePath(), algorithm, defaultContainerFolder, maxMessagesAllowedPerJob, forceUpload, leftNum, rightNum,config.getTaskID());
agent.setLogger(AnalysisLogger.getLogger()); agent.setLogger(AnalysisLogger.getLogger());
} }
@ -143,7 +148,7 @@ public class D4ScienceDistributedProcessing implements Generator {
@Override @Override
public String getDescription() { public String getDescription() {
return "A D4Science Cloud Processor for Species Distributions"; return "A D4Science Cloud Processor";
} }
} }

149
src/main/java/org/gcube/dataanalysis/executor/generators/D4ScienceDistributedProcessingExecutor.java Normal file
View File

@ -0,0 +1,149 @@
package org.gcube.dataanalysis.executor.generators;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters;
import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode;
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm;
import org.gcube.dataanalysis.executor.job.management.DistributedProcessingAgent;
public class D4ScienceDistributedProcessingExecutor implements Generator {
public static int maxMessagesAllowedPerJob = 20;
public static boolean forceUpload = true;
public static String defaultContainerFolder = "PARALLEL_PROCESSING";
protected AlgorithmConfiguration config;
protected ActorNode distributedModel;
protected String mainclass;
DistributedProcessingAgent agent;
public D4ScienceDistributedProcessingExecutor(){
}
public D4ScienceDistributedProcessingExecutor(AlgorithmConfiguration config) {
this.config = config;
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
}
public void compute() throws Exception {
try {
agent.compute();
distributedModel.postProcess(agent.hasResentMessages(),false);
} catch (Exception e) {
distributedModel.postProcess(false,true);
AnalysisLogger.getLogger().error("ERROR: An Error occurred ", e);
throw e;
} finally {
shutdown();
}
}
@Override
public List<StatisticalType> getInputParameters() {
List<StatisticalType> distributionModelParams = new ArrayList<StatisticalType>();
distributionModelParams.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name"));
return distributionModelParams;
}
@Override
public String getResources() {
return agent.getResources();
}
@Override
public float getStatus() {
return agent.getStatus();
}
@Override
public StatisticalType getOutput() {
return distributedModel.getOutput();
}
@Override
public ALG_PROPS[] getSupportedAlgorithms() {
ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON};
return p;
}
@Override
public INFRASTRUCTURE getInfrastructure() {
return INFRASTRUCTURE.D4SCIENCE;
}
@Override
public void init() throws Exception {
Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.nodeAlgorithmsFile);
String model = config.getModel();
String algorithm = null;
if ((model!=null) && (model.length()>0))
algorithm = model;
else
algorithm=config.getAgent();
mainclass = p.getProperty(algorithm);
distributedModel = (ActorNode) Class.forName(mainclass).newInstance();
distributedModel.setup(config);
String scope = config.getGcubeScope();
AnalysisLogger.getLogger().info("Using the following scope for the computation:"+scope);
String owner = config.getParam("ServiceUserName");
int leftNum = distributedModel.getNumberOfLeftElements();
int rightNum = distributedModel.getNumberOfRightElements();
agent = new DistributedProcessingAgent(config, scope, owner, mainclass, config.getPersistencePath(), algorithm, defaultContainerFolder, maxMessagesAllowedPerJob, forceUpload, leftNum, rightNum,config.getTaskID());
agent.setLogger(AnalysisLogger.getLogger());
}
@Override
public void setConfiguration(AlgorithmConfiguration config) {
this.config = config;
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
}
@Override
public void shutdown() {
try {
agent.shutdown();
} catch (Exception e) {
}
try {
distributedModel.stop();
} catch (Exception e) {
}
}
@Override
public String getLoad() {
return agent.getLoad();
}
@Override
public String getResourceLoad() {
return agent.getResourceLoad();
}
@Override
public GenericAlgorithm getAlgorithm() {
return distributedModel;
}
@Override
public String getDescription() {
return "A D4Science Cloud Processor for Species Distributions";
}
}

229
src/main/java/org/gcube/dataanalysis/executor/job/management/DistributedProcessingAgentWPS.java Normal file
View File

@ -0,0 +1,229 @@
package org.gcube.dataanalysis.executor.job.management;
import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Logger;
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools;
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE;
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad;
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources;
import org.gcube.dataanalysis.ecoengine.utils.Operations;
import com.thoughtworks.xstream.XStream;
public class DistributedProcessingAgentWPS {
protected WPSJobManager jobManager;
protected boolean deletefiles = true;
protected String mainclass;
public int maxElementsAllowedPerJob = 20;
protected boolean forceUpload = true;
protected boolean stop;
protected String gscope;
protected String userName;
protected String pathToLib;
protected String modelName;
protected String containerFolder;
protected AlgorithmConfiguration configuration;
protected int rightSetNumberOfElements;
protected int leftSetNumberOfElements;
protected List<String> endpoints;
protected int subdivisiondiv;
protected String sessionID;
protected static String defaultJobOutput = "execution.output";
protected static String defaultScriptFile = "script";
protected Logger logger;
/**
* A distributed processing agent. Performs a distributed computation doing the MAP of the product of two sets: A and B
* Splits over B : A x B1 , A x B2, ... , A x Bn
* Prepares a script to be executed on remote nodes
* The computation is then sent to remote processors.
*/
public DistributedProcessingAgentWPS(AlgorithmConfiguration configuration,
String gCubeScope,
String computationOwner,
String mainClass,
String pathToLibFolder,
String modelName,
String containerFolder,
int maxElementsPerJob,
boolean forceReUploadofLibs,
int leftSetNumberOfElements,
int rightSetNumberOfElements,
String sessionID
) {
this.stop = false;
this.deletefiles = true;
this.gscope=gCubeScope;
this.mainclass=mainClass;
this.maxElementsAllowedPerJob=maxElementsPerJob;
this.forceUpload=forceReUploadofLibs;
this.configuration=configuration;
this.rightSetNumberOfElements=rightSetNumberOfElements;
this.leftSetNumberOfElements=leftSetNumberOfElements;
this.userName=computationOwner;
this.pathToLib=pathToLibFolder;
this.modelName=modelName;
this.containerFolder=containerFolder;
this.sessionID = sessionID;
}
public void setLogger(Logger logger){
this.logger=logger;
}
public boolean hasResentMessages(){
return jobManager.hasResentMessages();
}
public void compute() throws Exception {
try {
if (logger == null){
logger = AnalysisLogger.getLogger();
}
if (gscope == null)
throw new Exception("Null Scope");
AnalysisLogger.getLogger().debug("SCOPE: "+gscope);
jobManager = new WPSJobManager();
// we split along right dimension so if elements are less than nodes, we should reduce the number of nodes
// chunkize the number of species in order to lower the computational effort of the workers
subdivisiondiv = rightSetNumberOfElements / (maxElementsAllowedPerJob);
int rest = rightSetNumberOfElements % (maxElementsAllowedPerJob);
if (rest > 0)
subdivisiondiv++;
if (subdivisiondiv == 0)
subdivisiondiv = 1;
executeWork(leftSetNumberOfElements, rightSetNumberOfElements, 0, subdivisiondiv, deletefiles, forceUpload);
if (jobManager.wasAborted()) {
logger.debug("Warning: Job was aborted");
// distributionModel.postProcess(false,true);
throw new Exception("Job System Error");
}
else{
//postprocess
// distributionModel.postProcess(jobManager.hasResentMessages(),false);
}
} catch (Exception e) {
logger.error("ERROR: An Error occurred ", e);
e.printStackTrace();
throw e;
} finally {
shutdown();
}
}
private void executeWork(int leftNum, int rightNum, int offset, int numberOfResources, boolean deletefiles, boolean forceUpload) throws Exception {
int[] chunkSizes = Operations.takeChunks(rightNum, numberOfResources);
List<String> arguments = new ArrayList<String>();
// chunkize respect to the cells: take a chunk of cells vs all species at each node!
for (int i = 0; i < chunkSizes.length; i++) {
String argumentString = "0 " + leftNum + " " + offset + " " + chunkSizes[i];
arguments.add(argumentString);
offset += chunkSizes[i];
logger.debug("Generator-> Argument " + i + ": " + argumentString);
}
jobManager.uploadAndExecuteChunkized(configuration,mainclass,arguments,sessionID);
}
public String getResources() {
Resources res = new Resources();
try {
int activeNodes = jobManager.getActiveNodes();
for (int i = 0; i < activeNodes; i++) {
try {
res.addResource("Worker_" + (i + 1), 100);
} catch (Exception e1) {
}
}
} catch (Exception e) {
AnalysisLogger.getLogger().debug("D4ScienceGenerator->active nodes not ready");
}
if ((res != null) && (res.list != null))
return HttpRequest.toJSon(res.list).replace("resId", "resID");
else
return "";
}
public float getStatus() {
try {
if (stop)
return 100f;
else
if (jobManager!=null)
return Math.max(0.5f, jobManager.getStatus() * 100f);
else
return 0;
} catch (Exception e) {
return 0f;
}
}
public ALG_PROPS[] getSupportedAlgorithms() {
ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON};
return p;
}
public INFRASTRUCTURE getInfrastructure() {
return INFRASTRUCTURE.D4SCIENCE;
}
public void shutdown() {
try {
jobManager.stop();
} catch (Exception e) {
}
stop = true;
}
public String getLoad() {
long tk = System.currentTimeMillis();
ResourceLoad rs = null;
if (jobManager!=null)
rs = new ResourceLoad(tk, 1*subdivisiondiv);
else
rs = new ResourceLoad(tk, 0);
return rs.toString();
}
private long lastTime;
private int lastProcessed;
public String getResourceLoad() {
long thisTime = System.currentTimeMillis();
int processedRecords = 0;
if ((jobManager!=null) && (subdivisiondiv>0))
processedRecords = 1*subdivisiondiv;
int estimatedProcessedRecords = 0;
if (processedRecords == lastProcessed) {
estimatedProcessedRecords = Math.round(((float) thisTime * (float) lastProcessed) / (float) lastTime);
} else {
lastProcessed = processedRecords;
estimatedProcessedRecords = lastProcessed;
}
lastTime = thisTime;
ResourceLoad rs = new ResourceLoad(thisTime, estimatedProcessedRecords);
return rs.toString();
}
}

56
src/main/java/org/gcube/dataanalysis/executor/job/management/GenericWorker.java
View File

@ -9,6 +9,7 @@ import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.PrintStream; import java.io.PrintStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.util.UUID; import java.util.UUID;
@ -37,6 +38,7 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
public static String OutputParameter = "Process_Outcome"; public static String OutputParameter = "Process_Outcome";
public static String TASK_SUCCESS = "TASK_SUCCESS"; public static String TASK_SUCCESS = "TASK_SUCCESS";
public static String TASK_FAILURE = "TASK_FAILURE"; public static String TASK_FAILURE = "TASK_FAILURE";
public static String TASK_UNDEFINED = "TASK_UNDEFINED";
private static void inputStreamToFile(InputStream is, String path) throws FileNotFoundException, IOException { private static void inputStreamToFile(InputStream is, String path) throws FileNotFoundException, IOException {
FileOutputStream out = new FileOutputStream(new File(path)); FileOutputStream out = new FileOutputStream(new File(path));
@ -65,9 +67,9 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
File tempDir = null ; File tempDir = null ;
try { try {
Handler.activateProtocol(); Handler.activateProtocol();
String locDir = session;
if (session == null) if (session == null)
locDir = ("" + UUID.randomUUID()).replace("-", "");
// invoke the algorithm // invoke the algorithm
logger.debug("GenericWorker-> Creating algorithm " + algorithmClass); logger.debug("GenericWorker-> Creating algorithm " + algorithmClass);
@ -91,7 +93,7 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
System.setOut(ps); System.setOut(ps);
System.setErr(ps); System.setErr(ps);
node.executeNode(rightStartIndex, numberOfRightElementsToProcess, leftStartIndex, numberOfLeftElementsToProcess, isduplicate, node.executeNode(leftStartIndex, numberOfLeftElementsToProcess, rightStartIndex, numberOfRightElementsToProcess, isduplicate,
config.getConfigPath(), nodeConfigurationFileObject.getName(), "log.txt"); config.getConfigPath(), nodeConfigurationFileObject.getName(), "log.txt");
String log = new String(baos.toByteArray(), StandardCharsets.UTF_8); String log = new String(baos.toByteArray(), StandardCharsets.UTF_8);
@ -103,8 +105,14 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
logger.debug("GenericWorker-> deleted sandbox file: "+del ); logger.debug("GenericWorker-> deleted sandbox file: "+del );
logger.debug("GenericWorker-> all done"); logger.debug("GenericWorker-> all done");
if (log.contains("Exception:") && log.contains("Caused by:")){ if (log.contains("Exception:")){
outputParameters.put(OutputParameter, TASK_FAILURE); outputParameters.put(OutputParameter, TASK_FAILURE);
String cutLog = URLEncoder.encode(log, "UTF-8");
int maxlen = 20240;
if (log.length()>maxlen)
cutLog = cutLog.substring(0,maxlen)+"...";
outputParameters.put("Log", cutLog);
logger.debug("GenericWorker-> Failure!"); logger.debug("GenericWorker-> Failure!");
} }
else else
@ -112,6 +120,7 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
logger.debug("GenericWorker-> Success!"); logger.debug("GenericWorker-> Success!");
} catch (Throwable e) { } catch (Throwable e) {
outputParameters.put(OutputParameter, TASK_FAILURE); outputParameters.put(OutputParameter, TASK_FAILURE);
outputParameters.put("Log", e.getLocalizedMessage());
e.printStackTrace(); e.printStackTrace();
logger.debug("GenericWorker-> ERROR: " + e.getLocalizedMessage()); logger.debug("GenericWorker-> ERROR: " + e.getLocalizedMessage());
status = 100f; status = 100f;
@ -134,6 +143,9 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
logger.debug("GenericWorker-> deleting temporary directory"); logger.debug("GenericWorker-> deleting temporary directory");
tempDir.delete(); tempDir.delete();
} }
if (nodeConfigurationFileObject!=null && nodeConfigurationFileObject.exists())
nodeConfigurationFileObject.delete();
}catch(Exception e3){ }catch(Exception e3){
e3.printStackTrace(); e3.printStackTrace();
logger.debug("GenericWorker-> Error deleting files"); logger.debug("GenericWorker-> Error deleting files");
@ -157,16 +169,16 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
protected void process() throws Exception { protected void process() throws Exception {
AnalysisLogger.getLogger().debug("Parameters: "+config.getGeneralProperties()); AnalysisLogger.getLogger().debug("Parameters: "+config.getGeneralProperties());
String algorithmClass = config.getParam("AlgorithmClass"); String algorithmClass = config.getParam(AlgorithmClassParameter);
int rightStartIndex = Integer.parseInt(config.getParam("RightSetStartIndex")); int rightStartIndex = Integer.parseInt(config.getParam(RightSetStartIndexParameter));
int numberOfRightElementsToProcess =Integer.parseInt(config.getParam("NumberOfRightElementsToProcess")); int numberOfRightElementsToProcess =Integer.parseInt(config.getParam(NumberOfRightElementsToProcessParameter));
int leftStartIndex =Integer.parseInt(config.getParam("LeftSetStartIndex")); int leftStartIndex =Integer.parseInt(config.getParam(LeftSetStartIndexParameter));
int numberOfLeftElementsToProcess =Integer.parseInt(config.getParam("NumberOfLeftElementsToProcess")); int numberOfLeftElementsToProcess =Integer.parseInt(config.getParam(NumberOfLeftElementsToProcessParameter));
boolean isduplicate=Boolean.parseBoolean(config.getParam("IsDuplicate")); boolean isduplicate=Boolean.parseBoolean(config.getParam(IsDuplicateParameter));
String session=config.getParam("Session"); String session=config.getParam(SessionParameter);
File nodeConfigurationFileObject=new File (config.getParam("ConfigurationFile")); File nodeConfigurationFileObject=new File (config.getParam(ConfigurationFileParameter));
boolean deleteFiles= Boolean.parseBoolean(config.getParam("DeleteTemporaryFiles")); boolean deleteFiles= Boolean.parseBoolean(config.getParam(DeleteTemporaryFilesParameter));
AnalysisLogger.getLogger().debug("Executing the algorithm"); AnalysisLogger.getLogger().debug("Executing the algorithm");
executeAlgorithm(algorithmClass, rightStartIndex, numberOfRightElementsToProcess, leftStartIndex, numberOfLeftElementsToProcess, isduplicate, session, nodeConfigurationFileObject, deleteFiles); executeAlgorithm(algorithmClass, rightStartIndex, numberOfRightElementsToProcess, leftStartIndex, numberOfLeftElementsToProcess, isduplicate, session, nodeConfigurationFileObject, deleteFiles);
@ -177,15 +189,15 @@ public class GenericWorker extends StandardLocalExternalAlgorithm{
@Override @Override
protected void setInputParameters() { protected void setInputParameters() {
addStringInput("AlgorithmClass", "The full class path of the algorithm", "org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer"); addStringInput(AlgorithmClassParameter, "The full class path of the algorithm", "org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer");
addIntegerInput("RightSetStartIndex", "The start index of the right set in a cartesian product of the input", "1"); addIntegerInput(RightSetStartIndexParameter, "The start index of the right set in a cartesian product of the input", "1");
addIntegerInput("NumberOfRightElementsToProcess", "The number of elements to process in the right set", "1"); addIntegerInput(NumberOfRightElementsToProcessParameter, "The number of elements to process in the right set", "1");
addIntegerInput("LeftSetStartIndex", "The start index of the left set in a cartesian product of the input", "1"); addIntegerInput(LeftSetStartIndexParameter, "The start index of the left set in a cartesian product of the input", "1");
addIntegerInput("NumberOfLeftElementsToProcess", "The number of elements to process in the left set", "1"); addIntegerInput(NumberOfLeftElementsToProcessParameter, "The number of elements to process in the left set", "1");
addBooleanInput("IsDuplicate", "Indicate if this sub computation is a duplicate of another sub-computation", "false"); addBooleanInput(IsDuplicateParameter, "Indicate if this sub computation is a duplicate of another sub-computation", "false");
addStringInput("Session", "The session this sub-computation belongs to", "123456"); addStringInput(SessionParameter, "The session this sub-computation belongs to", "123456");
addFileInput("ConfigurationFile", "A configuration file for the algorithm in an XML serialisation format for the AlgorithmConfiguration Object", "config.dat"); addFileInput(ConfigurationFileParameter, "A configuration file for the algorithm in an XML serialisation format for the AlgorithmConfiguration Object", "config.dat");
addBooleanInput("DeleteTemporaryFiles","Delete local temporary files after the computation","true"); addBooleanInput(DeleteTemporaryFilesParameter,"Delete local temporary files after the computation","true");
} }
@Override @Override

46
src/main/java/org/gcube/dataanalysis/executor/job/management/GenericWorkerCaller.java Normal file
View File

@ -0,0 +1,46 @@
package org.gcube.dataanalysis.executor.job.management;
import java.io.File;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import com.thoughtworks.xstream.XStream;
public class GenericWorkerCaller {
public static String getGenericWorkerCall(String algorithm, String session, AlgorithmConfiguration configuration,int leftSetIndex,int rightSetIndex,int leftElements,int rightElements, boolean isduplicate,boolean deleteTemporaryFiles) throws Exception{
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
String xmlconfig = new XStream().toXML(configuration);
xmlconfig = xmlconfig.replace("\n", "").replace("\t", "");
xmlconfig = xmlconfig.replaceAll(">[ ]+<", "> <");
AnalysisLogger.getLogger().debug("CONFIG of Task:");
AnalysisLogger.getLogger().debug("algorithm: "+algorithm);
AnalysisLogger.getLogger().debug("leftSetIndex: "+leftSetIndex);
AnalysisLogger.getLogger().debug("leftElements: "+leftElements);
AnalysisLogger.getLogger().debug("rightSetIndex: "+rightSetIndex);
AnalysisLogger.getLogger().debug("rightElements: "+rightElements);
AnalysisLogger.getLogger().debug("session: "+session);
AnalysisLogger.getLogger().debug("isduplicate: "+isduplicate);
AnalysisLogger.getLogger().debug("deleteTemporaryFiles: "+deleteTemporaryFiles);
File is = new File(classLoader.getResource("WPSGWTemplate.xml").getFile());
String call=FileTools.loadString(is.getAbsolutePath(), "UTF-8");
AnalysisLogger.getLogger().debug("call template : "+call);
call = call.replace("#"+GenericWorker.AlgorithmClassParameter+"#", algorithm);
call = call.replace("#"+GenericWorker.LeftSetStartIndexParameter+"#", ""+leftSetIndex);
call = call.replace("#"+GenericWorker.NumberOfLeftElementsToProcessParameter+"#", ""+leftElements);
call = call.replace("#"+GenericWorker.RightSetStartIndexParameter+"#", ""+rightSetIndex);
call = call.replace("#"+GenericWorker.NumberOfRightElementsToProcessParameter+"#", ""+rightElements);
call = call.replace("#"+GenericWorker.SessionParameter+"#", session);
call = call.replace("#"+GenericWorker.IsDuplicateParameter+"#", ""+isduplicate);
call = call.replace("#"+GenericWorker.DeleteTemporaryFilesParameter+"#", ""+deleteTemporaryFiles);
call = call.replace("#"+GenericWorker.ConfigurationFileParameter+"#", ""+xmlconfig);
return call;
}
}

6
src/main/java/org/gcube/dataanalysis/executor/job/management/QueueJobManager.java
View File

@ -444,7 +444,7 @@ public class QueueJobManager {
// AnalysisLogger.getLogger().info("Contacting node " + (order + 1) + " OK on " + selectedEPR); // AnalysisLogger.getLogger().info("Contacting node " + (order + 1) + " OK on " + selectedEPR);
*/ */
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
ExecutorPlugin runExecutorPlugin = new ExecutorPlugin(); ExecutorPlugin runExecutorPlugin = new ExecutorPlugin();
SmartExecutorPluginQuery runQuery = new SmartExecutorPluginQuery(runExecutorPlugin); SmartExecutorPluginQuery runQuery = new SmartExecutorPluginQuery(runExecutorPlugin);
@ -458,7 +458,9 @@ public class QueueJobManager {
SpecificEndpointDiscoveryFilter sedf = new SpecificEndpointDiscoveryFilter(selectedEPR); SpecificEndpointDiscoveryFilter sedf = new SpecificEndpointDiscoveryFilter(selectedEPR);
runQuery.setEndpointDiscoveryFilter(sedf); runQuery.setEndpointDiscoveryFilter(sedf);
SmartExecutorProxy proxy = new ProxyBuilderImpl<SmartExecutor, SmartExecutorProxy>(runExecutorPlugin, runQuery).build(); SmartExecutorProxy proxy = new ProxyBuilderImpl<SmartExecutor, SmartExecutorProxy>(runExecutorPlugin, runQuery).build();
AnalysisLogger.getLogger().debug("Launching Smart Executor in namely Scope: "+scope+" real scope "+ScopeProvider.instance.get());
//AnalysisLogger.getLogger().debug("Launching Smart Executor in namely Scope: "+scope+" real scope "+ScopeProvider.instance.get());
AnalysisLogger.getLogger().debug("Launching Smart Executor in namely Scope: "+scope);
LaunchParameter launchParameter = new LaunchParameter(pluginName, inputs); LaunchParameter launchParameter = new LaunchParameter(pluginName, inputs);
String excecutionIdentifier = proxy.launch(launchParameter); String excecutionIdentifier = proxy.launch(launchParameter);
tasksProxies.add(new WorkerWatcher(proxy, excecutionIdentifier, AnalysisLogger.getLogger())); tasksProxies.add(new WorkerWatcher(proxy, excecutionIdentifier, AnalysisLogger.getLogger()));

311
src/main/java/org/gcube/dataanalysis/executor/job/management/RemoteJobManager.java
View File

@ -1,311 +0,0 @@
package org.gcube.dataanalysis.executor.job.management;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.xml.ws.EndpointReference;
import org.gcube.common.clients.ProxyBuilderImpl;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
import org.gcube.dataanalysis.executor.scripts.ScriptIOWorker;
import org.gcube.vremanagement.executor.api.SmartExecutor;
import org.gcube.vremanagement.executor.api.types.LaunchParameter;
import org.gcube.vremanagement.executor.client.plugins.ExecutorPlugin;
import org.gcube.vremanagement.executor.client.plugins.query.SmartExecutorPluginQuery;
import org.gcube.vremanagement.executor.client.plugins.query.filter.SpecificEndpointDiscoveryFilter;
import org.gcube.vremanagement.executor.client.proxies.SmartExecutorProxy;
import org.gcube.vremanagement.executor.plugin.PluginState;
public class RemoteJobManager {
// TODO Chek here:
//private static String pluginName = "ExecutorScript";
private static String pluginName = "SmartGenericWorker";
private int actualNumberOfNodes;
private List<String> eprs;
float status;
boolean abort;
boolean shutdown;
protected int activeNodes;
String scope;
public int getActiveNodes() {
return activeNodes;
}
public float getStatus() {
return status;
}
public int getNumberOfNodes() {
return actualNumberOfNodes;
}
public void setNumberOfNodes(int newNumberOfNodes) {
actualNumberOfNodes = newNumberOfNodes;
}
public void init(String scope, int numberOfNodes) throws Exception {
this.scope = scope;
AnalysisLogger.getLogger().debug("Using the following scope for this computation: "+ scope);
shutdown = false;
yetuploaded = false;
if (eprs == null)
actualNumberOfNodes = findNodes(scope);
else
actualNumberOfNodes = eprs.size();
if (numberOfNodes < actualNumberOfNodes)
actualNumberOfNodes = numberOfNodes;
}
public RemoteJobManager(String scope, int numberOfNodes) throws Exception {
init(scope, numberOfNodes);
}
public RemoteJobManager(String scope, int numberOfNodes, List<String> eprs) throws Exception {
this.eprs = eprs;
init(scope, numberOfNodes);
}
List<String> filenames;
List<String> fileurls;
boolean yetuploaded;
String session;
@SuppressWarnings("unchecked")
public boolean uploadAndExecute(String serviceClass, String serviceName, String owner, String localDir, String remoteDir, String outputDir, String script, List<String> arguments, boolean deletefiles) throws Exception {
boolean executeAll = false;
long t0 = System.currentTimeMillis();
//if not yet uploaded , upload required files
if (!yetuploaded) {
ScopeProvider.instance.set(scope);
IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED).getClient();
// IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED, gscope).getClient();
File dir = new File(localDir);
File[] files = dir.listFiles();
AnalysisLogger.getLogger().debug("Start uploading");
filenames = new ArrayList<String>();
fileurls = new ArrayList<String>();
for (File sfile : files) {
String localf = sfile.getAbsolutePath();
String filename = sfile.getName();
String remotef = remoteDir + sfile.getName();
client.put(true).LFile(localf).RFile(remotef);
String url = client.getUrl().RFile(remotef);
AnalysisLogger.getLogger().debug("URL created: " + url);
filenames.add(filename);
fileurls.add(url);
}
AnalysisLogger.getLogger().debug("Upload end");
yetuploaded = true;
session = (""+UUID.randomUUID()).replace("-", "");
}
//if the number of available nodes is higher than zero launch the tasks
if (actualNumberOfNodes > 0) {
AnalysisLogger.getLogger().debug("Executing script on " + actualNumberOfNodes + " nodes");
int len = arguments.size();
List<WorkerWatcher> tasksProxies = new ArrayList<WorkerWatcher>();
activeNodes = 0;
//launch the tasks
for (int i = 0; i < actualNumberOfNodes; i++) {
String argum = "";
//supply the arguments if they are available
if (i < len)
argum = arguments.get(i);
//generate the input map according to the arguments
Map<String, Object> inputs = generateInput(filenames, fileurls, outputDir, script, argum, i, scope, serviceClass, serviceName, owner, remoteDir,session,deletefiles);
AnalysisLogger.getLogger().debug("-> Owner: " + owner + " ServiceClass: " + serviceClass + " ServiceName:" + serviceName + " remoteDir:" + remoteDir);
//take the i-th endpoint of the executor
String selectedEPR = eprs.get(i);
AnalysisLogger.getLogger().debug("Launching node " + (i + 1) + " on " + selectedEPR);
//run the executor script
/*
ExecutorCall call = new ExecutorCall(pluginName, gscope);
call.setEndpointReference(selectedEPR);
TaskCall task = null;
task = call.launch(inputs);
TaskProxy proxy = task.getProxy();
*/
ExecutorPlugin runExecutorPlugin = new ExecutorPlugin();
SmartExecutorPluginQuery runQuery = new SmartExecutorPluginQuery(runExecutorPlugin);
/* TODO Add key_value filter here
* Tuple<String, String>[] tuples = new Tuple[n];
*
* runQuery.addConditions(pluginName, tuples);
*/
runQuery.addConditions(pluginName);
SpecificEndpointDiscoveryFilter sedf = new SpecificEndpointDiscoveryFilter(selectedEPR);
runQuery.setEndpointDiscoveryFilter(sedf);
SmartExecutorProxy proxy = new ProxyBuilderImpl<SmartExecutor, SmartExecutorProxy>(runExecutorPlugin, runQuery).build();
LaunchParameter launchParameter = new LaunchParameter(pluginName, inputs);
String excecutionIdentifier = proxy.launch(launchParameter);
tasksProxies.add(new WorkerWatcher(proxy, excecutionIdentifier, AnalysisLogger.getLogger()));
AnalysisLogger.getLogger().debug("Launching node " + (i + 1) + " OK on " + selectedEPR);
//add the task to the list in order to reuse it
}
activeNodes = actualNumberOfNodes;
AnalysisLogger.getLogger().debug("Launch Finished - Controlling Status");
int allstatus = 0;
abort = false;
//control the execution: go until there are active nodes or the process must stop
while ((activeNodes != 0) && (!abort) && (!shutdown)) {
//for each node get the task state
int nworkers = tasksProxies.size();
int i=0;
while (i < nworkers) {
WorkerWatcher proxy = tasksProxies.get(i);
/* ---- */
PluginState enumState = proxy.getState();
String state = enumState.toString();
/* ----- */
AnalysisLogger.getLogger().debug("REMOTE JOB MANAGER-> STATE " + state );
//control for aborted computation
abort = ((state == null) || state.equals("FAILED") || (!state.equals("DONE") && !state.equals("RUNNING")));
//control for finished computation
boolean finished = false;
if (state != null)
finished = state.equals("DONE");
//if finished update the active nodes
if (finished) {
tasksProxies.remove(i);
allstatus++;
activeNodes--;
nworkers--;
if (activeNodes == 0)
break;
}
else
i++;
status = Math.min(((float) allstatus / (float) actualNumberOfNodes) * 100f, 95f);
if (abort)
break;
if (shutdown)
break;
// AnalysisLogger.getLogger().debug(String.format("Task " + i + "executed started at %Tc with %s state ", proxy.getStartTime(), state));
//sleep before polling again
Thread.sleep(2000);
}
}
activeNodes = 0;
AnalysisLogger.getLogger().debug("All Tasks have Finished");
if (!abort) {
AnalysisLogger.getLogger().debug("All Task were successful");
/*
* List<StorageObject> listElements = client.showDir().RDir(remoteDir); for (StorageObject obj : listElements) { AnalysisLogger.getLogger().debug("obj stored in directory " + remoteDir + ": " + obj.getName()); }
*/
} else
AnalysisLogger.getLogger().debug("Tasks were NOT successful");
} else
AnalysisLogger.getLogger().debug("Warning: could not execute tasks: No Nodes Available!");
AnalysisLogger.getLogger().debug("Whole procedure done in " + (System.currentTimeMillis() - t0) + " ms");
status = 100f;
return executeAll;
}
public boolean wasAborted() {
return abort;
}
public void stop() {
shutdown = true;
}
@SuppressWarnings("unchecked")
private List<EndpointReference> getFilteredEndpoints(String scopeString){
ScopeProvider.instance.set(scopeString);
ExecutorPlugin executorPlugin = new ExecutorPlugin();
SmartExecutorPluginQuery query = new SmartExecutorPluginQuery(executorPlugin);
/*
Tuple<String, String>[] tuples = new Tuple[1];
tuples[0] = new Tuple<String, String>("Version", "1.0.0-SNAPSHOT");
query.addConditions("SmartGenericWorker", tuples);
*/
query.addConditions(pluginName);
/* Used to add extra filter to ServiceEndpoint discovery */
query.setServiceEndpointQueryFilter(null);
/* Used to add extra filter to GCore Endpoint discovery */
query.setEndpointDiscoveryFilter(null);
return query.fire();
}
private int findNodes(String scopeString) throws Exception {
return getFilteredEndpoints(scopeString).size();
}
/*
private int findNodes(String scopeString) throws Exception {
GCUBEScope scope = GCUBEScope.getScope(scopeString);
ISClient client = GHNContext.getImplementation(ISClient.class);
WSResourceQuery wsquery = client.getQuery(WSResourceQuery.class);
wsquery.addAtomicConditions(new AtomicCondition("//gc:ServiceName", "Executor"));
wsquery.addAtomicConditions(new AtomicCondition("/child::*[local-name()='Task']/name[text()='"+pluginName+"']", pluginName));
List<RPDocument> listdoc = client.execute(wsquery, scope);
EndpointReferenceType epr = null;
eprs = new ArrayList<EndpointReferenceType>();
int numberOfEP = 0;
for (RPDocument resource : listdoc) {
epr = resource.getEndpoint();
numberOfEP++;
eprs.add(epr);
}
AnalysisLogger.getLogger().debug("Found " + numberOfEP + " endpoints");
return numberOfEP;
}
*/
private Map<String, Object> generateInput(Object filenames, Object fileurls, String outputDir, String script, String argum, int i, String scope, String serviceClass, String serviceName, String owner, String remoteDir,String session,boolean deletefiles) {
Map<String, Object> inputs = new HashMap<String, Object>();
inputs.put("FILE_NAMES", filenames);
inputs.put("FILE_URLS", fileurls);
inputs.put("OUTPUTDIR", ScriptIOWorker.toInputString(outputDir));
inputs.put("SCRIPT", ScriptIOWorker.toInputString(script));
inputs.put("ARGUMENTS", ScriptIOWorker.toInputString(argum));
inputs.put("NODE_IDENTIFIER", "" + i);
inputs.put("SCOPE", ScriptIOWorker.toInputString(scope));
inputs.put("SERVICE_CLASS", ScriptIOWorker.toInputString(serviceClass));
inputs.put("SERVICE_NAME", ScriptIOWorker.toInputString(serviceName));
inputs.put("OWNER", ScriptIOWorker.toInputString(owner));
inputs.put("REMOTEDIR", ScriptIOWorker.toInputString(remoteDir));
inputs.put("CLEAN_CACHE",""+deletefiles);
// inputs.put("SESSION", ScriptIO.toInputString(session));
return inputs;
}
}

326
src/main/java/org/gcube/dataanalysis/executor/job/management/WPSJobManager.java Normal file
View File

@ -0,0 +1,326 @@
package org.gcube.dataanalysis.executor.job.management;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.jms.ExceptionListener;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageListener;
import org.apache.activemq.ActiveMQConnection;
import org.gcube.common.clients.ProxyBuilderImpl;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.blobstorage.resource.StorageObject;
import org.gcube.contentmanagement.blobstorage.service.IClient;
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanager.storageclient.wrapper.AccessType;
import org.gcube.contentmanager.storageclient.wrapper.MemoryType;
import org.gcube.contentmanager.storageclient.wrapper.StorageClient;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.utils.Operations;
import org.gcube.dataanalysis.executor.messagequeue.ATTRIBUTE;
import org.gcube.dataanalysis.executor.messagequeue.Consumer;
import org.gcube.dataanalysis.executor.messagequeue.Producer;
import org.gcube.dataanalysis.executor.messagequeue.QCONSTANTS;
import org.gcube.dataanalysis.executor.messagequeue.QueueManager;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
import org.gcube.dataanalysis.executor.scripts.ScriptIOWorker;
import org.gcube.dataanalysis.executor.util.InfraRetrieval;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.vremanagement.executor.api.SmartExecutor;
import org.gcube.vremanagement.executor.api.types.LaunchParameter;
import org.gcube.vremanagement.executor.client.plugins.ExecutorPlugin;
import org.gcube.vremanagement.executor.client.plugins.query.SmartExecutorPluginQuery;
import org.gcube.vremanagement.executor.client.plugins.query.filter.ListEndpointDiscoveryFilter;
import org.gcube.vremanagement.executor.client.plugins.query.filter.SpecificEndpointDiscoveryFilter;
import org.gcube.vremanagement.executor.client.proxies.SmartExecutorProxy;
import com.thoughtworks.xstream.XStream;
import static org.gcube.resources.discovery.icclient.ICFactory.*;
public class WPSJobManager {
static final int pollingTime = 5000;
static final int maxTrialsPerThread = 3;
int overallFailures = 0;
int overallSuccess = 0;
int overallTasks = 0;
boolean stopThreads = false;
boolean hasResentMessages = false;
final public synchronized void incrementOverallFailures() {
overallFailures++;
}
final public synchronized void hasResentTrue() {
if (!hasResentMessages)
hasResentMessages=true;
}
final public synchronized void incrementOverallSuccess() {
overallSuccess++;
}
final public synchronized void stop() {
stopThreads=true;
}
final public synchronized boolean isStopped() {
return stopThreads;
}
public class TasksWatcher implements Runnable {
AlgorithmConfiguration configuration;
String algorithm;
String username;
String token;
String wpsHost;
int wpsPort;
int taskNumber;
String session;
public String exitstatus=GenericWorker.TASK_UNDEFINED;
int leftSetIndex;
int rightSetIndex;
int leftElements;
int rightElements;
public TasksWatcher(String algorithm, String username, String token, String wpsHost, int wpsPort, String session, int taskNumber, AlgorithmConfiguration configuration, int leftSetIndex, int rightSetIndex, int leftElements, int rightElements) {
this.algorithm = algorithm;
this.token = token;
this.wpsHost = wpsHost;
this.wpsPort = wpsPort;
this.taskNumber = taskNumber;
this.session = session;
this.username = username;
this.configuration = configuration;
this.leftSetIndex = leftSetIndex;
this.leftElements = leftElements;
this.rightSetIndex = rightSetIndex;
this.rightElements = rightElements;
}
public void callTask(boolean isduplicate){
String url = "http://" + wpsHost + ":" + wpsPort + "/wps/WebProcessingService";
boolean deleteTemporaryFiles = true;
AnalysisLogger.getLogger().debug("Task Number : " + taskNumber+" GO!");
try {
String algorithmCall = GenericWorkerCaller.getGenericWorkerCall(algorithm, session, configuration, leftSetIndex, rightSetIndex, leftElements, rightElements, isduplicate, deleteTemporaryFiles);
String result = HttpRequest.PostXmlString(url, wpsHost, wpsPort, new LinkedHashMap<String, String>(), username, token, algorithmCall);
// AnalysisLogger.getLogger().debug("Result: " + result);
boolean success = false;
boolean failure = false;
if (result.contains(GenericWorker.TASK_SUCCESS))
success = true;
else if (result.contains(GenericWorker.TASK_FAILURE))
failure = true;
String statusLocation = "";
while (!success && !isStopped() && (!failure) ) { //while !success and failure
if (result == null || result.contains(GenericWorker.TASK_FAILURE) || result.contains("Exception"))
failure = true;
else if (result.contains(GenericWorker.TASK_SUCCESS))
success = true;
else if (result.contains("<wps:ProcessAccepted>Process Accepted</wps:ProcessAccepted>")) {
statusLocation = result.substring(result.indexOf("statusLocation=") + "statusLocation=".length());
statusLocation = statusLocation.substring(0, statusLocation.indexOf(">"));
statusLocation = statusLocation.replace("\"", "");
statusLocation = statusLocation + "&gcube-token=" + token;
// AnalysisLogger.getLogger().debug("Status Location: " + statusLocation);
result= "";
} else {
Thread.sleep(pollingTime);
result = HttpRequest.sendGetRequest(statusLocation, "");
// AnalysisLogger.getLogger().debug("Result in location: " + result);
}
// request = HttpRequest.sendGetRequest(url, ""); // AnalysisLogger.getLogger().debug("Answer for task "+taskNumber+": "+request); }catch(Exception e){ AnalysisLogger.getLogger().debug("Request failure for task "+taskNumber+": "+e.getLocalizedMessage()); } if (request.contains("<wps:ProcessSucceeded>")) success = true; if (request.contains("<ows:Exception>")){ failure = true; incrementOverallFailures(); } try { Thread.sleep(pollingTime); } catch (InterruptedException e) { e.printStackTrace(); } }
}
if (isStopped() && statusLocation!=null && statusLocation.length()>0){
String wpscancel = statusLocation.replace("RetrieveResultServlet", "CancelComputationServlet");
result = HttpRequest.sendGetRequest(wpscancel, "");
}
exitstatus = GenericWorker.TASK_SUCCESS;
if (failure)
{
exitstatus = GenericWorker.TASK_FAILURE;
AnalysisLogger.getLogger().debug("Task Number "+taskNumber+" - Failure cause: " + result);
}
// AnalysisLogger.getLogger().debug("Process execution finished: " + exitstatus);
} catch (Exception e) {
e.printStackTrace();
AnalysisLogger.getLogger().debug(e);
AnalysisLogger.getLogger().debug("Task Number "+taskNumber+" - Process exception: " + e.getLocalizedMessage());
exitstatus = GenericWorker.TASK_FAILURE;
}finally{
}
}
@Override
public void run() {
int trials = 0;
boolean duplicate = false;
while (!exitstatus.equals(GenericWorker.TASK_SUCCESS) && trials<maxTrialsPerThread){
callTask(duplicate);
if (exitstatus.equals(GenericWorker.TASK_FAILURE)){
trials++;
hasResentTrue();
duplicate = true;
AnalysisLogger.getLogger().debug("Task Number "+taskNumber+" - Retrying n."+trials);
}
}
if (exitstatus.equals(GenericWorker.TASK_SUCCESS))
incrementOverallSuccess();
else
incrementOverallFailures();
AnalysisLogger.getLogger().debug("Task Number "+taskNumber+" - Finished: " + exitstatus);
}
}
public int getNumberOfNodes() {
return 1;
}
public int getActiveNodes() {
return 1;
}
public float getStatus() {
return (float)(overallFailures+overallSuccess)/(float)overallTasks;
}
public boolean wasAborted() {
return stopThreads;
}
public boolean hasResentMessages() {
return hasResentMessages;
}
public void uploadAndExecuteChunkized(AlgorithmConfiguration configuration, String algorithmClass, List<String> arguments, String session) {
ExecutorService executor = null;
try{
int numberofservices = 1;
AnalysisLogger.getLogger().debug("Estimating the number of services");
List<String> wpsservices = InfraRetrieval.retrieveService("DataMiner", configuration.getGcubeScope());
if (wpsservices==null || wpsservices.size()==0)
throw new Exception ("No Dataminer GCore Endpoint found in the VRE "+configuration.getGcubeScope());
List<String> differentServices = new ArrayList<String>();
for (String service:wpsservices){
service = service.substring(service.indexOf("/")+2);
service = service.substring(0,service.indexOf(":"));
if (!differentServices.contains(service))
differentServices.add(service);
}
numberofservices = differentServices.size();
AnalysisLogger.getLogger().debug("WPSJobManager->Number of dataminer services "+numberofservices);
int parallelisation = numberofservices*2;
AnalysisLogger.getLogger().debug("WPSJobManager->Number of parallel processes (parallelisation) : "+parallelisation);
List<String> wpshosts = InfraRetrieval.retrieveAddresses("DataAnalysis",configuration.getGcubeScope(),"-----");
if (wpshosts==null || wpshosts.size()==0)
throw new Exception ("WPSJobManager->No Dataminer Service Endpoint found in the VRE "+configuration.getGcubeScope());
String wpshost = wpshosts.get(0);
wpshost = wpshost.substring(wpshost.indexOf("/")+2);
//String wpshostAddress = wpshost.substring(0,wpshost.indexOf(":"));
String wpshostAddress = wpshost.substring(0,wpshost.indexOf("/"));
//String wpshostPort = wpshost.substring(wpshost.indexOf(":")+1,wpshost.indexOf("/"));
//http://dataminer1-devnext.d4science.org:80/wps/gcube/resourc
wpshost=wpshostAddress;
int wpsport = 80;
overallTasks=arguments.size();
executor = Executors.newFixedThreadPool(parallelisation);
int taskNumber = 0;
AnalysisLogger.getLogger().debug("WPSJobManager->Executing algorithm class:"+algorithmClass);
for (String argument:arguments) {
String[] lfnlnr = argument.split(" ");
int leftOff = Integer.parseInt(lfnlnr[0]);
int leftNum = Integer.parseInt(lfnlnr[1]);
int rightOff = Integer.parseInt(lfnlnr[2]);
int rightNum = Integer.parseInt(lfnlnr[3]);
TasksWatcher watcher = new TasksWatcher(algorithmClass,
configuration.getGcubeUserName(),
configuration.getGcubeToken(),wpshost,wpsport,session,taskNumber,configuration, leftOff, rightOff,leftNum,rightNum);
executor.execute(watcher);
AnalysisLogger.getLogger().debug("WPSJobManager->Task number "+taskNumber+" launched!");
taskNumber++;
}
int njobs = overallFailures+overallSuccess;
int pnjobs =njobs;
while (njobs<overallTasks){
Thread.sleep(pollingTime);
float percFailure = (float)(overallFailures)/(float)overallTasks;
//if (percFailure>0.5)
if (overallFailures>0)
stop();
njobs = overallFailures+overallSuccess;
if (pnjobs<njobs){
AnalysisLogger.getLogger().debug("WPSJobManager->Number of finished jobs "+njobs+" of "+overallTasks);
AnalysisLogger.getLogger().debug("WPSJobManager->Number of errors "+overallFailures+" - perc failure "+percFailure);
}
}
AnalysisLogger.getLogger().debug("WPSJobManager->Overall computation finished");
}catch(Exception e){
e.printStackTrace();
}
finally{
if (executor!=null){
AnalysisLogger.getLogger().debug("WPSJobManager->Shutting down the executions");
executor.shutdown();
}
}
}
}

2
src/main/java/org/gcube/dataanalysis/executor/rscripts/generic/GenericRScript.java
View File

@ -70,6 +70,8 @@ public abstract class GenericRScript extends StandardLocalExternalAlgorithm {
List<StatisticalType> inputs = getInputParameters(); List<StatisticalType> inputs = getInputParameters();
for (String input : inputvariables) { for (String input : inputvariables) {
String value = config.getParam(input); String value = config.getParam(input);
if (value == null)
value = "";
String defaultValue = inputs.get(i).getDefaultValue(); String defaultValue = inputs.get(i).getDefaultValue();
defaultValue = defaultValue.replace("(", "\\(").replace(")", "\\)").replace("[", "\\[").replace("]", "\\]").replace("|", "\\|").replace(".", "\\.").replace("?", "\\?").replace("*", "\\*").replace("+", "\\+").replace("{", "\\{").replace("}", "\\}"); defaultValue = defaultValue.replace("(", "\\(").replace(")", "\\)").replace("[", "\\[").replace("]", "\\]").replace("|", "\\|").replace(".", "\\.").replace("?", "\\?").replace("*", "\\*").replace("+", "\\+").replace("{", "\\{").replace("}", "\\}");
// inputParameters.put(defaultValue, value); // inputParameters.put(defaultValue, value);

22
src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym.java
View File

@ -1,5 +1,8 @@
package org.gcube.dataanalysis.executor.tests; package org.gcube.dataanalysis.executor.tests;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer;
@ -11,7 +14,7 @@ public class RegressionTestBiOnym {
// Generate // Generate
AlgorithmConfiguration config = new AlgorithmConfiguration(); AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/"); config.setConfigPath("./cfg/");
config.setParam("DatabaseUserName","utente"); config.setParam("DatabaseUserName","utente");
config.setParam("DatabasePassword","d4science"); config.setParam("DatabasePassword","d4science");
config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
@ -43,14 +46,15 @@ public class RegressionTestBiOnym {
config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "taxamatchoutputlocal"); config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "taxamatchoutputlocal");
config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "taxamatchoutputlabel"); config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "taxamatchoutputlabel");
//1000 //1000
// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput1000"); config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput1000");
// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings"); config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings");
// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput"); // config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput");
// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings"); // config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings");
//4 //4
config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id1ecb405c_980f_47a4_926a_3043d065fc7d"); //config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id1ecb405c_980f_47a4_926a_3043d065fc7d");
config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); //config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0");
//2 //2
// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id471e6d50_d243_4112_bc07_e22152438e5c"); // config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id471e6d50_d243_4112_bc07_e22152438e5c");
// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); // config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0");
@ -62,7 +66,13 @@ public class RegressionTestBiOnym {
config.setAgent("BIONYM"); config.setAgent("BIONYM");
config.setPersistencePath("./"); config.setPersistencePath("./");
config.setGcubeScope( "/gcube"); String scope = "/gcube/devNext/NextNext";
config.setGcubeScope( scope );
ScopeProvider.instance.set(scope);
String authorizationToken = "cb289202-e7d6-45ee-8076-a80bc4d4be51-98187548";
SecurityTokenProvider.instance.set(authorizationToken);
config.setGcubeToken(authorizationToken);
config.setGcubeUserName("gianpaolo.coro");
// config.setGcubeScope( "/d4science.research-infrastructures.eu"); // config.setGcubeScope( "/d4science.research-infrastructures.eu");
config.setParam("ServiceUserName", "gianpaolo.coro"); config.setParam("ServiceUserName", "gianpaolo.coro");

29
src/main/java/org/gcube/dataanalysis/executor/tests/TestRemoteJobLaunch.java
View File

@ -1,29 +0,0 @@
package org.gcube.dataanalysis.executor.tests;
import java.util.ArrayList;
import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.executor.job.management.RemoteJobManager;
public class TestRemoteJobLaunch {
public static void main(String [] args) throws Exception{
String scope = "/gcube";
String serviceClass = "TestGP";
String serviceName = "TestGPHome";
String owner = "GP";
String directory = "./shipping/";
String remotedirectory = "/shipping/";
String tempDir = "./";
String scriptName = "execute.sh";
int numberOfNodes = 1;
List<String> argums = new ArrayList<String>();
argums.add("0_178204_0_3_./");
AnalysisLogger.setLogger("./cfg/ALog.properties");
RemoteJobManager job = new RemoteJobManager(scope,numberOfNodes);
// job.uploadAndExecute(serviceClass, serviceName, owner, directory, remotedirectory, tempDir, scriptName, argums);
}
}

105
src/main/java/org/gcube/dataanalysis/executor/tests/TestWPSJobs.java Normal file
View File

@ -0,0 +1,105 @@
package org.gcube.dataanalysis.executor.tests;
import java.util.ArrayList;
import java.util.List;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.executor.job.management.GenericWorker;
import org.gcube.dataanalysis.executor.job.management.WPSJobManager;
import org.gcube.dataanalysis.executor.job.management.WPSJobManager.TasksWatcher;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer;
import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters;
public class TestWPSJobs {
public static AlgorithmConfiguration buildTestConfiguration(){
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam(YasmeenGlobalParameters.parserNameParam, YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name());
config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam, YasmeenGlobalParameters.BuiltinDataSources.WORMS_PISCES.name());
config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing, "true");
config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies, "false");
config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 1, YasmeenGlobalParameters.BuiltinMatchers.GSAy.name());
config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 1, "0.6");
config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 1, "10");
config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 2, YasmeenGlobalParameters.BuiltinMatchers.FUZZYMATCH.name());
config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 2, "0.6");
config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 2, "10");
config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 3, YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name());
config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 3, "0.4");
config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 3, "10");
config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 4, YasmeenGlobalParameters.BuiltinMatchers.TRIGRAM.name());
config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 4, "0.4");
config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 4, "10");
config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "taxamatchoutputlocal");
config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "taxamatchoutputlabel");
// 4
//config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id1ecb405c_980f_47a4_926a_3043d065fc7d");
//config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0");
config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput1000");
config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings");
config.setAgent("BIONYM");
config.setPersistencePath("./");
config.setGcubeScope("/gcube/devNext/NextNext");
// config.setGcubeScope("/gcube/devsec/devVRE");
config.setParam("ServiceUserName", "gianpaolo.coro");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
config.setGcubeUserName("gianpaolo.coro");
config.setGcubeToken("cb289202-e7d6-45ee-8076-a80bc4d4be51-98187548");
return config;
}
public static void main1(String[] args) throws Exception {
String host = "dataminer1-devnext.d4science.org";
String session = "12345";
int port = 80;
String algorithm = "org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer";
AlgorithmConfiguration config = buildTestConfiguration();
WPSJobManager manager = new WPSJobManager();
TasksWatcher taskWatcher = manager.new TasksWatcher(algorithm, config.getGcubeUserName(), config.getGcubeToken(), host, port, session, 1, config, 1, 1, 1, 1);
Thread t = new Thread(taskWatcher);
t.start();
while (taskWatcher.exitstatus.equals(GenericWorker.TASK_UNDEFINED)){
Thread.sleep(1000);
System.out.print(".");
}
AnalysisLogger.getLogger().debug("Task 1 terminated with output "+taskWatcher.exitstatus );
//taskWatcher.run();
}
public static void main(String[] args) throws Exception {
AlgorithmConfiguration config = buildTestConfiguration();
String algorithm = "org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer";
ScopeProvider.instance.set(config.getGcubeScope());
WPSJobManager jobmanager = new WPSJobManager();
int nArguments = 100;
List<String> arguments = new ArrayList<String>();
for (int i=1;i<=nArguments;i++){
String argument = "1 1 "+i+" 1";
arguments.add(argument);
}
String sessionID ="1234";
jobmanager.uploadAndExecuteChunkized(config, algorithm, arguments,sessionID);
}
}

6
src/main/java/org/gcube/dataanalysis/executor/util/DataTransferer.java
View File

@ -29,7 +29,7 @@ public class DataTransferer {
//String scope = "/d4science.research-infrastructures.eu/gCubeApps"; //String scope = "/d4science.research-infrastructures.eu/gCubeApps";
String scope = "/gcube/devsec/devVRE"; String scope = "/gcube/devsec/devVRE";
ScopeProvider.instance.set(scope); //test only ScopeProvider.instance.set(scope);
//String transferGHN = "dewn04.madgik.di.uoa.gr"; //String transferGHN = "dewn04.madgik.di.uoa.gr";
String transferGHN = "access.d4science.org"; String transferGHN = "access.d4science.org";
int transferPort = 8080; int transferPort = 8080;
@ -71,7 +71,7 @@ public class DataTransferer {
// returns the number of transferred bytes // returns the number of transferred bytes
public static boolean transferFileToService(String scope, String username, String service, int port, String fileAbsolutePath, String remoteFolder) throws Exception { public static boolean transferFileToService(String scope, String username, String service, int port, String fileAbsolutePath, String remoteFolder) throws Exception {
AnalysisLogger.getLogger().debug("Transferring file " + fileAbsolutePath + " to " + service + ":" + port); AnalysisLogger.getLogger().debug("Transferring file " + fileAbsolutePath + " to " + service + ":" + port);
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
AgentLibrary library = transferAgent().at(service, port).build(); AgentLibrary library = transferAgent().at(service, port).build();
ArrayList<URI> input = new ArrayList<URI>(); ArrayList<URI> input = new ArrayList<URI>();
@ -82,7 +82,7 @@ public class DataTransferer {
String localfolder = localFile.getParent(); String localfolder = localFile.getParent();
String file = localFile.getName(); String file = localFile.getName();
AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage"); AnalysisLogger.getLogger().debug("Uploading file " + file + " onto storage");
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
AnalysisLogger.getLogger().info("Loading file on scope: " + scope); AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true); String storagesmpurl = StorageUtils.uploadFilesOnStorage(scope, username, localfolder, "/",file,true);

6
src/main/java/org/gcube/dataanalysis/executor/util/InfraRetrieval.java
View File

@ -17,7 +17,7 @@ public class InfraRetrieval {
return new ArrayList<String>(); return new ArrayList<String>();
// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Setting Scope to " + scope+" and executing query"); // AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Setting Scope to " + scope+" and executing query");
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class); SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()"); query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
@ -31,7 +31,7 @@ public class InfraRetrieval {
if (scope == null || scope.length() == 0) if (scope == null || scope.length() == 0)
return new ArrayList<String>(); return new ArrayList<String>();
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class); SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile/Name/text() eq '" + Name+ "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()"); query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile/Name/text() eq '" + Name+ "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()");
@ -45,7 +45,7 @@ public class InfraRetrieval {
if (scope == null || scope.length() == 0) if (scope == null || scope.length() == 0)
return new ArrayList<String>(); return new ArrayList<String>();
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class); SimpleQuery query = ICFactory.queryFor(GCoreEndpoint.class);
query.addCondition("$resource/Profile/ServiceName/text() eq '"+service+"'").setResult("$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint/text()"); query.addCondition("$resource/Profile/ServiceName/text() eq '"+service+"'").setResult("$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint/text()");

2
src/main/java/org/gcube/dataanalysis/executor/util/RScriptsManager.java
View File

@ -13,6 +13,7 @@ import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
@ -114,6 +115,7 @@ public class RScriptsManager {
String scope = config.getGcubeScope(); String scope = config.getGcubeScope();
if (scope == null) if (scope == null)
scope = ScopeProvider.instance.get(); scope = ScopeProvider.instance.get();
//SecurityTokenProvider.instance.set(authorizationToken);
AnalysisLogger.getLogger().debug("Current User: " + owner); AnalysisLogger.getLogger().debug("Current User: " + owner);
AnalysisLogger.getLogger().debug("Current Scope: " + scope); AnalysisLogger.getLogger().debug("Current Scope: " + scope);

6
src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java
View File

@ -55,7 +55,7 @@ public class StorageUtils {
public static String uploadFilesOnStorage(String scope, String user, String localFolder, String remoteFolder, String file, boolean httplink) throws Exception { public static String uploadFilesOnStorage(String scope, String user, String localFolder, String remoteFolder, String file, boolean httplink) throws Exception {
try { try {
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
AnalysisLogger.getLogger().info("Loading file on scope: " + scope); AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient(); IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient();
String remotef = remoteFolder+file.replace(" ","%20"); String remotef = remoteFolder+file.replace(" ","%20");
@ -77,7 +77,7 @@ public class StorageUtils {
public static String uploadFilesOnStorage(String scope, String user, String localFolder, String file) throws Exception { public static String uploadFilesOnStorage(String scope, String user, String localFolder, String file) throws Exception {
try { try {
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
AnalysisLogger.getLogger().info("Loading file on scope: " + scope); AnalysisLogger.getLogger().info("Loading file on scope: " + scope);
IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient(); IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient();
String remotef = "/"+file; String remotef = "/"+file;
@ -142,7 +142,7 @@ public class StorageUtils {
public static void downloadFilefromStorage(String scope, String user, String localFolder, String file) throws Exception { public static void downloadFilefromStorage(String scope, String user, String localFolder, String file) throws Exception {
try { try {
ScopeProvider.instance.set(scope); // ScopeProvider.instance.set(scope);
AnalysisLogger.getLogger().info("Retrieving file on scope: " + scope); AnalysisLogger.getLogger().info("Retrieving file on scope: " + scope);
IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient(); IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient();
String remotef = "/"+file; String remotef = "/"+file;