gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@153273 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2017-09-19 15:19:27 +00:00
parent 3c04ac2495
commit 9fa8280374
6 changed files with 70 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/codegeneration/ClassGenerator.java
View File

@ -49,15 +49,16 @@ public class ClassGenerator {
// build class preamble
config.setAgent(algorithm);
config.setModel(algorithm);
config.setAlgorithmClassLoader(Thread.currentThread().getContextClassLoader());
String packageString = "";
String interfaceString = "";
try{
if (algorithmSet.equals("DISTRIBUTIONS")) {
packageString = "generators";
interfaceString = "IGenerator";
inputs = GeneratorsFactory.getAlgorithmParameters(configPath, algorithm);
description = GeneratorsFactory.getDescription(configPath, algorithm);
outputs = GeneratorsFactory.getAlgorithmOutput(configPath, algorithm);
inputs = GeneratorsFactory.getAlgorithmParameters(configPath, algorithm, config);
description = GeneratorsFactory.getDescription(configPath, algorithm,config);
outputs = GeneratorsFactory.getAlgorithmOutput(configPath, algorithm,config);
} else if (algorithmSet.equals("TRANSDUCERS")) {
packageString = "transducerers";
interfaceString = "ITransducer";
@ -67,23 +68,23 @@ public class ClassGenerator {
} else if (algorithmSet.equals("MODELS")) {
packageString = "modellers";
interfaceString = "IModeller";
inputs = ModelersFactory.getModelParameters(configPath, algorithm);
description = ModelersFactory.getDescription(configPath, algorithm);
outputs = ModelersFactory.getModelOutput(configPath, algorithm);
inputs = ModelersFactory.getModelParameters(configPath, algorithm,config);
description = ModelersFactory.getDescription(configPath, algorithm,config);
outputs = ModelersFactory.getModelOutput(configPath, algorithm,config);
} else if (algorithmSet.equals("CLUSTERERS")) {
packageString = "clusterers";
interfaceString = "IClusterer";
inputs = ClusterersFactory.getClustererParameters(configPath, algorithm);
description = ClusterersFactory.getDescription(configPath, algorithm);
outputs = ClusterersFactory.getClustererOutput(configPath, algorithm);
inputs = ClusterersFactory.getClustererParameters(configPath, algorithm,config);
description = ClusterersFactory.getDescription(configPath, algorithm,config);
outputs = ClusterersFactory.getClustererOutput(configPath, algorithm,config);
} else if (algorithmSet.equals("TEMPORAL_ANALYSIS")) {
} else if (algorithmSet.equals("EVALUATORS")) {
packageString = "evaluators";
interfaceString = "IEvaluator";
inputs = EvaluatorsFactory.getEvaluatorParameters(configPath, algorithm);
description = EvaluatorsFactory.getDescription(configPath, algorithm);
outputs = EvaluatorsFactory.getEvaluatorOutput(configPath, algorithm);
inputs = EvaluatorsFactory.getEvaluatorParameters(configPath, algorithm,config);
description = EvaluatorsFactory.getDescription(configPath, algorithm,config);
outputs = EvaluatorsFactory.getEvaluatorOutput(configPath, algorithm,config);
}
}catch(Exception e){
LOGGER.error("Error in retrieving output: ",e);

20
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java
View File

@ -1,9 +1,9 @@
package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.ArrayList;
import java.util.List;
import org.gcube.common.encryption.StringEncrypter;
@ -28,6 +28,7 @@ public class InfrastructureDialoguer {
public DatabaseInfo getDatabaseInfo(String resourceName) throws Exception{
DatabaseInfo dbi = new DatabaseInfo();
LOGGER.debug("Searching for Database "+resourceName+" in scope "+scope);
SimpleQuery query = queryFor(ServiceEndpoint.class);
// query.addCondition("$resource/Profile/Category/text() eq 'Database' and $resource/Profile/Name eq 'StatisticalManagerDataBase' ");
@ -61,20 +62,19 @@ public class InfrastructureDialoguer {
public List<String> getAlgorithmsInScope() throws Exception{
LOGGER.debug("Searching for Algorithms in scope {} with classloader type {}",scope,Thread.currentThread().getContextClassLoader());
LOGGER.debug("Searching for Algorithms in scope {} with classloader type {}",scope,Thread.currentThread().getContextClassLoader().getClass().getSimpleName());
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType eq 'StatisticalManagerAlgorithm' ");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> resources = client.submit(query);
query.addCondition("$resource/Profile/SecondaryType/string() eq 'StatisticalManagerAlgorithm' ");
query.setResult("$resource/Profile/Name/text()");
DiscoveryClient<String> client = client();
List<String> resources = client.submit(query);
if (resources==null || resources.size()==0){
throw new Exception("No resource named StatisticalManagerAlgorithm available in scope "+scope);
}
List<String> resourcesNames = new ArrayList<String>();
LOGGER.debug("Found {} algorithms",resources.size());
for (GenericResource resource: resources){
resourcesNames.add(resource.profile().name());
}
return resourcesNames;
return resources;
}

70
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java
View File

@ -4,7 +4,6 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
@ -51,13 +50,15 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
/**
* Deploying procedure: 1 - modify configuration files 2 - modify resource file: resources/templates/setup.cfg 3 - generate classes with ClassGenerator 4 - add new classes in the wps_config.xml on the wps web app config folder 5 - produce the Jar file of this project 6 - copy the jar file in the lib folder of the wps web app change the server parameters in the wps_config.xml file
*/
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEcologicalEngineMapper.class);
private Observer observer = null;
private boolean cancelled = false;
private TokenManager tokenm = null;
// inputs and outputs
public LinkedHashMap<String, Object> inputs = new LinkedHashMap<String, Object>();
public LinkedHashMap<String, Object> outputs = new LinkedHashMap<String, Object>();
@ -141,13 +142,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
if (this instanceof ITransducer)
parameters = TransducerersFactory.getTransducerParameters(config, algorithmName);
else if (this instanceof IClusterer)
parameters = ClusterersFactory.getClustererParameters(config.getConfigPath(), algorithmName);
parameters = ClusterersFactory.getClustererParameters(config.getConfigPath(), algorithmName, config);
else if (this instanceof IEvaluator)
parameters = EvaluatorsFactory.getEvaluatorParameters(config.getConfigPath(), algorithmName);
parameters = EvaluatorsFactory.getEvaluatorParameters(config.getConfigPath(), algorithmName, config);
else if (this instanceof IGenerator)
parameters = GeneratorsFactory.getAlgorithmParameters(config.getConfigPath(), algorithmName);
parameters = GeneratorsFactory.getAlgorithmParameters(config.getConfigPath(), algorithmName, config);
else if (this instanceof IModeller)
parameters = ModelersFactory.getModelParameters(config.getConfigPath(), algorithmName);
parameters = ModelersFactory.getModelParameters(config.getConfigPath(), algorithmName, config);
if (parameters != null) {
LOGGER.debug("Found " + parameters.size() + " Parameters for " + algorithmName);
@ -163,13 +164,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
if (this instanceof ITransducer)
output = TransducerersFactory.getTransducerOutput(config, algorithmName);
else if (this instanceof IClusterer)
output = ClusterersFactory.getClustererOutput(config.getConfigPath(), algorithmName);
output = ClusterersFactory.getClustererOutput(config.getConfigPath(), algorithmName, config);
else if (this instanceof IEvaluator)
output = EvaluatorsFactory.getEvaluatorOutput(config.getConfigPath(), algorithmName);
output = EvaluatorsFactory.getEvaluatorOutput(config.getConfigPath(), algorithmName, config);
else if (this instanceof IGenerator)
output = GeneratorsFactory.getAlgorithmOutput(config.getConfigPath(), algorithmName);
output = GeneratorsFactory.getAlgorithmOutput(config.getConfigPath(), algorithmName, config);
else if (this instanceof IModeller)
output = ModelersFactory.getModelOutput(config.getConfigPath(), algorithmName);
output = ModelersFactory.getModelOutput(config.getConfigPath(), algorithmName, config);
if (output != null) {
LOGGER.debug("Found " + output + " for " + algorithmName);
@ -229,7 +230,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
String token = null;
// DONE get scope and username from SmartGears
// get scope from SmartGears
TokenManager tokenm = new TokenManager();
tokenm = new TokenManager();
tokenm.getCredentials();
scope = tokenm.getScope();
username = tokenm.getUserName();
@ -246,7 +247,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
if (agent != null) {
if (status != previousStatus) {
LOGGER.debug("STATUS update to: {} ", status );
previousStatus = status;
previousStatus = status;
super.update(new Integer((int) status));
try {
updateComputationOnWS(status, null);
@ -301,11 +302,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
public void run() throws Exception {
if (observer!=null)
observer.isStarted(this);
LOGGER.info("classloader context in this thread is {}",Thread.currentThread().getContextClassLoader());
long startTimeLong = System.currentTimeMillis();
OperationResult operationResult = null;
String algorithm = "";
List<String> generatedInputTables = null;
List<String> generatedOutputTables = null;
@ -327,7 +330,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
time("WPS Algorithm objects Initialization: Session " + computationSession);
// set the configuration environment for this algorithm
configManager.configAlgorithmEnvironment(inputs);
configManager.setComputationId(computationSession);
config = configManager.getConfig();
@ -458,6 +461,8 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
else
throw e;
} finally {
LOGGER.debug("accounting algorithm");
accountAlgorithmExecution(startTimeLong, System.currentTimeMillis(), operationResult);
LOGGER.debug("Deleting Input Tables");
deleteTemporaryTables(generatedInputTables);
LOGGER.debug("Deleting Output Tables");
@ -472,28 +477,27 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
cleanResources();
if (observer!=null) observer.isFinished(this);
LOGGER.debug("All done - Computation Finished");
accountAlgorithmExecution(configManager, startTimeLong, System.currentTimeMillis(), operationResult);
}
}
private void accountAlgorithmExecution(ConfigurationManager confManager, long start, long end, OperationResult result) {
private void accountAlgorithmExecution(long start, long end, OperationResult result) {
try{
JobUsageRecord jobUsageRecord = new JobUsageRecord();
jobUsageRecord.setJobName(this.getAlgorithmClass().getSimpleName());
jobUsageRecord.setConsumerId(confManager.getUsername());
Calendar startCal = Calendar.getInstance();
startCal.setTimeInMillis(start);
jobUsageRecord.setDuration(end-start);
jobUsageRecord.setOperationResult(result);
jobUsageRecord.setServiceName("DataMiner");
jobUsageRecord.setServiceClass("WPS");
jobUsageRecord.setHost(WPSConfig.getInstance().getWPSConfig().getServer().getHostname());
AccountingPersistence accountingPersistence =
AccountingPersistenceFactory.getPersistence();
accountingPersistence.account(jobUsageRecord);
}catch(Exception e){
JobUsageRecord jobUsageRecord = new JobUsageRecord();
jobUsageRecord.setJobName(this.getAlgorithmClass().getSimpleName());
jobUsageRecord.setConsumerId(tokenm.getUserName());
jobUsageRecord.setDuration(end-start);
jobUsageRecord.setOperationResult(result);
jobUsageRecord.setServiceName("DataMiner");
jobUsageRecord.setServiceClass("WPS");
jobUsageRecord.setHost(WPSConfig.getInstance().getWPSConfig().getServer().getHostname());
jobUsageRecord.setCallerQualifier(tokenm.getTokenQualifier());
AccountingPersistence accountingPersistence =
AccountingPersistenceFactory.getPersistence();
accountingPersistence.account(jobUsageRecord);
}catch(Throwable e){
LOGGER.error("error accounting algorithm execution",e);
}

1
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java
View File

@ -85,6 +85,7 @@ public class ConfigurationManager {
public void configAlgorithmEnvironment(LinkedHashMap<String, Object> inputs) throws Exception {
// set config container
config = new AlgorithmConfiguration();
config.setAlgorithmClassLoader(Thread.currentThread().getContextClassLoader());
String webperspath = WPSConfig.getConfigDir() + "../persistence/";
// selecting persistence path
// String persistencePath = File.createTempFile("wpsstatcheck", ".sm").getParent() + "/../cfg/";

8
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java
View File

@ -15,7 +15,8 @@ public class TokenManager {
String username;
String scope;
String token;
String tokenQualifier;
public String getScope(){
return scope;
}
@ -27,6 +28,10 @@ public class TokenManager {
public String getToken(){
return token;
}
public String getTokenQualifier() {
return tokenQualifier;
}
public void getCredentials() {
try{
@ -36,6 +41,7 @@ public class TokenManager {
token = SecurityTokenProvider.instance.get();
AuthorizationEntry entry = authorizationService().get(token);
scope = entry.getContext();
tokenQualifier = entry.getQualifier();
}catch(Exception e){
LOGGER.error("Error Retrieving token credentials ",e);

4
src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java
View File

@ -18,8 +18,8 @@ public class AlgorithmTest {
@Test
public void executeAlgorithmsFromFile() throws Exception{
String protocol = "http";
String hostname = "dataminer1-d-d4s.d4science.org";
String token = "595ca591-9921-423c-bfca-f8be19f05882-98187548";
String hostname = "dataminer1-pre.d4science.org";
String token = "a5b623b6-6577-4271-aba6-7ada687d29cf-98187548";
Iterator<String> uris = getUrisIterator();