gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@153273 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2017-09-19 15:19:27 +00:00
parent 3c04ac2495
commit 9fa8280374
6 changed files with 70 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/codegeneration/ClassGenerator.java
View File

@ -49,15 +49,16 @@ public class ClassGenerator {
// build class preamble // build class preamble
config.setAgent(algorithm); config.setAgent(algorithm);
config.setModel(algorithm); config.setModel(algorithm);
config.setAlgorithmClassLoader(Thread.currentThread().getContextClassLoader());
String packageString = ""; String packageString = "";
String interfaceString = ""; String interfaceString = "";
try{ try{
if (algorithmSet.equals("DISTRIBUTIONS")) { if (algorithmSet.equals("DISTRIBUTIONS")) {
packageString = "generators"; packageString = "generators";
interfaceString = "IGenerator"; interfaceString = "IGenerator";
inputs = GeneratorsFactory.getAlgorithmParameters(configPath, algorithm); inputs = GeneratorsFactory.getAlgorithmParameters(configPath, algorithm, config);
description = GeneratorsFactory.getDescription(configPath, algorithm); description = GeneratorsFactory.getDescription(configPath, algorithm,config);
outputs = GeneratorsFactory.getAlgorithmOutput(configPath, algorithm); outputs = GeneratorsFactory.getAlgorithmOutput(configPath, algorithm,config);
} else if (algorithmSet.equals("TRANSDUCERS")) { } else if (algorithmSet.equals("TRANSDUCERS")) {
packageString = "transducerers"; packageString = "transducerers";
interfaceString = "ITransducer"; interfaceString = "ITransducer";
@ -67,23 +68,23 @@ public class ClassGenerator {
} else if (algorithmSet.equals("MODELS")) { } else if (algorithmSet.equals("MODELS")) {
packageString = "modellers"; packageString = "modellers";
interfaceString = "IModeller"; interfaceString = "IModeller";
inputs = ModelersFactory.getModelParameters(configPath, algorithm); inputs = ModelersFactory.getModelParameters(configPath, algorithm,config);
description = ModelersFactory.getDescription(configPath, algorithm); description = ModelersFactory.getDescription(configPath, algorithm,config);
outputs = ModelersFactory.getModelOutput(configPath, algorithm); outputs = ModelersFactory.getModelOutput(configPath, algorithm,config);
} else if (algorithmSet.equals("CLUSTERERS")) { } else if (algorithmSet.equals("CLUSTERERS")) {
packageString = "clusterers"; packageString = "clusterers";
interfaceString = "IClusterer"; interfaceString = "IClusterer";
inputs = ClusterersFactory.getClustererParameters(configPath, algorithm); inputs = ClusterersFactory.getClustererParameters(configPath, algorithm,config);
description = ClusterersFactory.getDescription(configPath, algorithm); description = ClusterersFactory.getDescription(configPath, algorithm,config);
outputs = ClusterersFactory.getClustererOutput(configPath, algorithm); outputs = ClusterersFactory.getClustererOutput(configPath, algorithm,config);
} else if (algorithmSet.equals("TEMPORAL_ANALYSIS")) { } else if (algorithmSet.equals("TEMPORAL_ANALYSIS")) {
} else if (algorithmSet.equals("EVALUATORS")) { } else if (algorithmSet.equals("EVALUATORS")) {
packageString = "evaluators"; packageString = "evaluators";
interfaceString = "IEvaluator"; interfaceString = "IEvaluator";
inputs = EvaluatorsFactory.getEvaluatorParameters(configPath, algorithm); inputs = EvaluatorsFactory.getEvaluatorParameters(configPath, algorithm,config);
description = EvaluatorsFactory.getDescription(configPath, algorithm); description = EvaluatorsFactory.getDescription(configPath, algorithm,config);
outputs = EvaluatorsFactory.getEvaluatorOutput(configPath, algorithm); outputs = EvaluatorsFactory.getEvaluatorOutput(configPath, algorithm,config);
} }
}catch(Exception e){ }catch(Exception e){
LOGGER.error("Error in retrieving output: ",e); LOGGER.error("Error in retrieving output: ",e);

20
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java
View File

@ -1,9 +1,9 @@
package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure; package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.gcube.common.encryption.StringEncrypter; import org.gcube.common.encryption.StringEncrypter;
@ -28,6 +28,7 @@ public class InfrastructureDialoguer {
public DatabaseInfo getDatabaseInfo(String resourceName) throws Exception{ public DatabaseInfo getDatabaseInfo(String resourceName) throws Exception{
DatabaseInfo dbi = new DatabaseInfo(); DatabaseInfo dbi = new DatabaseInfo();
LOGGER.debug("Searching for Database "+resourceName+" in scope "+scope); LOGGER.debug("Searching for Database "+resourceName+" in scope "+scope);
SimpleQuery query = queryFor(ServiceEndpoint.class); SimpleQuery query = queryFor(ServiceEndpoint.class);
// query.addCondition("$resource/Profile/Category/text() eq 'Database' and $resource/Profile/Name eq 'StatisticalManagerDataBase' "); // query.addCondition("$resource/Profile/Category/text() eq 'Database' and $resource/Profile/Name eq 'StatisticalManagerDataBase' ");
@ -61,20 +62,19 @@ public class InfrastructureDialoguer {
public List<String> getAlgorithmsInScope() throws Exception{ public List<String> getAlgorithmsInScope() throws Exception{
LOGGER.debug("Searching for Algorithms in scope {} with classloader type {}",scope,Thread.currentThread().getContextClassLoader()); LOGGER.debug("Searching for Algorithms in scope {} with classloader type {}",scope,Thread.currentThread().getContextClassLoader().getClass().getSimpleName());
SimpleQuery query = queryFor(GenericResource.class); SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType eq 'StatisticalManagerAlgorithm' "); query.addCondition("$resource/Profile/SecondaryType/string() eq 'StatisticalManagerAlgorithm' ");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class); query.setResult("$resource/Profile/Name/text()");
List<GenericResource> resources = client.submit(query); DiscoveryClient<String> client = client();
List<String> resources = client.submit(query);
if (resources==null || resources.size()==0){ if (resources==null || resources.size()==0){
throw new Exception("No resource named StatisticalManagerAlgorithm available in scope "+scope); throw new Exception("No resource named StatisticalManagerAlgorithm available in scope "+scope);
} }
List<String> resourcesNames = new ArrayList<String>();
LOGGER.debug("Found {} algorithms",resources.size()); LOGGER.debug("Found {} algorithms",resources.size());
for (GenericResource resource: resources){
resourcesNames.add(resource.profile().name()); return resources;
}
return resourcesNames;
} }

70
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java
View File

@ -4,7 +4,6 @@ import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@ -51,13 +50,15 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
/** /**
* Deploying procedure: 1 - modify configuration files 2 - modify resource file: resources/templates/setup.cfg 3 - generate classes with ClassGenerator 4 - add new classes in the wps_config.xml on the wps web app config folder 5 - produce the Jar file of this project 6 - copy the jar file in the lib folder of the wps web app change the server parameters in the wps_config.xml file * Deploying procedure: 1 - modify configuration files 2 - modify resource file: resources/templates/setup.cfg 3 - generate classes with ClassGenerator 4 - add new classes in the wps_config.xml on the wps web app config folder 5 - produce the Jar file of this project 6 - copy the jar file in the lib folder of the wps web app change the server parameters in the wps_config.xml file
*/ */
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEcologicalEngineMapper.class); private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEcologicalEngineMapper.class);
private Observer observer = null; private Observer observer = null;
private boolean cancelled = false; private boolean cancelled = false;
private TokenManager tokenm = null;
// inputs and outputs // inputs and outputs
public LinkedHashMap<String, Object> inputs = new LinkedHashMap<String, Object>(); public LinkedHashMap<String, Object> inputs = new LinkedHashMap<String, Object>();
public LinkedHashMap<String, Object> outputs = new LinkedHashMap<String, Object>(); public LinkedHashMap<String, Object> outputs = new LinkedHashMap<String, Object>();
@ -141,13 +142,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
if (this instanceof ITransducer) if (this instanceof ITransducer)
parameters = TransducerersFactory.getTransducerParameters(config, algorithmName); parameters = TransducerersFactory.getTransducerParameters(config, algorithmName);
else if (this instanceof IClusterer) else if (this instanceof IClusterer)
parameters = ClusterersFactory.getClustererParameters(config.getConfigPath(), algorithmName); parameters = ClusterersFactory.getClustererParameters(config.getConfigPath(), algorithmName, config);
else if (this instanceof IEvaluator) else if (this instanceof IEvaluator)
parameters = EvaluatorsFactory.getEvaluatorParameters(config.getConfigPath(), algorithmName); parameters = EvaluatorsFactory.getEvaluatorParameters(config.getConfigPath(), algorithmName, config);
else if (this instanceof IGenerator) else if (this instanceof IGenerator)
parameters = GeneratorsFactory.getAlgorithmParameters(config.getConfigPath(), algorithmName); parameters = GeneratorsFactory.getAlgorithmParameters(config.getConfigPath(), algorithmName, config);
else if (this instanceof IModeller) else if (this instanceof IModeller)
parameters = ModelersFactory.getModelParameters(config.getConfigPath(), algorithmName); parameters = ModelersFactory.getModelParameters(config.getConfigPath(), algorithmName, config);
if (parameters != null) { if (parameters != null) {
LOGGER.debug("Found " + parameters.size() + " Parameters for " + algorithmName); LOGGER.debug("Found " + parameters.size() + " Parameters for " + algorithmName);
@ -163,13 +164,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
if (this instanceof ITransducer) if (this instanceof ITransducer)
output = TransducerersFactory.getTransducerOutput(config, algorithmName); output = TransducerersFactory.getTransducerOutput(config, algorithmName);
else if (this instanceof IClusterer) else if (this instanceof IClusterer)
output = ClusterersFactory.getClustererOutput(config.getConfigPath(), algorithmName); output = ClusterersFactory.getClustererOutput(config.getConfigPath(), algorithmName, config);
else if (this instanceof IEvaluator) else if (this instanceof IEvaluator)
output = EvaluatorsFactory.getEvaluatorOutput(config.getConfigPath(), algorithmName); output = EvaluatorsFactory.getEvaluatorOutput(config.getConfigPath(), algorithmName, config);
else if (this instanceof IGenerator) else if (this instanceof IGenerator)
output = GeneratorsFactory.getAlgorithmOutput(config.getConfigPath(), algorithmName); output = GeneratorsFactory.getAlgorithmOutput(config.getConfigPath(), algorithmName, config);
else if (this instanceof IModeller) else if (this instanceof IModeller)
output = ModelersFactory.getModelOutput(config.getConfigPath(), algorithmName); output = ModelersFactory.getModelOutput(config.getConfigPath(), algorithmName, config);
if (output != null) { if (output != null) {
LOGGER.debug("Found " + output + " for " + algorithmName); LOGGER.debug("Found " + output + " for " + algorithmName);
@ -229,7 +230,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
String token = null; String token = null;
// DONE get scope and username from SmartGears // DONE get scope and username from SmartGears
// get scope from SmartGears // get scope from SmartGears
TokenManager tokenm = new TokenManager(); tokenm = new TokenManager();
tokenm.getCredentials(); tokenm.getCredentials();
scope = tokenm.getScope(); scope = tokenm.getScope();
username = tokenm.getUserName(); username = tokenm.getUserName();
@ -246,7 +247,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
if (agent != null) { if (agent != null) {
if (status != previousStatus) { if (status != previousStatus) {
LOGGER.debug("STATUS update to: {} ", status ); LOGGER.debug("STATUS update to: {} ", status );
previousStatus = status; previousStatus = status;
super.update(new Integer((int) status)); super.update(new Integer((int) status));
try { try {
updateComputationOnWS(status, null); updateComputationOnWS(status, null);
@ -301,11 +302,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
public void run() throws Exception { public void run() throws Exception {
if (observer!=null) if (observer!=null)
observer.isStarted(this); observer.isStarted(this);
LOGGER.info("classloader context in this thread is {}",Thread.currentThread().getContextClassLoader());
long startTimeLong = System.currentTimeMillis(); long startTimeLong = System.currentTimeMillis();
OperationResult operationResult = null; OperationResult operationResult = null;
String algorithm = ""; String algorithm = "";
List<String> generatedInputTables = null; List<String> generatedInputTables = null;
List<String> generatedOutputTables = null; List<String> generatedOutputTables = null;
@ -327,7 +330,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
time("WPS Algorithm objects Initialization: Session " + computationSession); time("WPS Algorithm objects Initialization: Session " + computationSession);
// set the configuration environment for this algorithm // set the configuration environment for this algorithm
configManager.configAlgorithmEnvironment(inputs); configManager.configAlgorithmEnvironment(inputs);
configManager.setComputationId(computationSession); configManager.setComputationId(computationSession);
config = configManager.getConfig(); config = configManager.getConfig();
@ -458,6 +461,8 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
else else
throw e; throw e;
} finally { } finally {
LOGGER.debug("accounting algorithm");
accountAlgorithmExecution(startTimeLong, System.currentTimeMillis(), operationResult);
LOGGER.debug("Deleting Input Tables"); LOGGER.debug("Deleting Input Tables");
deleteTemporaryTables(generatedInputTables); deleteTemporaryTables(generatedInputTables);
LOGGER.debug("Deleting Output Tables"); LOGGER.debug("Deleting Output Tables");
@ -472,28 +477,27 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i
cleanResources(); cleanResources();
if (observer!=null) observer.isFinished(this); if (observer!=null) observer.isFinished(this);
LOGGER.debug("All done - Computation Finished"); LOGGER.debug("All done - Computation Finished");
accountAlgorithmExecution(configManager, startTimeLong, System.currentTimeMillis(), operationResult);
} }
} }
private void accountAlgorithmExecution(ConfigurationManager confManager, long start, long end, OperationResult result) { private void accountAlgorithmExecution(long start, long end, OperationResult result) {
try{ try{
JobUsageRecord jobUsageRecord = new JobUsageRecord(); JobUsageRecord jobUsageRecord = new JobUsageRecord();
jobUsageRecord.setJobName(this.getAlgorithmClass().getSimpleName()); jobUsageRecord.setJobName(this.getAlgorithmClass().getSimpleName());
jobUsageRecord.setConsumerId(confManager.getUsername()); jobUsageRecord.setConsumerId(tokenm.getUserName());
Calendar startCal = Calendar.getInstance(); jobUsageRecord.setDuration(end-start);
startCal.setTimeInMillis(start); jobUsageRecord.setOperationResult(result);
jobUsageRecord.setDuration(end-start); jobUsageRecord.setServiceName("DataMiner");
jobUsageRecord.setOperationResult(result); jobUsageRecord.setServiceClass("WPS");
jobUsageRecord.setServiceName("DataMiner"); jobUsageRecord.setHost(WPSConfig.getInstance().getWPSConfig().getServer().getHostname());
jobUsageRecord.setServiceClass("WPS"); jobUsageRecord.setCallerQualifier(tokenm.getTokenQualifier());
jobUsageRecord.setHost(WPSConfig.getInstance().getWPSConfig().getServer().getHostname());
AccountingPersistence accountingPersistence =
AccountingPersistence accountingPersistence = AccountingPersistenceFactory.getPersistence();
AccountingPersistenceFactory.getPersistence(); accountingPersistence.account(jobUsageRecord);
accountingPersistence.account(jobUsageRecord); }catch(Throwable e){
}catch(Exception e){
LOGGER.error("error accounting algorithm execution",e); LOGGER.error("error accounting algorithm execution",e);
} }

1
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java
View File

@ -85,6 +85,7 @@ public class ConfigurationManager {
public void configAlgorithmEnvironment(LinkedHashMap<String, Object> inputs) throws Exception { public void configAlgorithmEnvironment(LinkedHashMap<String, Object> inputs) throws Exception {
// set config container // set config container
config = new AlgorithmConfiguration(); config = new AlgorithmConfiguration();
config.setAlgorithmClassLoader(Thread.currentThread().getContextClassLoader());
String webperspath = WPSConfig.getConfigDir() + "../persistence/"; String webperspath = WPSConfig.getConfigDir() + "../persistence/";
// selecting persistence path // selecting persistence path
// String persistencePath = File.createTempFile("wpsstatcheck", ".sm").getParent() + "/../cfg/"; // String persistencePath = File.createTempFile("wpsstatcheck", ".sm").getParent() + "/../cfg/";

8
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java
View File

@ -15,7 +15,8 @@ public class TokenManager {
String username; String username;
String scope; String scope;
String token; String token;
String tokenQualifier;
public String getScope(){ public String getScope(){
return scope; return scope;
} }
@ -27,6 +28,10 @@ public class TokenManager {
public String getToken(){ public String getToken(){
return token; return token;
} }
public String getTokenQualifier() {
return tokenQualifier;
}
public void getCredentials() { public void getCredentials() {
try{ try{
@ -36,6 +41,7 @@ public class TokenManager {
token = SecurityTokenProvider.instance.get(); token = SecurityTokenProvider.instance.get();
AuthorizationEntry entry = authorizationService().get(token); AuthorizationEntry entry = authorizationService().get(token);
scope = entry.getContext(); scope = entry.getContext();
tokenQualifier = entry.getQualifier();
}catch(Exception e){ }catch(Exception e){
LOGGER.error("Error Retrieving token credentials ",e); LOGGER.error("Error Retrieving token credentials ",e);

4
src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java
View File

@ -18,8 +18,8 @@ public class AlgorithmTest {
@Test @Test
public void executeAlgorithmsFromFile() throws Exception{ public void executeAlgorithmsFromFile() throws Exception{
String protocol = "http"; String protocol = "http";
String hostname = "dataminer1-d-d4s.d4science.org"; String hostname = "dataminer1-pre.d4science.org";
String token = "595ca591-9921-423c-bfca-f8be19f05882-98187548"; String token = "a5b623b6-6577-4271-aba6-7ada687d29cf-98187548";
Iterator<String> uris = getUrisIterator(); Iterator<String> uris = getUrisIterator();