From 9fa8280374801ee8ca330c9e109055bc5a72a828 Mon Sep 17 00:00:00 2001 From: Lucio Lelii Date: Tue, 19 Sep 2017 15:19:27 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@153273 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../codegeneration/ClassGenerator.java | 25 +++---- .../InfrastructureDialoguer.java | 20 +++--- .../AbstractEcologicalEngineMapper.java | 70 ++++++++++--------- .../mapping/ConfigurationManager.java | 1 + .../synchserver/mapping/TokenManager.java | 8 ++- .../synchserver/test/AlgorithmTest.java | 4 +- 6 files changed, 70 insertions(+), 58 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/codegeneration/ClassGenerator.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/codegeneration/ClassGenerator.java index 5af1590..2b1437a 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/codegeneration/ClassGenerator.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/codegeneration/ClassGenerator.java @@ -49,15 +49,16 @@ public class ClassGenerator { // build class preamble config.setAgent(algorithm); config.setModel(algorithm); + config.setAlgorithmClassLoader(Thread.currentThread().getContextClassLoader()); String packageString = ""; String interfaceString = ""; try{ if (algorithmSet.equals("DISTRIBUTIONS")) { packageString = "generators"; interfaceString = "IGenerator"; - inputs = GeneratorsFactory.getAlgorithmParameters(configPath, algorithm); - description = GeneratorsFactory.getDescription(configPath, algorithm); - outputs = GeneratorsFactory.getAlgorithmOutput(configPath, algorithm); + inputs = GeneratorsFactory.getAlgorithmParameters(configPath, algorithm, config); + description = GeneratorsFactory.getDescription(configPath, algorithm,config); + outputs = GeneratorsFactory.getAlgorithmOutput(configPath, algorithm,config); } else if (algorithmSet.equals("TRANSDUCERS")) { packageString = "transducerers"; interfaceString = "ITransducer"; @@ -67,23 +68,23 @@ public class ClassGenerator { } else if (algorithmSet.equals("MODELS")) { packageString = "modellers"; interfaceString = "IModeller"; - inputs = ModelersFactory.getModelParameters(configPath, algorithm); - description = ModelersFactory.getDescription(configPath, algorithm); - outputs = ModelersFactory.getModelOutput(configPath, algorithm); + inputs = ModelersFactory.getModelParameters(configPath, algorithm,config); + description = ModelersFactory.getDescription(configPath, algorithm,config); + outputs = ModelersFactory.getModelOutput(configPath, algorithm,config); } else if (algorithmSet.equals("CLUSTERERS")) { packageString = "clusterers"; interfaceString = "IClusterer"; - inputs = ClusterersFactory.getClustererParameters(configPath, algorithm); - description = ClusterersFactory.getDescription(configPath, algorithm); - outputs = ClusterersFactory.getClustererOutput(configPath, algorithm); + inputs = ClusterersFactory.getClustererParameters(configPath, algorithm,config); + description = ClusterersFactory.getDescription(configPath, algorithm,config); + outputs = ClusterersFactory.getClustererOutput(configPath, algorithm,config); } else if (algorithmSet.equals("TEMPORAL_ANALYSIS")) { } else if (algorithmSet.equals("EVALUATORS")) { packageString = "evaluators"; interfaceString = "IEvaluator"; - inputs = EvaluatorsFactory.getEvaluatorParameters(configPath, algorithm); - description = EvaluatorsFactory.getDescription(configPath, algorithm); - outputs = EvaluatorsFactory.getEvaluatorOutput(configPath, algorithm); + inputs = EvaluatorsFactory.getEvaluatorParameters(configPath, algorithm,config); + description = EvaluatorsFactory.getDescription(configPath, algorithm,config); + outputs = EvaluatorsFactory.getEvaluatorOutput(configPath, algorithm,config); } }catch(Exception e){ LOGGER.error("Error in retrieving output: ",e); diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java index c8693da..7cbe2ac 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java @@ -1,9 +1,9 @@ package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure; +import static org.gcube.resources.discovery.icclient.ICFactory.client; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; -import java.util.ArrayList; import java.util.List; import org.gcube.common.encryption.StringEncrypter; @@ -28,6 +28,7 @@ public class InfrastructureDialoguer { public DatabaseInfo getDatabaseInfo(String resourceName) throws Exception{ DatabaseInfo dbi = new DatabaseInfo(); + LOGGER.debug("Searching for Database "+resourceName+" in scope "+scope); SimpleQuery query = queryFor(ServiceEndpoint.class); // query.addCondition("$resource/Profile/Category/text() eq 'Database' and $resource/Profile/Name eq 'StatisticalManagerDataBase' "); @@ -61,20 +62,19 @@ public class InfrastructureDialoguer { public List getAlgorithmsInScope() throws Exception{ - LOGGER.debug("Searching for Algorithms in scope {} with classloader type {}",scope,Thread.currentThread().getContextClassLoader()); + LOGGER.debug("Searching for Algorithms in scope {} with classloader type {}",scope,Thread.currentThread().getContextClassLoader().getClass().getSimpleName()); SimpleQuery query = queryFor(GenericResource.class); - query.addCondition("$resource/Profile/SecondaryType eq 'StatisticalManagerAlgorithm' "); - DiscoveryClient client = clientFor(GenericResource.class); - List resources = client.submit(query); + query.addCondition("$resource/Profile/SecondaryType/string() eq 'StatisticalManagerAlgorithm' "); + query.setResult("$resource/Profile/Name/text()"); + DiscoveryClient client = client(); + List resources = client.submit(query); if (resources==null || resources.size()==0){ throw new Exception("No resource named StatisticalManagerAlgorithm available in scope "+scope); } - List resourcesNames = new ArrayList(); LOGGER.debug("Found {} algorithms",resources.size()); - for (GenericResource resource: resources){ - resourcesNames.add(resource.profile().name()); - } - return resourcesNames; + + return resources; + } diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java index 245665f..a73860a 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java @@ -4,7 +4,6 @@ import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; @@ -51,13 +50,15 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i /** * Deploying procedure: 1 - modify configuration files 2 - modify resource file: resources/templates/setup.cfg 3 - generate classes with ClassGenerator 4 - add new classes in the wps_config.xml on the wps web app config folder 5 - produce the Jar file of this project 6 - copy the jar file in the lib folder of the wps web app change the server parameters in the wps_config.xml file */ - + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEcologicalEngineMapper.class); private Observer observer = null; private boolean cancelled = false; - + + private TokenManager tokenm = null; + // inputs and outputs public LinkedHashMap inputs = new LinkedHashMap(); public LinkedHashMap outputs = new LinkedHashMap(); @@ -141,13 +142,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i if (this instanceof ITransducer) parameters = TransducerersFactory.getTransducerParameters(config, algorithmName); else if (this instanceof IClusterer) - parameters = ClusterersFactory.getClustererParameters(config.getConfigPath(), algorithmName); + parameters = ClusterersFactory.getClustererParameters(config.getConfigPath(), algorithmName, config); else if (this instanceof IEvaluator) - parameters = EvaluatorsFactory.getEvaluatorParameters(config.getConfigPath(), algorithmName); + parameters = EvaluatorsFactory.getEvaluatorParameters(config.getConfigPath(), algorithmName, config); else if (this instanceof IGenerator) - parameters = GeneratorsFactory.getAlgorithmParameters(config.getConfigPath(), algorithmName); + parameters = GeneratorsFactory.getAlgorithmParameters(config.getConfigPath(), algorithmName, config); else if (this instanceof IModeller) - parameters = ModelersFactory.getModelParameters(config.getConfigPath(), algorithmName); + parameters = ModelersFactory.getModelParameters(config.getConfigPath(), algorithmName, config); if (parameters != null) { LOGGER.debug("Found " + parameters.size() + " Parameters for " + algorithmName); @@ -163,13 +164,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i if (this instanceof ITransducer) output = TransducerersFactory.getTransducerOutput(config, algorithmName); else if (this instanceof IClusterer) - output = ClusterersFactory.getClustererOutput(config.getConfigPath(), algorithmName); + output = ClusterersFactory.getClustererOutput(config.getConfigPath(), algorithmName, config); else if (this instanceof IEvaluator) - output = EvaluatorsFactory.getEvaluatorOutput(config.getConfigPath(), algorithmName); + output = EvaluatorsFactory.getEvaluatorOutput(config.getConfigPath(), algorithmName, config); else if (this instanceof IGenerator) - output = GeneratorsFactory.getAlgorithmOutput(config.getConfigPath(), algorithmName); + output = GeneratorsFactory.getAlgorithmOutput(config.getConfigPath(), algorithmName, config); else if (this instanceof IModeller) - output = ModelersFactory.getModelOutput(config.getConfigPath(), algorithmName); + output = ModelersFactory.getModelOutput(config.getConfigPath(), algorithmName, config); if (output != null) { LOGGER.debug("Found " + output + " for " + algorithmName); @@ -229,7 +230,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i String token = null; // DONE get scope and username from SmartGears // get scope from SmartGears - TokenManager tokenm = new TokenManager(); + tokenm = new TokenManager(); tokenm.getCredentials(); scope = tokenm.getScope(); username = tokenm.getUserName(); @@ -246,7 +247,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i if (agent != null) { if (status != previousStatus) { LOGGER.debug("STATUS update to: {} ", status ); - previousStatus = status; + previousStatus = status; super.update(new Integer((int) status)); try { updateComputationOnWS(status, null); @@ -301,11 +302,13 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i public void run() throws Exception { if (observer!=null) observer.isStarted(this); + + LOGGER.info("classloader context in this thread is {}",Thread.currentThread().getContextClassLoader()); long startTimeLong = System.currentTimeMillis(); - + OperationResult operationResult = null; - + String algorithm = ""; List generatedInputTables = null; List generatedOutputTables = null; @@ -327,7 +330,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i time("WPS Algorithm objects Initialization: Session " + computationSession); // set the configuration environment for this algorithm - + configManager.configAlgorithmEnvironment(inputs); configManager.setComputationId(computationSession); config = configManager.getConfig(); @@ -458,6 +461,8 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i else throw e; } finally { + LOGGER.debug("accounting algorithm"); + accountAlgorithmExecution(startTimeLong, System.currentTimeMillis(), operationResult); LOGGER.debug("Deleting Input Tables"); deleteTemporaryTables(generatedInputTables); LOGGER.debug("Deleting Output Tables"); @@ -472,28 +477,27 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm i cleanResources(); if (observer!=null) observer.isFinished(this); LOGGER.debug("All done - Computation Finished"); - accountAlgorithmExecution(configManager, startTimeLong, System.currentTimeMillis(), operationResult); + } } - private void accountAlgorithmExecution(ConfigurationManager confManager, long start, long end, OperationResult result) { + private void accountAlgorithmExecution(long start, long end, OperationResult result) { try{ - JobUsageRecord jobUsageRecord = new JobUsageRecord(); - jobUsageRecord.setJobName(this.getAlgorithmClass().getSimpleName()); - jobUsageRecord.setConsumerId(confManager.getUsername()); - Calendar startCal = Calendar.getInstance(); - startCal.setTimeInMillis(start); - jobUsageRecord.setDuration(end-start); - jobUsageRecord.setOperationResult(result); - jobUsageRecord.setServiceName("DataMiner"); - jobUsageRecord.setServiceClass("WPS"); - jobUsageRecord.setHost(WPSConfig.getInstance().getWPSConfig().getServer().getHostname()); - - AccountingPersistence accountingPersistence = - AccountingPersistenceFactory.getPersistence(); - accountingPersistence.account(jobUsageRecord); - }catch(Exception e){ + JobUsageRecord jobUsageRecord = new JobUsageRecord(); + jobUsageRecord.setJobName(this.getAlgorithmClass().getSimpleName()); + jobUsageRecord.setConsumerId(tokenm.getUserName()); + jobUsageRecord.setDuration(end-start); + jobUsageRecord.setOperationResult(result); + jobUsageRecord.setServiceName("DataMiner"); + jobUsageRecord.setServiceClass("WPS"); + jobUsageRecord.setHost(WPSConfig.getInstance().getWPSConfig().getServer().getHostname()); + jobUsageRecord.setCallerQualifier(tokenm.getTokenQualifier()); + + AccountingPersistence accountingPersistence = + AccountingPersistenceFactory.getPersistence(); + accountingPersistence.account(jobUsageRecord); + }catch(Throwable e){ LOGGER.error("error accounting algorithm execution",e); } diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java index 734a940..60be6d6 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java @@ -85,6 +85,7 @@ public class ConfigurationManager { public void configAlgorithmEnvironment(LinkedHashMap inputs) throws Exception { // set config container config = new AlgorithmConfiguration(); + config.setAlgorithmClassLoader(Thread.currentThread().getContextClassLoader()); String webperspath = WPSConfig.getConfigDir() + "../persistence/"; // selecting persistence path // String persistencePath = File.createTempFile("wpsstatcheck", ".sm").getParent() + "/../cfg/"; diff --git a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java index c191bd8..8896f5f 100644 --- a/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java +++ b/src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java @@ -15,7 +15,8 @@ public class TokenManager { String username; String scope; String token; - + String tokenQualifier; + public String getScope(){ return scope; } @@ -27,6 +28,10 @@ public class TokenManager { public String getToken(){ return token; } + + public String getTokenQualifier() { + return tokenQualifier; + } public void getCredentials() { try{ @@ -36,6 +41,7 @@ public class TokenManager { token = SecurityTokenProvider.instance.get(); AuthorizationEntry entry = authorizationService().get(token); scope = entry.getContext(); + tokenQualifier = entry.getQualifier(); }catch(Exception e){ LOGGER.error("Error Retrieving token credentials ",e); diff --git a/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java b/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java index 711e1a9..68c1a43 100644 --- a/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java +++ b/src/test/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/AlgorithmTest.java @@ -18,8 +18,8 @@ public class AlgorithmTest { @Test public void executeAlgorithmsFromFile() throws Exception{ String protocol = "http"; - String hostname = "dataminer1-d-d4s.d4science.org"; - String token = "595ca591-9921-423c-bfca-f8be19f05882-98187548"; + String hostname = "dataminer1-pre.d4science.org"; + String token = "a5b623b6-6577-4271-aba6-7ada687d29cf-98187548"; Iterator uris = getUrisIterator();