From 198f8390789fee976f7afcd41ed4cb7e762bda53 Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Thu, 15 Dec 2011 15:59:04 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@49455 82a268e6-3cf1-43bd-a215-b396298e98cf --- cfg/algorithms.properties | 20 +- cfg/generators.properties | 6 +- cfg/models.properties | 4 +- .../configuration/AlgorithmConfiguration.java | 144 +++++++++ .../configuration/GenericConfiguration.java | 278 ------------------ .../ecoengine/interfaces/Generator.java | 9 +- .../SpatialProbabilityDistribution.java | 7 + ...SpatialProbabilityDistributionGeneric.java | 5 +- .../SpatialProbabilityDistributionTable.java | 4 +- .../ecoengine/models/ModelHSPEN.java | 36 +-- .../processing/LocalSimpleSplitGenerator.java | 21 +- .../processing/LocalSplitGenerator.java | 50 ++-- .../processing/RainyCloudGenerator.java | 162 +++++----- .../factories/GeneratorsFactory.java | 82 ++++-- .../processing/factories/ModelersFactory.java | 8 +- .../AquamapsNative2050.java | 4 +- .../AquamapsSuitable.java | 35 ++- .../AquamapsSuitable2050.java | 4 +- .../spatialdistributions/DummyAlgorithm.java | 10 +- .../spatialdistributions/TestAlgorithm.java | 11 +- .../test/RegressionTestsGeneration.java | 164 ----------- .../test/RegressionTestsGenerationList.java | 174 ----------- .../test/RegressionTestsModeling.java | 93 ------ .../ecoengine/test/TestsMetaInfo.java | 92 ++++++ .../ecoengine/test/TestsTESTGeneration.java | 8 +- .../GenerationHSPECValidation.java | 26 +- .../RegressionComplexGeneration.java | 129 ++++++++ .../RegressionSimpleGeneration.java} | 41 ++- .../RegressionTestLocalGenerationStatus.java | 69 ----- ...onTestNative2050LocalGenerationStatus.java | 69 ----- ...essionTestNativeLocalGenerationStatus.java | 69 ----- .../RegressionTestRemoteGenerationStatus.java | 83 ------ ...TestSuitable2050LocalGenerationStatus.java | 69 ----- .../test/tables/CompareHspecSingle.java | 22 +- .../ecoengine/test/tables/CompareTables.java | 21 +- ...java => HspecDiscrepanciesCalculator.java} | 30 +- .../utils/TablesDiscrepanciesCalculator.java | 26 +- 37 files changed, 746 insertions(+), 1339 deletions(-) create mode 100644 src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/configuration/GenericConfiguration.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGeneration.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGenerationList.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsModeling.java create mode 100644 src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java create mode 100644 src/org/gcube/dataanalysis/ecoengine/test/regressions/RegressionComplexGeneration.java rename src/org/gcube/dataanalysis/ecoengine/test/{TestsDummyGeneration.java => regressions/RegressionSimpleGeneration.java} (57%) delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestLocalGenerationStatus.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNative2050LocalGenerationStatus.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNativeLocalGenerationStatus.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestRemoteGenerationStatus.java delete mode 100644 src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestSuitable2050LocalGenerationStatus.java rename src/org/gcube/dataanalysis/ecoengine/utils/{HpecDiscrepanciesCalculator.java => HspecDiscrepanciesCalculator.java} (86%) diff --git a/cfg/algorithms.properties b/cfg/algorithms.properties index de95467..0d85bec 100644 --- a/cfg/algorithms.properties +++ b/cfg/algorithms.properties @@ -1,10 +1,10 @@ -AQUAMAPS_SUITABLE=org.gcube.application.aquamaps.ecomodelling.generators.spatialdistributions.AquamapsSuitable -AQUAMAPS_NATIVE=org.gcube.application.aquamaps.ecomodelling.generators.spatialdistributions.AquamapsNative -AQUAMAPS_NATIVE_2050=org.gcube.application.aquamaps.ecomodelling.generators.spatialdistributions.AquamapsNative2050 -AQUAMAPS_SUITABLE_2050=org.gcube.application.aquamaps.ecomodelling.generators.spatialdistributions.AquamapsSuitable2050 -REMOTE_AQUAMAPS_SUITABLE=org.gcube.application.aquamaps.ecomodelling.generators.processing.RainyCloudGenerator -REMOTE_AQUAMAPS_NATIVE=org.gcube.application.aquamaps.ecomodelling.generators.processing.RainyCloudGenerator -REMOTE_AQUAMAPS_NATIVE_2050=org.gcube.application.aquamaps.ecomodelling.generators.processing.RainyCloudGenerator -REMOTE_AQUAMAPS_SUITABLE_2050=org.gcube.application.aquamaps.ecomodelling.generators.processing.RainyCloudGenerator -DUMMY=org.gcube.application.aquamaps.ecomodelling.generators.spatialdistributions.DummyAlgorithm -TEST=org.gcube.application.aquamaps.ecomodelling.generators.spatialdistributions.TestAlgorithm \ No newline at end of file +AQUAMAPS_SUITABLE=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsSuitable +AQUAMAPS_NATIVE=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNative +AQUAMAPS_NATIVE_2050=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNative2050 +AQUAMAPS_SUITABLE_2050=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsSuitable2050 +REMOTE_AQUAMAPS_SUITABLE=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator +REMOTE_AQUAMAPS_NATIVE=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator +REMOTE_AQUAMAPS_NATIVE_2050=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator +REMOTE_AQUAMAPS_SUITABLE_2050=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator +DUMMY=org.gcube.dataanalysis.ecoengine.spatialdistributions.DummyAlgorithm +TEST=org.gcube.dataanalysis.ecoengine.spatialdistributions.TestAlgorithm \ No newline at end of file diff --git a/cfg/generators.properties b/cfg/generators.properties index e57f150..0752409 100644 --- a/cfg/generators.properties +++ b/cfg/generators.properties @@ -1,3 +1,3 @@ -LOCAL_WITH_DATABASE=org.gcube.application.aquamaps.ecomodelling.generators.processing.LocalSplitGenerator -SIMPLE_LOCAL=org.gcube.application.aquamaps.ecomodelling.generators.processing.LocalSimpleSplitGenerator -REMOTE_RAINYCLOUD=org.gcube.application.aquamaps.ecomodelling.generators.processing.RainyCloudGenerator \ No newline at end of file +LOCAL_WITH_DATABASE=org.gcube.dataanalysis.ecoengine.processing.LocalSplitGenerator +SIMPLE_LOCAL=org.gcube.dataanalysis.ecoengine.processing.LocalSimpleSplitGenerator +REMOTE_RAINYCLOUD=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator \ No newline at end of file diff --git a/cfg/models.properties b/cfg/models.properties index 005d336..5021dd7 100644 --- a/cfg/models.properties +++ b/cfg/models.properties @@ -1,2 +1,2 @@ -HSPEN=org.gcube.application.aquamaps.ecomodelling.generators.models.ModelHSPEN -HSPEN_MODELER=org.gcube.application.aquamaps.ecomodelling.generators.modeling.SimpleModeler \ No newline at end of file +HSPEN=org.gcube.dataanalysis.ecoengine.models.ModelHSPEN +HSPEN_MODELER=org.gcube.dataanalysis.ecoengine.modeling.SimpleModeler \ No newline at end of file diff --git a/src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java b/src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java new file mode 100644 index 0000000..8fc5a5f --- /dev/null +++ b/src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java @@ -0,0 +1,144 @@ +package org.gcube.dataanalysis.ecoengine.configuration; + +import java.io.FileInputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; + +import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; + +public class AlgorithmConfiguration extends LexicalEngineConfiguration{ + + public static Properties getProperties(String absoluteFilePath) { + Properties props = new Properties(); + FileInputStream fis = null; + try { + fis = new FileInputStream(absoluteFilePath); + props.load(fis); + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + fis.close(); + } catch (Exception e) { + } + } + return props; + } + + // constants + public static String defaultConnectionFile = "DestinationDBHibernate.cfg.xml"; + public static String defaultLoggerFile = "ALog.properties"; + public static String algorithmsFile = "algorithms.properties"; + public static String generatorsFile = "generators.properties"; + public static String modelsFile = "models.properties"; + public static int chunkSize = 100000; + public static int refreshResourcesTime = 10; + + // database parameters + private String databaseDriver = "org.postgresql.Driver"; + private String databaseURL = null; + private String databaseUserName = null; + private String databasePassword = null; + private String databaseDialect = null; + private String databaseIdleConnectionTestPeriod = null; + private String databaseAutomaticTestTable = null; + + // Algorithm Parameters + private String configPath; + private String cachePath; + private String persistencePath; + private String distributionTable; + private Boolean createTable = false; + private Boolean useDB = true; + + private String envelopeTable; + private String csquarecodesTable; + private String occurrenceCellsTable; + private List featuresTable; + private List preprocessedTables; + + //service and remote + private String remoteCalculatorEndpoint; + private String serviceUserName; + private String remoteEnvironment; + private Integer numberOfResources; + + //modeling + private String model; + private String generator; + + //other properties + private HashMap generalProperties; + + public String getParam(String key){ + if (generalProperties != null) + return generalProperties.get(key); + else return null; + } + + public void setParam(String key,String value){ + if (generalProperties == null) + generalProperties = new HashMap(); + + generalProperties.put(key,value); + } + + + public void setConfigPath(String configPath) { + if (!configPath.endsWith("/")) + configPath+="/"; + this.configPath = configPath; + } + + public String getConfigPath() { + return configPath; + } + + public void setNumberOfResources(Integer numberOfThreads) { + this.numberOfResources = numberOfThreads; + } + + public Integer getNumberOfResources() { + return numberOfResources; + } + + public void addGeneralProperties(HashMap generalProperties) { + for (String key:generalProperties.keySet()) { + this.generalProperties.put(key,generalProperties.get(key)); + } + } + + public void setGeneralProperties(HashMap generalProperties) { + this.generalProperties = generalProperties; + } + + public HashMap getGeneralProperties() { + return generalProperties; + } + + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + public String getPersistencePath() { + return persistencePath; + } + + public void setPersistencePath(String persistencePath) { + this.persistencePath = persistencePath; + } + + public String getGenerator() { + return generator; + } + + public void setGenerator(String generator) { + this.generator = generator; + } + +} diff --git a/src/org/gcube/dataanalysis/ecoengine/configuration/GenericConfiguration.java b/src/org/gcube/dataanalysis/ecoengine/configuration/GenericConfiguration.java deleted file mode 100644 index 5732f97..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/configuration/GenericConfiguration.java +++ /dev/null @@ -1,278 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.configuration; - -import java.io.FileInputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; - -public class GenericConfiguration extends LexicalEngineConfiguration{ - - public static Properties getProperties(String absoluteFilePath) { - Properties props = new Properties(); - FileInputStream fis = null; - try { - fis = new FileInputStream(absoluteFilePath); - props.load(fis); - } catch (Exception e) { - e.printStackTrace(); - } finally { - try { - fis.close(); - } catch (Exception e) { - } - } - return props; - } - - // constants - public static String defaultConnectionFile = "DestinationDBHibernate.cfg.xml"; - public static String defaultLoggerFile = "ALog.properties"; - public static String algorithmsFile = "algorithms.properties"; - public static String generatorsFile = "generators.properties"; - public static String modelsFile = "models.properties"; - public static int chunkSize = 100000; - public static int refreshResourcesTime = 10; - - // database parameters - private String databaseDriver = "org.postgresql.Driver"; - private String databaseURL = null; - private String databaseUserName = null; - private String databasePassword = null; - private String databaseDialect = null; - private String databaseIdleConnectionTestPeriod = null; - private String databaseAutomaticTestTable = null; - - // Algorithm Parameters - private String configPath; - private String cachePath; - private String persistencePath; - private String distributionTable; - private Boolean createTable = false; - private Boolean useDB = true; - - private String envelopeTable; - private String csquarecodesTable; - private String occurrenceCellsTable; - private List featuresTable; - private List preprocessedTables; - - //service and remote - private String remoteCalculatorEndpoint; - private String serviceUserName; - private String remoteEnvironment; - private Integer numberOfResources; - - //modeling - private String model; - private String generator; - - //other properties - private HashMap generalProperties; - - - - public void setDatabaseDriver(String databaseDriver) { - this.databaseDriver = databaseDriver; - } - - public String getDatabaseDriver() { - return databaseDriver; - } - - public void setDatabaseURL(String databaseURL) { - this.databaseURL = databaseURL; - } - - public String getDatabaseURL() { - return databaseURL; - } - - public void setDatabaseUserName(String databaseUserName) { - this.databaseUserName = databaseUserName; - } - - public String getDatabaseUserName() { - return databaseUserName; - } - - public void setDatabasePassword(String databasePassword) { - this.databasePassword = databasePassword; - } - - public String getDatabasePassword() { - return databasePassword; - } - - public void setDatabaseDialect(String databaseDialect) { - this.databaseDialect = databaseDialect; - } - - public String getDatabaseDialect() { - return databaseDialect; - } - - public void setDatabaseIdleConnectionTestPeriod(String databaseIdleConnectionTestPeriod) { - this.databaseIdleConnectionTestPeriod = databaseIdleConnectionTestPeriod; - } - - public String getDatabaseIdleConnectionTestPeriod() { - return databaseIdleConnectionTestPeriod; - } - - public void setDatabaseAutomaticTestTable(String databaseAutomaticTestTable) { - this.databaseAutomaticTestTable = databaseAutomaticTestTable; - } - - public String getDatabaseAutomaticTestTable() { - return databaseAutomaticTestTable; - } - - public void setConfigPath(String configPath) { - if (!configPath.endsWith("/")) - configPath+="/"; - this.configPath = configPath; - } - - public String getConfigPath() { - return configPath; - } - - public void setDistributionTable(String distributionTable) { - this.distributionTable = distributionTable; - } - - public String getDistributionTable() { - return distributionTable; - } - - public void setCreateTable(Boolean createTable) { - this.createTable = createTable; - } - - public Boolean createTable() { - return createTable; - } - - public void setNumberOfResources(Integer numberOfThreads) { - this.numberOfResources = numberOfThreads; - } - - public Integer getNumberOfResources() { - return numberOfResources; - } - - public void setUseDB(Boolean writeOnDB) { - this.useDB = writeOnDB; - } - - public Boolean useDB() { - return useDB; - } - - public void setRemoteCalculator(String remoteCalculator) { - this.remoteCalculatorEndpoint = remoteCalculator; - } - - public String getRemoteCalculator() { - return remoteCalculatorEndpoint; - } - - public void setServiceUserName(String serviceUserName) { - this.serviceUserName = serviceUserName; - } - - public String getServiceUserName() { - return serviceUserName; - } - - public void setCachePath(String cachePath) { - this.cachePath = cachePath; - } - - public String getCachePath() { - return cachePath; - } - - public void setGeneralProperties(HashMap generalProperties) { - this.generalProperties = generalProperties; - } - - public HashMap getGeneralProperties() { - return generalProperties; - } - - public void setRemoteEnvironment(String remoteEnvironment) { - this.remoteEnvironment = remoteEnvironment; - } - - public String getRemoteEnvironment() { - return remoteEnvironment; - } - - public String getModel() { - return model; - } - - public void setModel(String model) { - this.model = model; - } - - public String getEnvelopeTable() { - return envelopeTable; - } - - public void setEnvelopeTable(String envelopeTable) { - this.envelopeTable = envelopeTable; - } - - public String getCsquarecodesTable() { - return csquarecodesTable; - } - - public void setCsquarecodesTable(String csquarecodesTable) { - this.csquarecodesTable = csquarecodesTable; - } - - public List getFeaturesTable() { - return featuresTable; - } - - public void setFeaturesTable(List featuresTable) { - this.featuresTable = featuresTable; - } - - public List getPreprocessedTables() { - return preprocessedTables; - } - - public void setPreprocessedTables(List preprocessedTables) { - this.preprocessedTables = preprocessedTables; - } - - public String getOccurrenceCellsTable() { - return occurrenceCellsTable; - } - - public void setOccurrenceCellsTable(String occurrenceCellsTable) { - this.occurrenceCellsTable = occurrenceCellsTable; - } - - public String getPersistencePath() { - return persistencePath; - } - - public void setPersistencePath(String persistencePath) { - this.persistencePath = persistencePath; - } - - public String getGenerator() { - return generator; - } - - public void setGenerator(String generator) { - this.generator = generator; - } - -} diff --git a/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java b/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java index f6214af..de692ec 100644 --- a/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java +++ b/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java @@ -1,13 +1,18 @@ package org.gcube.dataanalysis.ecoengine.interfaces; +import java.util.HashMap; + import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; public interface Generator { public ALG_PROPS[] getSupportedAlgorithms(); + //set the input parameters for this generator + public HashMap getInputParameters(); + //gets the weight of the generator: according to this the generator will be placed in the execution order public GENERATOR_WEIGHT getWeight(); @@ -15,7 +20,7 @@ public interface Generator { public void init(); - public void setConfiguration(GenericConfiguration config); + public void setConfiguration(AlgorithmConfiguration config); public void shutdown(); diff --git a/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistribution.java b/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistribution.java index 10321e7..73c91e8 100644 --- a/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistribution.java +++ b/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistribution.java @@ -1,8 +1,11 @@ package org.gcube.dataanalysis.ecoengine.interfaces; +import java.util.HashMap; + import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; public interface SpatialProbabilityDistribution { + //defines the properties of this algorithm public ALG_PROPS[] getProperties(); @@ -11,4 +14,8 @@ public interface SpatialProbabilityDistribution { //gets the description of the algorithm public String getDescription(); + + //set the input parameters for this generator + public HashMap getInputParameters(); + } diff --git a/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionGeneric.java b/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionGeneric.java index 50d99ae..f10189a 100644 --- a/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionGeneric.java +++ b/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionGeneric.java @@ -3,14 +3,13 @@ package org.gcube.dataanalysis.ecoengine.interfaces; import java.util.List; import java.util.Map; -import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; //implements a SpatialProbabilityDistribution where data are taken from a Database public interface SpatialProbabilityDistributionGeneric extends SpatialProbabilityDistribution{ //initialization of the distribution model - public void init(GenericConfiguration config); + public void init(AlgorithmConfiguration config); public String getMainInfoType(); diff --git a/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionTable.java b/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionTable.java index 0ef48e4..610f206 100644 --- a/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionTable.java +++ b/src/org/gcube/dataanalysis/ecoengine/interfaces/SpatialProbabilityDistributionTable.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.ecoengine.interfaces; import java.util.Queue; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; //implements a SpatialProbabilityDistribution where data are taken from a Database @@ -13,7 +13,7 @@ public interface SpatialProbabilityDistributionTable extends SpatialProbabilityD public ALG_PROPS[] getProperties(); //initialization of the distribution model - public void init(GenericConfiguration config,SessionFactory dbHibConnection); + public void init(AlgorithmConfiguration config,SessionFactory dbHibConnection); //get the way principal info will be queried public String getMainInfoQuery(); diff --git a/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java b/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java index 13d7c27..2c92298 100644 --- a/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java +++ b/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java @@ -9,7 +9,7 @@ import java.util.concurrent.Executors; import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; import org.gcube.dataanalysis.ecoengine.interfaces.Model; @@ -52,7 +52,7 @@ public class ModelHSPEN implements Model { HashMap> allSpeciesHspen; private int lastProcessedRecordsNumber; private long lastTime; - GenericConfiguration outconfig; + AlgorithmConfiguration outconfig; @Override public float getVersion() { @@ -67,11 +67,11 @@ public class ModelHSPEN implements Model { @Override public void init(Object Input, Model previousModel, Object Destination) { - GenericConfiguration config = (GenericConfiguration) Input; - outconfig = (GenericConfiguration) Destination; + AlgorithmConfiguration config = (AlgorithmConfiguration) Input; + outconfig = (AlgorithmConfiguration) Destination; defaultDatabaseFile = config.getConfigPath() + defaultDatabaseFile; - AnalysisLogger.setLogger(config.getConfigPath() + GenericConfiguration.defaultLoggerFile); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); try { connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, config); } catch (Exception e) { @@ -80,14 +80,14 @@ public class ModelHSPEN implements Model { } // initialize queries - dynamicAlterQuery = alterQuery.replace("%HSPEN%", outconfig.getEnvelopeTable()); - dynamicDropTable = dropHspenTable.replace("%HSPEN%", outconfig.getEnvelopeTable()); - dynamicCreateTable = createHspenTable.replace("%HSPEN%", outconfig.getEnvelopeTable()); - dynamicPopulateNewHspen = populateNewHspen.replace("%HSPEN_ORIGIN%", config.getEnvelopeTable()).replace("%HSPEN%", outconfig.getEnvelopeTable()); - dynamicSpeciesListQuery = speciesListQuery.replace("%HSPEN%", config.getEnvelopeTable()); - dynamicHspenInformationQuery = hspenListQuery.replace("%HSPEN%", config.getEnvelopeTable()); - currentHCAFTable = config.getCsquarecodesTable(); - currentOccurrenceTable = config.getOccurrenceCellsTable(); + dynamicAlterQuery = alterQuery.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); + dynamicDropTable = dropHspenTable.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); + dynamicCreateTable = createHspenTable.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); + dynamicPopulateNewHspen = populateNewHspen.replace("%HSPEN_ORIGIN%", config.getParam("EnvelopeTable")).replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); + dynamicSpeciesListQuery = speciesListQuery.replace("%HSPEN%", config.getParam("EnvelopeTable")); + dynamicHspenInformationQuery = hspenListQuery.replace("%HSPEN%", config.getParam("EnvelopeTable")); + currentHCAFTable = config.getParam("CsquarecodesTable"); + currentOccurrenceTable = config.getParam("OccurrenceCellsTable"); // Threads numberOfthreads = config.getNumberOfResources(); @@ -151,9 +151,9 @@ public class ModelHSPEN implements Model { } private void generateTable(Object Input) throws Exception { - GenericConfiguration config = (GenericConfiguration) Input; + AlgorithmConfiguration config = (AlgorithmConfiguration) Input; // create and populate the novel table - if (config.createTable()){ + if (config.getParam("CreateTable").equalsIgnoreCase("true")){ AnalysisLogger.getLogger().trace("Distribution Generator->recreating new table " + dynamicCreateTable); try{ DatabaseFactory.executeSQLUpdate(String.format(dynamicDropTable, config.getDatabaseUserName()), connection); @@ -326,7 +326,7 @@ public class ModelHSPEN implements Model { @Override public String getContentType() { - return GenericConfiguration.class.getName(); + return AlgorithmConfiguration.class.getName(); } @Override @@ -397,12 +397,12 @@ public class ModelHSPEN implements Model { @Override public String getInputType() { - return GenericConfiguration.class.getName(); + return AlgorithmConfiguration.class.getName(); } @Override public String getOutputType() { - return GenericConfiguration.class.getName(); + return AlgorithmConfiguration.class.getName(); } } diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java index a4cfaeb..0c01c4f 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java @@ -1,5 +1,6 @@ package org.gcube.dataanalysis.ecoengine.processing; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -12,8 +13,8 @@ import java.util.concurrent.Executors; import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; @@ -21,7 +22,7 @@ import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributio public class LocalSimpleSplitGenerator implements Generator { - private GenericConfiguration config; + private AlgorithmConfiguration config; private ExecutorService executorService; private int numberOfThreadsToUse; private boolean threadActivity[]; @@ -39,11 +40,6 @@ public class LocalSimpleSplitGenerator implements Generator { //species Objects -> (geographical Object , Probability) ConcurrentHashMap> completeDistribution; - public LocalSimpleSplitGenerator(GenericConfiguration config) { - setConfiguration(config); - init(); - } - public LocalSimpleSplitGenerator() { } @@ -93,7 +89,7 @@ public class LocalSimpleSplitGenerator implements Generator { @Override public void init() { - AnalysisLogger.setLogger(config.getConfigPath() + GenericConfiguration.defaultLoggerFile); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); stopInterrupt = false; completeDistribution = new ConcurrentHashMap>(); try { @@ -108,14 +104,14 @@ public class LocalSimpleSplitGenerator implements Generator { } private void initModel() throws Exception { - Properties p = GenericConfiguration.getProperties(config.getConfigPath() + GenericConfiguration.algorithmsFile); + Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.algorithmsFile); String objectclass = p.getProperty(config.getModel()); distributionModel = (SpatialProbabilityDistributionGeneric) Class.forName(objectclass).newInstance(); distributionModel.init(config); } @Override - public void setConfiguration(GenericConfiguration config) { + public void setConfiguration(AlgorithmConfiguration config) { this.config = config; if (config.getNumberOfResources() == 0) this.numberOfThreadsToUse = 1; @@ -334,6 +330,11 @@ public class LocalSimpleSplitGenerator implements Generator { return GENERATOR_WEIGHT.LOWEST; } + @Override + public HashMap getInputParameters() { + return distributionModel.getInputParameters(); + } + } diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java index 70f316c..d76fea5 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java @@ -1,5 +1,6 @@ package org.gcube.dataanalysis.ecoengine.processing; +import java.util.HashMap; import java.util.List; import java.util.Properties; import java.util.Queue; @@ -14,8 +15,8 @@ import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; @@ -25,7 +26,7 @@ import org.hibernate.SessionFactory; public class LocalSplitGenerator implements Generator { - private GenericConfiguration config; + private AlgorithmConfiguration config; private ExecutorService executorService; private int numberOfThreadsToUse; private boolean threadActivity[]; @@ -45,7 +46,7 @@ public class LocalSplitGenerator implements Generator { String probabilityInsertionStatement = "insert into %1$s (speciesid,csquarecode,probability %ADDEDINFORMATION%) VALUES %2$s"; - public LocalSplitGenerator(GenericConfiguration config) { + public LocalSplitGenerator(AlgorithmConfiguration config) { setConfiguration(config); init(); } @@ -99,7 +100,7 @@ public class LocalSplitGenerator implements Generator { @Override public void init() { - AnalysisLogger.setLogger(config.getConfigPath() + GenericConfiguration.defaultLoggerFile); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); stopInterrupt = false; flushInterrupt = false; initDBSession(); @@ -124,19 +125,19 @@ public class LocalSplitGenerator implements Generator { AnalysisLogger.getLogger().trace("init()->insertion scheduler initialized"); // inizialize the scheduler for the insertions Timer writerScheduler = new Timer(); - writerScheduler.schedule(new DatabaseWriter(), 0, GenericConfiguration.refreshResourcesTime); + writerScheduler.schedule(new DatabaseWriter(), 0, AlgorithmConfiguration.refreshResourcesTime); } } private void initModel() throws Exception { - Properties p = GenericConfiguration.getProperties(config.getConfigPath() + GenericConfiguration.algorithmsFile); + Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.algorithmsFile); String objectclass = p.getProperty(config.getModel()); distributionModel = (SpatialProbabilityDistributionTable) Class.forName(objectclass).newInstance(); distributionModel.init(config, dbHibConnection); } @Override - public void setConfiguration(GenericConfiguration config) { + public void setConfiguration(AlgorithmConfiguration config) { this.config = config; if (config.getNumberOfResources() == 0) this.numberOfThreadsToUse = 1; @@ -158,7 +159,13 @@ public class LocalSplitGenerator implements Generator { try { if ((config != null) && (config.getConfigPath() != null)) { - String defaultDatabaseFile = config.getConfigPath() + GenericConfiguration.defaultConnectionFile; + String defaultDatabaseFile = config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile; + + config.setDatabaseDriver(config.getParam("DatabaseDriver")); + config.setDatabaseUserName(config.getParam("DatabaseUserName")); + config.setDatabasePassword(config.getParam("DatabasePassword")); + config.setDatabaseURL(config.getParam("DatabaseURL")); + dbHibConnection = DatabaseFactory.initDBConnection(defaultDatabaseFile, config); } } catch (Exception e) { @@ -169,13 +176,15 @@ public class LocalSplitGenerator implements Generator { } private void createTable() throws Exception { - if (config.createTable()) { + if (config.getParam("CreateTable")!=null && config.getParam("CreateTable").equalsIgnoreCase("true")) { try { - DatabaseFactory.executeSQLUpdate("drop table " + config.getDistributionTable(), dbHibConnection); + AnalysisLogger.getLogger().trace("recreating table: "+"drop table " + config.getParam("DistributionTable")); + DatabaseFactory.executeSQLUpdate("drop table " + config.getParam("DistributionTable"), dbHibConnection); + AnalysisLogger.getLogger().trace("recreating table->OK"); } catch (Exception e) { - + AnalysisLogger.getLogger().trace("recreating table->"+e.getLocalizedMessage()); } - DatabaseFactory.executeUpdateNoTransaction(distributionModel.getDistributionTableStatement(), config.getDatabaseDriver(), config.getDatabaseUserName(), config.getDatabasePassword(), config.getDatabaseURL(), true); + DatabaseFactory.executeUpdateNoTransaction(distributionModel.getDistributionTableStatement(), config.getParam("DatabaseDriver"), config.getParam("DatabaseUserName"), config.getParam("DatabasePassword"), config.getParam("DatabaseURL"), true); AnalysisLogger.getLogger().trace("createTable()->OK!"); } @@ -322,10 +331,12 @@ public class LocalSplitGenerator implements Generator { AnalysisLogger.getLogger().error(e); throw e; } finally { + try{ // REPORT OVERALL ELAPSED TIME distributionModel.postProcess(); // shutdown all shutdown(); + }catch(Exception e){} long tend = System.currentTimeMillis(); long ttotal = tend - tstart; AnalysisLogger.getLogger().warn("generate->Distribution Generator->Algorithm finished in: " + ((double) ttotal / (double) 60000) + " min\n"); @@ -398,9 +409,9 @@ public class LocalSplitGenerator implements Generator { AnalysisLogger.getLogger().trace("\t...finished flushing on db"); flushInterrupt = true; this.cancel(); - } else if ((probabilityBuffer != null) && (probabilityBuffer.size() > GenericConfiguration.chunkSize)) { + } else if ((probabilityBuffer != null) && (probabilityBuffer.size() > AlgorithmConfiguration.chunkSize)) { // AnalysisLogger.getLogger().trace("\t...writing on db"); - writeOnDB(GenericConfiguration.chunkSize); + writeOnDB(AlgorithmConfiguration.chunkSize); // AnalysisLogger.getLogger().trace("\t...finished writing on db"); } } catch (Exception e) { @@ -414,8 +425,8 @@ public class LocalSplitGenerator implements Generator { public void flushBuffer() { if ((probabilityBuffer != null) && (probabilityBuffer.size() > 0)) { - while (probabilityBuffer.size() > GenericConfiguration.chunkSize) - writeOnDB(GenericConfiguration.chunkSize); + while (probabilityBuffer.size() > AlgorithmConfiguration.chunkSize) + writeOnDB(AlgorithmConfiguration.chunkSize); writeOnDB(probabilityBuffer.size()); } @@ -434,7 +445,7 @@ public class LocalSplitGenerator implements Generator { } } - String insertionString = String.format(probabilityInsertionStatement, config.getDistributionTable(), sb.toString()); + String insertionString = String.format(probabilityInsertionStatement, config.getParam("DistributionTable"), sb.toString()); try { // AnalysisLogger.getLogger().debug("->"+insertionString); @@ -462,4 +473,9 @@ public class LocalSplitGenerator implements Generator { return GENERATOR_WEIGHT.LOWEST; } + @Override + public HashMap getInputParameters() { + return distributionModel.getInputParameters(); + } + } diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java b/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java index 407aa55..5a5ee20 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java @@ -1,10 +1,12 @@ package org.gcube.dataanalysis.ecoengine.processing; +import java.util.HashMap; + import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; import org.gcube.dataanalysis.ecoengine.connectors.RemoteGenerationManager; import org.gcube.dataanalysis.ecoengine.connectors.RemoteHspecInputObject; import org.gcube.dataanalysis.ecoengine.connectors.RemoteHspecOutputObject; @@ -12,85 +14,81 @@ import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; -public class RainyCloudGenerator implements Generator{ +public class RainyCloudGenerator implements Generator { - GenericConfiguration config; + AlgorithmConfiguration config; private boolean interruptProcessing; RemoteGenerationManager remoteGenerationManager; RemoteHspecInputObject rhio; - - public RainyCloudGenerator(GenericConfiguration config) { + + public RainyCloudGenerator(AlgorithmConfiguration config) { setConfiguration(config); init(); } - + public RainyCloudGenerator() { } - + @Override public float getStatus() { RemoteHspecOutputObject oo = remoteGenerationManager.retrieveCompleteStatus(); - -// if (oo.status.equals("DONE")||oo.status.equals("ERROR")) - if (oo.status.equals("DONE")) - { + + // if (oo.status.equals("DONE")||oo.status.equals("ERROR")) + if (oo.status.equals("DONE")) { stopProcess(); - return 100f; - } - else { - float remoteStatus =(float)remoteGenerationManager.retrieveCompletion(); - return (remoteStatus==100)?99:remoteStatus; + return 100f; + } else { + float remoteStatus = (float) remoteGenerationManager.retrieveCompletion(); + return (remoteStatus == 100) ? 99 : remoteStatus; } } @Override public void init() { - AnalysisLogger.setLogger(config.getConfigPath() + GenericConfiguration.defaultLoggerFile); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); interruptProcessing = false; rhio = new RemoteHspecInputObject(); - rhio.userName = config.getServiceUserName(); - rhio.environment = config.getRemoteEnvironment(); + rhio.userName = config.getParam("ServiceUserName"); + rhio.environment = config.getParam("RemoteEnvironment"); rhio.configuration = config.getGeneralProperties(); - - rhio.generativeModel = config.getModel(); - - String jdbcUrl = config.getDatabaseURL(); - String userName = config.getDatabaseUserName(); - String password = config.getDatabasePassword(); - jdbcUrl += ";username="+userName+";password="+password; - - //jdbc:sqlserver://localhost;user=MyUserName;password=*****; - rhio.hcafTableName.tableName = config.getCsquarecodesTable(); - rhio.hcafTableName.jdbcUrl=jdbcUrl; - rhio.hspecDestinationTableName.tableName = config.getDistributionTable(); - rhio.hspecDestinationTableName.jdbcUrl=jdbcUrl; - - rhio.hspenTableName.tableName = config.getEnvelopeTable(); - rhio.hspenTableName.jdbcUrl=jdbcUrl; - - rhio.occurrenceCellsTable.tableName = "maxminlat_"+config.getEnvelopeTable(); - rhio.occurrenceCellsTable.jdbcUrl=jdbcUrl; + rhio.generativeModel = config.getModel(); + + String jdbcUrl = config.getParam("DatabaseURL"); + String userName = config.getParam("DatabaseUserName"); + String password = config.getParam("DatabasePassword"); + jdbcUrl += ";username=" + userName + ";password=" + password; + + // jdbc:sqlserver://localhost;user=MyUserName;password=*****; + rhio.hcafTableName.tableName = config.getParam("CsquarecodesTable"); + rhio.hcafTableName.jdbcUrl = jdbcUrl; + + rhio.hspecDestinationTableName.tableName = config.getParam("DistributionTable"); + rhio.hspecDestinationTableName.jdbcUrl = jdbcUrl; + + rhio.hspenTableName.tableName = config.getParam("EnvelopeTable"); + rhio.hspenTableName.jdbcUrl = jdbcUrl; + + rhio.occurrenceCellsTable.tableName = "maxminlat_" + config.getParam("EnvelopeTable"); + rhio.occurrenceCellsTable.jdbcUrl = jdbcUrl; rhio.nWorkers = config.getNumberOfResources(); - + if (config.getModel().contains("2050")) rhio.is2050 = true; - else + else rhio.is2050 = false; - + if (config.getModel().contains("NATIVE")) rhio.isNativeGeneration = true; else rhio.isNativeGeneration = false; - - //create and call the remote generator - remoteGenerationManager = new RemoteGenerationManager(config.getRemoteCalculator()); + + // create and call the remote generator + remoteGenerationManager = new RemoteGenerationManager(config.getParam("RemoteCalculator")); } - - @Override - public void setConfiguration(GenericConfiguration config) { + public void setConfiguration(AlgorithmConfiguration config) { this.config = config; } @@ -106,29 +104,30 @@ public class RainyCloudGenerator implements Generator{ @Override public String getResourceLoad() { String returnString = "[]"; - - try{ + + try { RemoteHspecOutputObject rhoo = remoteGenerationManager.retrieveCompleteStatus(); - if (rhoo.metrics.throughput.size()>1) - { - ResourceLoad rs = new ResourceLoad(rhoo.metrics.throughput.get(0),rhoo.metrics.throughput.get(1)); + if (rhoo.metrics.throughput.size() > 1) { + ResourceLoad rs = new ResourceLoad(rhoo.metrics.throughput.get(0), rhoo.metrics.throughput.get(1)); returnString = rs.toString(); } - - }catch(Exception e){} + + } catch (Exception e) { + } return returnString; } @Override public String getResources() { Resources res = new Resources(); - try{ + try { RemoteHspecOutputObject rhoo = remoteGenerationManager.retrieveCompleteStatus(); res.list = rhoo.metrics.load; - }catch(Exception e){} - if ((res!=null) && (res.list!=null)) + } catch (Exception e) { + } + if ((res != null) && (res.list != null)) return HttpRequest.toJSon(res.list).replace("resId", "resID"); - else + else return "[]"; } @@ -136,9 +135,8 @@ public class RainyCloudGenerator implements Generator{ public String getLoad() { RemoteHspecOutputObject rhoo = remoteGenerationManager.retrieveCompleteStatus(); String returnString = "[]"; - if ((rhoo.metrics.throughput!=null)&&(rhoo.metrics.throughput.size()>1)) - { - ResourceLoad rs = new ResourceLoad(rhoo.metrics.throughput.get(0),rhoo.metrics.throughput.get(1)); + if ((rhoo.metrics.throughput != null) && (rhoo.metrics.throughput.size() > 1)) { + ResourceLoad rs = new ResourceLoad(rhoo.metrics.throughput.get(0), rhoo.metrics.throughput.get(1)); returnString = rs.toString(); } return returnString; @@ -146,36 +144,52 @@ public class RainyCloudGenerator implements Generator{ @Override public void generate() throws Exception { - - try{ + + try { remoteGenerationManager.submitJob(rhio); - }catch(Exception e){ + } catch (Exception e) { e.printStackTrace(); } - + AnalysisLogger.getLogger().trace("REMOTE PROCESSING STARTED"); boolean finish = false; - - while (!finish && !interruptProcessing){ + + while (!finish && !interruptProcessing) { float status = getStatus(); -// AnalysisLogger.getLogger().trace("Status "+status); - if (status==100) finish = true; + // AnalysisLogger.getLogger().trace("Status "+status); + if (status == 100) + finish = true; Thread.sleep(500); } - + AnalysisLogger.getLogger().trace("REMOTE PROCESSING ENDED"); } - @Override public ALG_PROPS[] getSupportedAlgorithms() { - ALG_PROPS[] p = {ALG_PROPS.SPECIES_VS_CSQUARE_REMOTE_FROM_DATABASE}; + ALG_PROPS[] p = { ALG_PROPS.SPECIES_VS_CSQUARE_REMOTE_FROM_DATABASE }; return p; } - + @Override public GENERATOR_WEIGHT getWeight() { return GENERATOR_WEIGHT.HIGH; } - + + @Override + public HashMap getInputParameters() { + HashMap parameters = new HashMap(); + parameters.put("RemoteEnvironment", ""); + parameters.put("ServiceUserName", ""); + parameters.put("CsquarecodesTable", ""); + parameters.put("DatabaseURL", ""); + parameters.put("DatabaseUserName", ""); + parameters.put("DatabasePassword", ""); + parameters.put("DistributionTable", ""); + parameters.put("EnvelopeTable", ""); + parameters.put("RemoteCalculator", ""); + + return parameters; + } + } diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java b/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java index 1506f70..dbdbeee 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java @@ -8,7 +8,7 @@ import java.util.ServiceLoader; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistribution; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionGeneric; @@ -51,45 +51,65 @@ public class GeneratorsFactory { gens.get(i).init(); } */ + + } + + + public static List getProbabilityDistributionAlgorithms(String configPath) throws Exception{ + + Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.algorithmsFile); + List algs = new ArrayList(); + for (Object algName: p.keySet()){ + algs.add((String)algName); + } + return algs; } - public static void main1(String[] args) throws Exception { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_remote_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); - config.setCreateTable(true); - config.setNumberOfResources(20); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setRemoteCalculator("http://node1.d.venusc.research-infrastructures.eu:5942/api/"); - config.setServiceUserName("gianpaolo.coro"); - config.setRemoteEnvironment("windows azure"); - HashMap properties = new HashMap(); - properties.put("property1", "value1"); - properties.put("property2", "value2"); - - config.setModel("AQUAMAPS_SUITABLE"); - config.setGenerator("LOCAL_WITH_DATABASE"); - - Generator gen = GeneratorsFactory.getGenerator(config); - System.out.println(gen.getClass()); + public static List getAllGenerators(String configPath) throws Exception{ + Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.generatorsFile); + List gens = new ArrayList(); + for (Object genName: p.keySet()){ + gens.add((String)genName); + } + return gens; } + public static List getModels(String configPath) throws Exception{ + + Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.modelsFile); + List models = new ArrayList(); + for (Object modelName: p.keySet()){ + models.add((String)modelName); + } + return models; + } - public static Generator getGenerator(GenericConfiguration config) throws Exception { + public static HashMap getAlgorithmParameters(String configPath, String algorithmName) throws Exception{ + + Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.algorithmsFile); + String algorithmclass = p.getProperty(algorithmName); + Object algclass = Class.forName(algorithmclass).newInstance(); + //if the algorithm is a generator itself then take it + if (algclass instanceof Generator){ + return ((Generator) algclass).getInputParameters(); + } + else + return ((SpatialProbabilityDistribution) algclass).getInputParameters(); + + } + + + public static Generator getGenerator(AlgorithmConfiguration config) throws Exception { //modify this class in order to take the right generator algorithm try { //initialize the logger - AnalysisLogger.setLogger(config.getConfigPath() + GenericConfiguration.defaultLoggerFile); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); //take the algorithm String algorithm = config.getGenerator(); if (algorithm == null) throw new Exception("GENERATOR NOT SPECIFIED"); //take the algorithms list - Properties p = GenericConfiguration.getProperties(config.getConfigPath() + GenericConfiguration.generatorsFile); + Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.generatorsFile); String algorithmclass = p.getProperty(algorithm); Object algclass = Class.forName(algorithmclass).newInstance(); Generator g = (Generator) algclass; @@ -103,17 +123,17 @@ public class GeneratorsFactory { } - public static List getGenerators(GenericConfiguration config) throws Exception { + public static List getGenerators(AlgorithmConfiguration config) throws Exception { //modify this class in order to manage generators weight and match algorithm vs generators List generators = new ArrayList(); try { //initialize the logger - AnalysisLogger.setLogger(config.getConfigPath() + GenericConfiguration.defaultLoggerFile); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); //take the algorithm String algorithm = config.getModel(); //take the algorithms list - Properties p = GenericConfiguration.getProperties(config.getConfigPath() + GenericConfiguration.algorithmsFile); + Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.algorithmsFile); String algorithmclass = p.getProperty(algorithm); Object algclass = Class.forName(algorithmclass).newInstance(); //if the algorithm is a generator itself then execute it @@ -128,7 +148,7 @@ public class GeneratorsFactory { //take alg's properties ALG_PROPS[] algp = sp.getProperties(); //take all generators - Properties pg = GenericConfiguration.getProperties(config.getConfigPath() + GenericConfiguration.generatorsFile); + Properties pg = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.generatorsFile); //investigate on possible suitable generators for (Object generatorName:pg.values()){ Generator gen = (Generator)Class.forName((String)generatorName).newInstance(); diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java b/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java index 6ca3654..538b151 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java @@ -3,16 +3,16 @@ package org.gcube.dataanalysis.ecoengine.processing.factories; import java.util.Properties; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.Model; import org.gcube.dataanalysis.ecoengine.interfaces.Modeler; public class ModelersFactory { - public static Modeler getGenerator(GenericConfiguration config) throws Exception { + public static Modeler getGenerator(AlgorithmConfiguration config) throws Exception { try { - AnalysisLogger.setLogger(config.getConfigPath() + GenericConfiguration.defaultLoggerFile); - Properties p = GenericConfiguration.getProperties(config.getConfigPath() + GenericConfiguration.modelsFile); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.modelsFile); String objectclass = p.getProperty(config.getModel() + "_MODELER"); Modeler g = (Modeler) Class.forName(objectclass).newInstance(); String modelclass = p.getProperty(config.getModel()); diff --git a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNative2050.java b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNative2050.java index f688cb3..11126df 100644 --- a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNative2050.java +++ b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNative2050.java @@ -1,12 +1,12 @@ package org.gcube.dataanalysis.ecoengine.spatialdistributions; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; public class AquamapsNative2050 extends AquamapsNative { - public void init(GenericConfiguration config,SessionFactory dbHibConnection) { + public void init(AlgorithmConfiguration config,SessionFactory dbHibConnection) { super.init(config, dbHibConnection); type = "2050"; } diff --git a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java index 9c0b929..9202ac3 100644 --- a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java +++ b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java @@ -7,7 +7,7 @@ import java.util.Queue; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionTable; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; @@ -29,15 +29,15 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{ protected String type = null; @Override - public void init(GenericConfiguration config,SessionFactory dbHibConnection) { - selectAllSpeciesQuery = String.format(selectAllSpeciesQuery, config.getEnvelopeTable()); - csquareCodeQuery = String.format(csquareCodeQuery, config.getCsquarecodesTable()); - createTableStatement = String.format(createTableStatement,config.getDistributionTable()); - destinationTable = config.getDistributionTable(); + public void init(AlgorithmConfiguration config,SessionFactory dbHibConnection) { + selectAllSpeciesQuery = String.format(selectAllSpeciesQuery, config.getParam("EnvelopeTable")); + csquareCodeQuery = String.format(csquareCodeQuery, config.getParam("CsquarecodesTable")); + createTableStatement = String.format(createTableStatement,config.getParam("DistributionTable")); + destinationTable = config.getParam("DistributionTable"); core = new AquamapsAlgorithmCore(); - if ((config.getPreprocessedTables()!=null)&&(config.getPreprocessedTables().size()>0)) - hspenMinMaxLat = config.getPreprocessedTables().get(0); + if ((config.getParam("PreprocessedTable")!=null)&&(config.getParam("PreprocessedTable").length()>0)) + hspenMinMaxLat = config.getParam("PreprocessedTable"); AnalysisLogger.getLogger().trace("Aquamaps Algorithm Init(->getting min max latitudes from "+hspenMinMaxLat); @@ -186,7 +186,8 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{ @Override public ALG_PROPS[] getProperties() { - ALG_PROPS [] p = {ALG_PROPS.SPECIES_VS_CSQUARE_FROM_DATABASE, ALG_PROPS.SPECIES_VS_CSQUARE_REMOTE_FROM_DATABASE}; +// ALG_PROPS [] p = {ALG_PROPS.SPECIES_VS_CSQUARE_FROM_DATABASE, ALG_PROPS.SPECIES_VS_CSQUARE_REMOTE_FROM_DATABASE}; + ALG_PROPS [] p = {ALG_PROPS.SPECIES_VS_CSQUARE_FROM_DATABASE}; return p; } @@ -200,6 +201,22 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{ return "Algorithm by Aquamaps"; } + @Override + public HashMap getInputParameters() { + HashMap parameters = new HashMap(); + parameters.put("EnvelopeTable", "hspen"); + parameters.put("CsquarecodesTable", "hcaf_d"); + parameters.put("DistributionTable", "hspec_default"); + parameters.put("PreprocessedTable", "maxminlat_hspen"); + parameters.put("CreateTable", "hspen_minmaxlat"); + parameters.put("DatabaseUserName",""); + parameters.put("DatabasePassword",""); + parameters.put("DatabaseURL",""); + parameters.put("DatabaseDriver",""); + + return parameters; + } + diff --git a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable2050.java b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable2050.java index 396db0c..516c2cd 100644 --- a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable2050.java +++ b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable2050.java @@ -1,12 +1,12 @@ package org.gcube.dataanalysis.ecoengine.spatialdistributions; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; public class AquamapsSuitable2050 extends AquamapsSuitable { - public void init(GenericConfiguration config,SessionFactory dbHibConnection) { + public void init(AlgorithmConfiguration config,SessionFactory dbHibConnection) { super.init(config, dbHibConnection); type = "2050"; } diff --git a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java index ac22e34..69d9de5 100644 --- a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java +++ b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java @@ -4,13 +4,14 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionGeneric; public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ @@ -58,7 +59,7 @@ public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ } @Override - public void init(GenericConfiguration config) { + public void init(AlgorithmConfiguration config) { AnalysisLogger.getLogger().trace("Dummy INIT"); randomElements = new ArrayList(); for (int i=0;i<170000;i++) @@ -187,4 +188,9 @@ public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ return null; } + @Override + public HashMap getInputParameters() { + return null; + } + } diff --git a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java index a79b9f9..ef8971f 100644 --- a/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java +++ b/src/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java @@ -4,13 +4,14 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionGeneric; public class TestAlgorithm implements SpatialProbabilityDistributionGeneric{ @@ -23,7 +24,7 @@ public class TestAlgorithm implements SpatialProbabilityDistributionGeneric{ String pers; @Override - public void init(GenericConfiguration config) { + public void init(AlgorithmConfiguration config) { pers = config.getPersistencePath(); } @@ -134,5 +135,11 @@ public class TestAlgorithm implements SpatialProbabilityDistributionGeneric{ return null; } + @Override + public HashMap getInputParameters() { + // TODO Auto-generated method stub + return null; + } + } diff --git a/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGeneration.java b/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGeneration.java deleted file mode 100644 index 64409b2..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGeneration.java +++ /dev/null @@ -1,164 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test; - -import java.util.HashMap; - -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.interfaces.Generator; -import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; - -public class RegressionTestsGeneration { - /** - * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species - * - */ - - public static void main(String[] args) throws Exception { - -// System.out.println("TEST 1"); -// Generator generator = GeneratorsFactory.getGenerator(testConfigRemote()); -// generate(generator); - System.out.println("TEST 2"); - Generator generator = GeneratorsFactory.getGenerator(testConfigSuitable()); - generate(generator); - - /* - System.out.println("TEST 3"); - generator = GeneratorsFactory.getGenerator(testConfigNative()); - generate(generator); - System.out.println("TEST 4"); - generator = GeneratorsFactory.getGenerator(testConfigSuitable2050()); - generate(generator); - System.out.println("TEST 5"); - generator = GeneratorsFactory.getGenerator(testConfigNative2050()); - generate(generator); - */ - - } - - - private static void generate(Generator generator) throws Exception { - - if (generator != null) { - RegressionTestsGeneration tgs = new RegressionTestsGeneration(); - ThreadCalculator tc = tgs.new ThreadCalculator(generator); - Thread t = new Thread(tc); - t.start(); - while (generator.getStatus() < 100) { - - String resLoad = generator.getResourceLoad(); - String ress = generator.getResources(); - String species = generator.getLoad(); - System.out.println("LOAD: " + resLoad); - System.out.println("RESOURCES: " + ress); - System.out.println("SPECIES: " + species); - System.out.println("STATUS: " + generator.getStatus()); - Thread.sleep(1000); - } - } else - AnalysisLogger.getLogger().trace("Generator Algorithm Not Supported"); - - } - - public class ThreadCalculator implements Runnable { - Generator dg; - - public ThreadCalculator(Generator dg) { - this.dg = dg; - } - - public void run() { - try { - - dg.generate(); - - } catch (Exception e) { - } - } - - } - - private static GenericConfiguration testConfigRemote() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_remote_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); - config.setCreateTable(true); - config.setNumberOfResources(20); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://146.48.87.169/testdb"); - config.setModel("REMOTE_AQUAMAPS_SUITABLE"); - config.setRemoteCalculator("http://node1.d.venusc.research-infrastructures.eu:5942/api/"); - config.setServiceUserName("gianpaolo.coro"); - config.setRemoteEnvironment("windows azure"); - HashMap properties = new HashMap(); - properties.put("property1", "value1"); - properties.put("property2", "value2"); - config.setGeneralProperties(properties); - return config; - } - - private static GenericConfiguration testConfigSuitable() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_SUITABLE"); - config.setGenerator("LOCAL_WITH_DATABASE"); - return config; - } - - private static GenericConfiguration testConfigNative() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_native_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_NATIVE"); - return config; - } - - - private static GenericConfiguration testConfigNative2050() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_native_2050_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_NATIVE_2050"); - return config; - } - - private static GenericConfiguration testConfigSuitable2050() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_2050_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_SUITABLE_2050"); - return config; - } -} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGenerationList.java b/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGenerationList.java deleted file mode 100644 index 0915372..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsGenerationList.java +++ /dev/null @@ -1,174 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test; - -import java.util.HashMap; -import java.util.List; - -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.interfaces.Generator; -import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; - -public class RegressionTestsGenerationList { - /** - * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species - * - */ - - public static void main(String[] args) throws Exception { - -// System.out.println("TEST 1"); -// Generator generator = GeneratorsFactory.getGenerator(testConfigRemote()); -// generate(generator); - - System.out.println("TEST 2"); - List generators = GeneratorsFactory.getGenerators(testConfigSuitable()); - Generator generator = generators.get(0); - generator.init(); - generate(generator); - - /* - System.out.println("TEST 3"); - generator = GeneratorsFactory.getGenerator(testConfigNative()); - generate(generator); - System.out.println("TEST 4"); - generator = GeneratorsFactory.getGenerator(testConfigSuitable2050()); - generate(generator); - System.out.println("TEST 5"); - generator = GeneratorsFactory.getGenerator(testConfigNative2050()); - generate(generator); - */ - - } - - - private static void generate(Generator generator) throws Exception { - - if (generator != null) { - RegressionTestsGenerationList tgs = new RegressionTestsGenerationList(); - ThreadCalculator tc = tgs.new ThreadCalculator(generator); - Thread t = new Thread(tc); - t.start(); - while (generator.getStatus() < 100) { - - String resLoad = generator.getResourceLoad(); - String ress = generator.getResources(); - String species = generator.getLoad(); - System.out.println("LOAD: " + resLoad); - System.out.println("RESOURCES: " + ress); - System.out.println("SPECIES: " + species); - System.out.println("STATUS: " + generator.getStatus()); - Thread.sleep(1000); - } - } else - AnalysisLogger.getLogger().trace("Generator Algorithm Not Supported"); - - } - - public class ThreadCalculator implements Runnable { - Generator dg; - - public ThreadCalculator(Generator dg) { - this.dg = dg; - } - - public void run() { - try { - - dg.generate(); - - } catch (Exception e) { - } - } - - } - - private static GenericConfiguration testConfigRemote() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_remote_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); - config.setCreateTable(true); - config.setNumberOfResources(20); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://146.48.87.169/testdb"); - config.setModel("REMOTE_AQUAMAPS_SUITABLE"); - config.setRemoteCalculator("http://node1.d.venusc.research-infrastructures.eu:5942/api/"); - config.setServiceUserName("gianpaolo.coro"); - config.setRemoteEnvironment("windows azure"); - HashMap properties = new HashMap(); - properties.put("property1", "value1"); - properties.put("property2", "value2"); - config.setGeneralProperties(properties); - return config; - } - - private static GenericConfiguration testConfigSuitable() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_micro"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://146.48.87.169/testdb"); - config.setRemoteCalculator("http://node1.d.venusc.research-infrastructures.eu:5942/api/"); - config.setServiceUserName("gianpaolo.coro"); - config.setRemoteEnvironment("windows azure"); - HashMap properties = new HashMap(); - properties.put("property1", "value1"); - properties.put("property2", "value2"); - config.setModel("AQUAMAPS_SUITABLE"); - - return config; - } - - private static GenericConfiguration testConfigNative() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_native_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_NATIVE"); - return config; - } - - - private static GenericConfiguration testConfigNative2050() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_native_2050_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_NATIVE_2050"); - return config; - } - - private static GenericConfiguration testConfigSuitable2050() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_2050_test"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_SUITABLE_2050"); - return config; - } -} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsModeling.java b/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsModeling.java deleted file mode 100644 index 6325f0f..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/RegressionTestsModeling.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test; - -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.interfaces.Modeler; -import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory; - -public class RegressionTestsModeling { - /** - * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species - * - */ - - public static void main(String[] args) throws Exception { - - System.out.println("TEST 1"); - - GenericConfiguration modelconfig = new GenericConfiguration(); - modelconfig.setConfigPath("./cfg/"); - modelconfig.setModel("HSPEN"); - Modeler modeler = ModelersFactory.getGenerator(modelconfig); - Object input = testInputConfig(); - Object output = testOutputConfig(); -// modeler.model(input, null, output); - generate(modeler, input, output); - } - - - private static void generate(Modeler modeler, Object input , Object output) throws Exception { - - if (modeler != null) { - RegressionTestsModeling tgs = new RegressionTestsModeling(); - ThreadCalculator tc = tgs.new ThreadCalculator(modeler,input,output); - Thread t = new Thread(tc); - t.start(); - while (modeler.getStatus() < 100) { - - String resLoad = modeler.getResourceLoad(); - String ress = modeler.getResources(); - System.out.println("LOAD: " + resLoad); - System.out.println("RESOURCES: " + ress); - - Thread.sleep(1000); - } - } else - AnalysisLogger.getLogger().trace("Modeling Algorithm Not Supported"); - - } - - public class ThreadCalculator implements Runnable { - Modeler mg; - Object input; - Object output; - - public ThreadCalculator(Modeler modeler, Object input ,Object output) { - this.mg = modeler; - this.input=input; - this.output=output; - } - - public void run() { - try { - - mg.model(input,null, output); - - } catch (Exception e) { - } - } - - } - - private static GenericConfiguration testInputConfig() { - GenericConfiguration config = new GenericConfiguration(); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); - config.setOccurrenceCellsTable("occurrencecells"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - return config; - } - - - private static GenericConfiguration testOutputConfig() { - GenericConfiguration config = new GenericConfiguration(); - config.setEnvelopeTable("hspen_validation_refactored"); - return config; - } - - } diff --git a/src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java b/src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java new file mode 100644 index 0000000..0ad4b2a --- /dev/null +++ b/src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java @@ -0,0 +1,92 @@ +package org.gcube.dataanalysis.ecoengine.test; + +import java.util.HashMap; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.Generator; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; + +public class TestsMetaInfo { + /** + * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species + * + */ + +public static void main(String[] args) throws Exception { + + System.out.println("***TEST 1 - Get Algorithm Information***"); + HashMap map = GeneratorsFactory.getAlgorithmParameters("./cfg/","DUMMY"); + System.out.println("input for DUMMY algorithm: "+map); + map = GeneratorsFactory.getAlgorithmParameters("./cfg/","AQUAMAPS_SUITABLE"); + System.out.println("input for AQUAMAPS_SUITABLE algorithm: "+map); + + System.out.println("\n***TEST 2 - Get Single Generator***"); + Generator g = GeneratorsFactory.getGenerator(testConfig()); + System.out.println("Found generator "+g); + + System.out.println("\n***TEST 3 - Get All Algorithms ***"); + System.out.println("Algs: "+GeneratorsFactory.getProbabilityDistributionAlgorithms("./cfg/")); + + System.out.println("\n***TEST 4 - Get All Generators ***"); + System.out.println("Gens: "+GeneratorsFactory.getAllGenerators("./cfg/")); + + System.out.println("\n***TEST 5 - Get All Models to be trained ***"); + System.out.println("Models: "+GeneratorsFactory.getModels("./cfg/")); +} + + + private static void generate(Generator generator) throws Exception { + + if (generator != null) { + TestsMetaInfo tgs = new TestsMetaInfo(); + ThreadCalculator tc = tgs.new ThreadCalculator(generator); + Thread t = new Thread(tc); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(1000); + } + } else + AnalysisLogger.getLogger().trace("Generator Algorithm Not Supported"); + + } + + public class ThreadCalculator implements Runnable { + Generator dg; + + public ThreadCalculator(Generator dg) { + this.dg = dg; + } + + public void run() { + try { + + dg.generate(); + + } catch (Exception e) { + } + } + + } + + + private static AlgorithmConfiguration testConfig() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(2); + config.setModel("TEST"); + config.setGenerator("SIMPLE_LOCAL"); + return config; + } + +} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/TestsTESTGeneration.java b/src/org/gcube/dataanalysis/ecoengine/test/TestsTESTGeneration.java index 56392db..01bd34f 100644 --- a/src/org/gcube/dataanalysis/ecoengine/test/TestsTESTGeneration.java +++ b/src/org/gcube/dataanalysis/ecoengine/test/TestsTESTGeneration.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.ecoengine.test; import java.util.List; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; @@ -16,7 +16,7 @@ public class TestsTESTGeneration { public static void main(String[] args) throws Exception { System.out.println("TEST 1"); - List generators = GeneratorsFactory.getGenerators(testConfigSuitable()); + List generators = GeneratorsFactory.getGenerators(testConfig()); generators.get(0).init(); generate(generators.get(0)); generators = null; @@ -65,8 +65,8 @@ public static void main(String[] args) throws Exception { } - private static GenericConfiguration testConfigSuitable() { - GenericConfiguration config = new GenericConfiguration(); + private static AlgorithmConfiguration testConfig() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); config.setConfigPath("./cfg/"); config.setPersistencePath("./"); config.setNumberOfResources(2); diff --git a/src/org/gcube/dataanalysis/ecoengine/test/generations/GenerationHSPECValidation.java b/src/org/gcube/dataanalysis/ecoengine/test/generations/GenerationHSPECValidation.java index a432208..6a07acc 100644 --- a/src/org/gcube/dataanalysis/ecoengine/test/generations/GenerationHSPECValidation.java +++ b/src/org/gcube/dataanalysis/ecoengine/test/generations/GenerationHSPECValidation.java @@ -1,9 +1,7 @@ package org.gcube.dataanalysis.ecoengine.test.generations; -import java.util.HashMap; - import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; @@ -64,18 +62,20 @@ public class GenerationHSPECValidation { } - private static GenericConfiguration testConfigSuitable() { - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_validation_automatic"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); - config.setCreateTable(true); + private static AlgorithmConfiguration testConfigSuitable() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setParam("DistributionTable","hspec_validation_automatic"); + config.setParam("ConfigPath","./cfg/"); + config.setParam("CsquarecodesTable","hcaf_d"); + config.setParam("EnvelopeTable","hspen_validation"); + config.setParam("CreateTable","true"); config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); config.setModel("AQUAMAPS_SUITABLE"); + return config; } diff --git a/src/org/gcube/dataanalysis/ecoengine/test/regressions/RegressionComplexGeneration.java b/src/org/gcube/dataanalysis/ecoengine/test/regressions/RegressionComplexGeneration.java new file mode 100644 index 0000000..021d597 --- /dev/null +++ b/src/org/gcube/dataanalysis/ecoengine/test/regressions/RegressionComplexGeneration.java @@ -0,0 +1,129 @@ +package org.gcube.dataanalysis.ecoengine.test.regressions; + +import java.util.HashMap; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.Generator; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; + +public class RegressionComplexGeneration { + /** + * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species + * + */ + +public static void main(String[] args) throws Exception { + + System.out.println("TEST 1"); + List generators = GeneratorsFactory.getGenerators(testConfigRemote()); + generators.get(0).init(); + generate(generators.get(0)); + generators = null; + + System.out.println("TEST 2"); + generators = GeneratorsFactory.getGenerators(testConfigLocal()); + generators.get(0).init(); + generate(generators.get(0)); + generators = null; + +} + + + private static void generate(Generator generator) throws Exception { + + if (generator != null) { + RegressionComplexGeneration tgs = new RegressionComplexGeneration(); + ThreadCalculator tc = tgs.new ThreadCalculator(generator); + Thread t = new Thread(tc); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(1000); + } + } else + AnalysisLogger.getLogger().trace("Generator Algorithm Not Supported"); + + } + + public class ThreadCalculator implements Runnable { + Generator dg; + + public ThreadCalculator(Generator dg) { + this.dg = dg; + } + + public void run() { + try { + + dg.generate(); + + } catch (Exception e) { + } + } + + } + + + private static AlgorithmConfiguration testConfigRemote() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(5); + config.setModel("REMOTE_AQUAMAPS_SUITABLE"); + + config.setParam("DistributionTable","hspec_suitable_remote_test"); + config.setParam("CsquarecodesTable","hcaf_d"); + config.setParam("EnvelopeTable","hspen_micro"); + config.setParam("CreateTable","true"); + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://146.48.87.169/testdb"); + config.setParam("DatabaseDriver","org.postgresql.Driver"); + config.setParam("RemoteCalculator","http://node1.d.venusc.research-infrastructures.eu:5942/api/"); + config.setParam("ServiceUserName","gianpaolo.coro"); + config.setParam("RemoteEnvironment","windows azure"); + HashMap properties = new HashMap(); + properties.put("property1", "value1"); + properties.put("property2", "value2"); + config.addGeneralProperties(properties); + + return config; + } + + + private static AlgorithmConfiguration testConfigLocal() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(5); + config.setModel("AQUAMAPS_SUITABLE"); + + config.setParam("DistributionTable","hspec_suitable_test"); + config.setParam("CsquarecodesTable","hcaf_d"); + config.setParam("EnvelopeTable","hspen_micro"); + config.setParam("PreprocessedTable", "maxminlat_hspen"); + config.setParam("CreateTable","true"); + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); + config.setParam("DatabaseDriver","org.postgresql.Driver"); + + HashMap properties = new HashMap(); + properties.put("property1", "value1"); + properties.put("property2", "value2"); + config.addGeneralProperties(properties); + + return config; + } +} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/TestsDummyGeneration.java b/src/org/gcube/dataanalysis/ecoengine/test/regressions/RegressionSimpleGeneration.java similarity index 57% rename from src/org/gcube/dataanalysis/ecoengine/test/TestsDummyGeneration.java rename to src/org/gcube/dataanalysis/ecoengine/test/regressions/RegressionSimpleGeneration.java index 8fc7246..2af7bfd 100644 --- a/src/org/gcube/dataanalysis/ecoengine/test/TestsDummyGeneration.java +++ b/src/org/gcube/dataanalysis/ecoengine/test/regressions/RegressionSimpleGeneration.java @@ -1,11 +1,13 @@ -package org.gcube.dataanalysis.ecoengine.test; +package org.gcube.dataanalysis.ecoengine.test.regressions; + +import java.util.List; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; -public class TestsDummyGeneration { +public class RegressionSimpleGeneration { /** * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species * @@ -14,16 +16,24 @@ public class TestsDummyGeneration { public static void main(String[] args) throws Exception { System.out.println("TEST 1"); - Generator generator = GeneratorsFactory.getGenerator(testConfigSuitable()); - generate(generator); - + List generators = GeneratorsFactory.getGenerators(testConfig1()); + generators.get(0).init(); + generate(generators.get(0)); + generators = null; + + System.out.println("TEST 2"); + generators = GeneratorsFactory.getGenerators(testConfig2()); + generators.get(0).init(); + generate(generators.get(0)); + generators = null; + } private static void generate(Generator generator) throws Exception { if (generator != null) { - TestsDummyGeneration tgs = new TestsDummyGeneration(); + RegressionSimpleGeneration tgs = new RegressionSimpleGeneration(); ThreadCalculator tc = tgs.new ThreadCalculator(generator); Thread t = new Thread(tc); t.start(); @@ -62,13 +72,22 @@ public static void main(String[] args) throws Exception { } - private static GenericConfiguration testConfigSuitable() { - GenericConfiguration config = new GenericConfiguration(); + private static AlgorithmConfiguration testConfig1() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); config.setConfigPath("./cfg/"); config.setPersistencePath("./"); - config.setNumberOfResources(2); - config.setModel("DUMMY"); + config.setNumberOfResources(5); + config.setModel("TEST"); return config; } + + private static AlgorithmConfiguration testConfig2() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(5); + config.setModel("DUMMY"); + return config; + } } diff --git a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestLocalGenerationStatus.java b/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestLocalGenerationStatus.java deleted file mode 100644 index 36070d7..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestLocalGenerationStatus.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test.regressions1; - -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.processing.LocalSplitGenerator; - -public class RegressionTestLocalGenerationStatus { - /** - * example of parallel processing on a single machine - * the procedure will generate a new table for a distribution on suitable species - * - */ - - public static void main(String[] args) throws Exception{ - - RegressionTestLocalGenerationStatus tgs = new RegressionTestLocalGenerationStatus(); - GenericConfiguration config = new GenericConfiguration(); - - //path to the cfg directory containing default parameters - config.setDistributionTable("hspec_suitable_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); -// config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_SUITABLE"); - - LocalSplitGenerator generator = new LocalSplitGenerator(config); - - ThreadCalculator tc = tgs.new ThreadCalculator(generator); - - Thread t = new Thread(tc); - t.start(); - - while (generator.getStatus()<100){ - - String resLoad = generator.getResourceLoad(); - String ress = generator.getResources(); - String species = generator.getLoad(); - System.out.println("LOAD: "+resLoad); - System.out.println("RESOURCES: "+ress); - System.out.println("SPECIES: "+species); - System.out.println("STATUS: "+generator.getStatus()); - Thread.sleep(30000); - } - - } - - - - public class ThreadCalculator implements Runnable { - LocalSplitGenerator dg ; - public ThreadCalculator(LocalSplitGenerator dg) { - this.dg = dg; - } - - public void run() { - try{ - - dg.generate(); - - }catch(Exception e){} - } - - } -} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNative2050LocalGenerationStatus.java b/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNative2050LocalGenerationStatus.java deleted file mode 100644 index 60c8986..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNative2050LocalGenerationStatus.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test.regressions1; - -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.processing.LocalSplitGenerator; - -public class RegressionTestNative2050LocalGenerationStatus { - /** - * example of parallel processing on a single machine - * the procedure will generate a new table for a distribution on native species - * - */ - - public static void main(String[] args) throws Exception{ - - RegressionTestNative2050LocalGenerationStatus tgs = new RegressionTestNative2050LocalGenerationStatus(); - GenericConfiguration config = new GenericConfiguration(); - - //path to the cfg directory containing default parameters - config.setDistributionTable("hspec_native_2050_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); -// config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_NATIVE_2050"); - - LocalSplitGenerator generator = new LocalSplitGenerator(config); - - ThreadCalculator tc = tgs.new ThreadCalculator(generator); - - Thread t = new Thread(tc); - t.start(); - - while (generator.getStatus()<100){ - - String resLoad = generator.getResourceLoad(); - String ress = generator.getResources(); - String species = generator.getLoad(); - System.out.println("LOAD: "+resLoad); - System.out.println("RESOURCES: "+ress); - System.out.println("SPECIES: "+species); - System.out.println("STATUS: "+generator.getStatus()); - Thread.sleep(30000); - } - - } - - - - public class ThreadCalculator implements Runnable { - LocalSplitGenerator dg ; - public ThreadCalculator(LocalSplitGenerator dg) { - this.dg = dg; - } - - public void run() { - try{ - - dg.generate(); - - }catch(Exception e){} - } - - } -} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNativeLocalGenerationStatus.java b/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNativeLocalGenerationStatus.java deleted file mode 100644 index 6eb556b..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestNativeLocalGenerationStatus.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test.regressions1; - -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.processing.LocalSplitGenerator; - -public class RegressionTestNativeLocalGenerationStatus { - /** - * example of parallel processing on a single machine - * the procedure will generate a new table for a distribution on native species - * - */ - - public static void main(String[] args) throws Exception{ - - RegressionTestNativeLocalGenerationStatus tgs = new RegressionTestNativeLocalGenerationStatus(); - GenericConfiguration config = new GenericConfiguration(); - - //path to the cfg directory containing default parameters - config.setDistributionTable("hspec_native_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); -// config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_NATIVE"); - - LocalSplitGenerator generator = new LocalSplitGenerator(config); - - ThreadCalculator tc = tgs.new ThreadCalculator(generator); - - Thread t = new Thread(tc); - t.start(); - - while (generator.getStatus()<100){ - - String resLoad = generator.getResourceLoad(); - String ress = generator.getResources(); - String species = generator.getLoad(); - System.out.println("LOAD: "+resLoad); - System.out.println("RESOURCES: "+ress); - System.out.println("SPECIES: "+species); - System.out.println("STATUS: "+generator.getStatus()); - Thread.sleep(30000); - } - - } - - - - public class ThreadCalculator implements Runnable { - LocalSplitGenerator dg ; - public ThreadCalculator(LocalSplitGenerator dg) { - this.dg = dg; - } - - public void run() { - try{ - - dg.generate(); - - }catch(Exception e){} - } - - } -} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestRemoteGenerationStatus.java b/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestRemoteGenerationStatus.java deleted file mode 100644 index 85c6ec5..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestRemoteGenerationStatus.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test.regressions1; - -import java.util.HashMap; - -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator; - -public class RegressionTestRemoteGenerationStatus { - /** - * example of parallel processing on a single machine - * the procedure will generate a new table for a distribution on suitable species - * - */ - - public static void main(String[] args) throws Exception{ - - RegressionTestRemoteGenerationStatus tgs = new RegressionTestRemoteGenerationStatus(); - GenericConfiguration config = new GenericConfiguration(); - - //path to the cfg directory containing default parameters - config.setDistributionTable("hspec_suitable_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); -// config.setEnvelopeTable("hspen_mini"); - - config.setCreateTable(true); - config.setNumberOfResources(20); - - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://146.48.87.169/testdb"); - config.setModel("REMOTE_AQUAMAPS_SUITABLE"); - - config.setRemoteCalculator("http://node1.d.venusc.research-infrastructures.eu:5942/api/"); - config.setServiceUserName("gianpaolo.coro"); - - //new parameters - config.setRemoteEnvironment("windows azure"); - HashMap properties = new HashMap(); - properties.put("property1", "value1"); - properties.put("property2", "value2"); - config.setGeneralProperties(properties); - - RainyCloudGenerator generator = new RainyCloudGenerator(config); - - ThreadCalculator tc = tgs.new ThreadCalculator(generator); - - Thread t = new Thread(tc); - t.start(); - - while (generator.getStatus()<100){ - - String resLoad = generator.getResourceLoad(); - String ress = generator.getResources(); - String species = generator.getLoad(); - System.out.println("LOAD: "+resLoad); - System.out.println("RESOURCES: "+ress); - System.out.println("SPECIES: "+species); - System.out.println("STATUS: "+generator.getStatus()); - Thread.sleep(1000); - } - - } - - - - public class ThreadCalculator implements Runnable { - RainyCloudGenerator dg ; - public ThreadCalculator(RainyCloudGenerator dg) { - this.dg = dg; - } - - public void run() { - try{ - - dg.generate(); - - }catch(Exception e){} - } - - } -} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestSuitable2050LocalGenerationStatus.java b/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestSuitable2050LocalGenerationStatus.java deleted file mode 100644 index fead0b4..0000000 --- a/src/org/gcube/dataanalysis/ecoengine/test/regressions1/RegressionTestSuitable2050LocalGenerationStatus.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.gcube.dataanalysis.ecoengine.test.regressions1; - -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.processing.LocalSplitGenerator; - -public class RegressionTestSuitable2050LocalGenerationStatus { - /** - * example of parallel processing on a single machine - * the procedure will generate a new table for a distribution on native species - * - */ - - public static void main(String[] args) throws Exception{ - - RegressionTestSuitable2050LocalGenerationStatus tgs = new RegressionTestSuitable2050LocalGenerationStatus(); - GenericConfiguration config = new GenericConfiguration(); - - //path to the cfg directory containing default parameters - config.setDistributionTable("hspec_suitable_2050_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_validation"); -// config.setEnvelopeTable("hspen_mini"); - config.setCreateTable(true); - config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - config.setModel("AQUAMAPS_SUITABLE_2050"); - - LocalSplitGenerator generator = new LocalSplitGenerator(config); - - ThreadCalculator tc = tgs.new ThreadCalculator(generator); - - Thread t = new Thread(tc); - t.start(); - - while (generator.getStatus()<100){ - - String resLoad = generator.getResourceLoad(); - String ress = generator.getResources(); - String species = generator.getLoad(); - System.out.println("LOAD: "+resLoad); - System.out.println("RESOURCES: "+ress); - System.out.println("SPECIES: "+species); - System.out.println("STATUS: "+generator.getStatus()); - Thread.sleep(30000); - } - - } - - - - public class ThreadCalculator implements Runnable { - LocalSplitGenerator dg ; - public ThreadCalculator(LocalSplitGenerator dg) { - this.dg = dg; - } - - public void run() { - try{ - - dg.generate(); - - }catch(Exception e){} - } - - } -} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareHspecSingle.java b/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareHspecSingle.java index 97288d0..1b6bf41 100644 --- a/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareHspecSingle.java +++ b/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareHspecSingle.java @@ -1,7 +1,7 @@ package org.gcube.dataanalysis.ecoengine.test.tables; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; -import org.gcube.dataanalysis.ecoengine.utils.HpecDiscrepanciesCalculator; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.utils.HspecDiscrepanciesCalculator; public class CompareHspecSingle { @@ -9,19 +9,19 @@ public class CompareHspecSingle { String configPath = "./cfg/"; - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_automatic_local"); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setParam("DistributionTable","hspec_suitable_automatic_local"); config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_micro"); + config.setParam("CsquarecodesTable","hcaf_d"); + config.setParam("EnvelopeTable","hspen_micro"); // config.setEnvelopeTable("hspen"); - config.setCreateTable(true); + config.setParam("CreateTable","true"); config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); - HpecDiscrepanciesCalculator ec = new HpecDiscrepanciesCalculator(config); + HspecDiscrepanciesCalculator ec = new HspecDiscrepanciesCalculator(config); ec.referenceTable = "hspec_validation where probability>0.19"; ec.analyzedTable = "hspec_suitable_automatic_local"; diff --git a/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareTables.java b/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareTables.java index 15d7d08..45f45c3 100644 --- a/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareTables.java +++ b/src/org/gcube/dataanalysis/ecoengine/test/tables/CompareTables.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.ecoengine.test.tables; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.utils.TablesDiscrepanciesCalculator; public class CompareTables { @@ -9,18 +9,17 @@ public class CompareTables { String configPath = "./cfg/"; - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_micro"); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setParam("DistributionTable","hspec_suitable_automatic_local"); + config.setParam("ConfigPath","./cfg/"); + config.setParam("CsquarecodesTable","hcaf_d"); + config.setParam("EnvelopeTable","hspen_micro"); // config.setEnvelopeTable("hspen"); - config.setCreateTable(true); + config.setParam("CreateTable","true"); config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); TablesDiscrepanciesCalculator ec = new TablesDiscrepanciesCalculator(config); ec.referenceTable = "hspec_suitable_automatic_local"; diff --git a/src/org/gcube/dataanalysis/ecoengine/utils/HpecDiscrepanciesCalculator.java b/src/org/gcube/dataanalysis/ecoengine/utils/HspecDiscrepanciesCalculator.java similarity index 86% rename from src/org/gcube/dataanalysis/ecoengine/utils/HpecDiscrepanciesCalculator.java rename to src/org/gcube/dataanalysis/ecoengine/utils/HspecDiscrepanciesCalculator.java index e922d4e..2f358d4 100644 --- a/src/org/gcube/dataanalysis/ecoengine/utils/HpecDiscrepanciesCalculator.java +++ b/src/org/gcube/dataanalysis/ecoengine/utils/HspecDiscrepanciesCalculator.java @@ -4,7 +4,7 @@ import java.math.BigInteger; import java.util.List; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; @@ -12,7 +12,7 @@ import org.hibernate.SessionFactory; * checks if two tables are equal * checks numbers at the second decimal position */ -public class HpecDiscrepanciesCalculator { +public class HspecDiscrepanciesCalculator { private BigInteger numOfElements; private int errorCounter; @@ -41,11 +41,11 @@ public class HpecDiscrepanciesCalculator { //init connections - public HpecDiscrepanciesCalculator(GenericConfiguration config) throws Exception { + public HspecDiscrepanciesCalculator(AlgorithmConfiguration config) throws Exception { AnalysisLogger.setLogger(config.getConfigPath() + LogFile); - referencedbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + GenericConfiguration.defaultConnectionFile,config); + referencedbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); AnalysisLogger.getLogger().debug("ReferenceDB initialized"); - destinationdbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + GenericConfiguration.defaultConnectionFile,config); + destinationdbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); AnalysisLogger.getLogger().debug("OriginalDB initialized"); } @@ -90,20 +90,20 @@ public class HpecDiscrepanciesCalculator { public static void main(String[] args) throws Exception { String configPath = "./cfg/"; - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_micro"); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setParam("DistributionTable","hspec_suitable_automatic_local"); + config.setParam("ConfigPath","./cfg/"); + config.setParam("CsquarecodesTable","hcaf_d"); + config.setParam("EnvelopeTable","hspen_micro"); // config.setEnvelopeTable("hspen"); - config.setCreateTable(true); + config.setParam("CreateTable","true"); config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); - HpecDiscrepanciesCalculator ec = new HpecDiscrepanciesCalculator(config); + HspecDiscrepanciesCalculator ec = new HspecDiscrepanciesCalculator(config); long t0 = System.currentTimeMillis(); ec.runTest(); long t1 = System.currentTimeMillis(); diff --git a/src/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java b/src/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java index a1d402a..d4e9253 100644 --- a/src/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java +++ b/src/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java @@ -4,7 +4,7 @@ import java.math.BigInteger; import java.util.List; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.GenericConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; @@ -40,11 +40,11 @@ public class TablesDiscrepanciesCalculator { //init connections - public TablesDiscrepanciesCalculator(GenericConfiguration config) throws Exception { + public TablesDiscrepanciesCalculator(AlgorithmConfiguration config) throws Exception { AnalysisLogger.setLogger(config.getConfigPath() + LogFile); - referencedbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + GenericConfiguration.defaultConnectionFile,config); + referencedbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); AnalysisLogger.getLogger().debug("ReferenceDB initialized"); - destinationdbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + GenericConfiguration.defaultConnectionFile,config); + destinationdbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); AnalysisLogger.getLogger().debug("OriginalDB initialized"); } @@ -81,17 +81,17 @@ public class TablesDiscrepanciesCalculator { public static void main(String[] args) throws Exception { String configPath = "./cfg/"; - GenericConfiguration config = new GenericConfiguration(); - config.setDistributionTable("hspec_suitable_automatic_local"); - config.setConfigPath("./cfg/"); - config.setCsquarecodesTable("hcaf_d"); - config.setEnvelopeTable("hspen_micro"); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setParam("DistributionTable","hspec_suitable_automatic_local"); + config.setParam("ConfigPath","./cfg/"); + config.setParam("CsquarecodesTable","hcaf_d"); + config.setParam("EnvelopeTable","hspen_micro"); // config.setEnvelopeTable("hspen"); - config.setCreateTable(true); + config.setParam("CreateTable","true"); config.setNumberOfResources(2); - config.setDatabaseUserName("gcube"); - config.setDatabasePassword("d4science2"); - config.setDatabaseURL("jdbc:postgresql://localhost/testdb"); + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); TablesDiscrepanciesCalculator ec = new TablesDiscrepanciesCalculator(config); long t0 = System.currentTimeMillis();