From ed7f25af8b81206316d4af787bb62884546ed6d6 Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Fri, 23 Dec 2011 13:21:12 +0000 Subject: [PATCH] added models and modelers management merry Christmas git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@49603 82a268e6-3cf1-43bd-a215-b396298e98cf --- cfg/modelers.properties | 1 + cfg/models.properties | 1 - .../ecoengine/configuration/ALG_PROPS.java | 3 +- .../configuration/AlgorithmConfiguration.java | 1 + .../{GENERATOR_WEIGHT.java => WEIGHT.java} | 2 +- .../ecoengine/interfaces/Generator.java | 4 +- .../ecoengine/interfaces/Model.java | 26 ++++- .../ecoengine/interfaces/Modeler.java | 17 +++- .../ecoengine/modeling/SimpleModeler.java | 29 +++++- .../ecoengine/models/ModelHSPEN.java | 69 +++++++++---- .../processing/LocalSimpleSplitGenerator.java | 6 +- .../processing/LocalSplitGenerator.java | 6 +- .../processing/RainyCloudGenerator.java | 6 +- .../factories/GeneratorsFactory.java | 59 ++--------- .../processing/factories/ModelersFactory.java | 97 +++++++++++++++++-- .../factories/ProcessorsFactory.java | 80 +++++++++++++++ .../ecoengine/test/TestsHSPENTraining.java | 90 +++++++++++++++++ .../ecoengine/test/TestsMetaInfo.java | 11 ++- 18 files changed, 405 insertions(+), 103 deletions(-) create mode 100644 cfg/modelers.properties rename src/org/gcube/dataanalysis/ecoengine/configuration/{GENERATOR_WEIGHT.java => WEIGHT.java} (75%) create mode 100644 src/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java create mode 100644 src/org/gcube/dataanalysis/ecoengine/test/TestsHSPENTraining.java diff --git a/cfg/modelers.properties b/cfg/modelers.properties new file mode 100644 index 0000000..d5c652e --- /dev/null +++ b/cfg/modelers.properties @@ -0,0 +1 @@ +HSPEN_MODELER=org.gcube.dataanalysis.ecoengine.modeling.SimpleModeler \ No newline at end of file diff --git a/cfg/models.properties b/cfg/models.properties index 5021dd7..9cb943e 100644 --- a/cfg/models.properties +++ b/cfg/models.properties @@ -1,2 +1 @@ HSPEN=org.gcube.dataanalysis.ecoengine.models.ModelHSPEN -HSPEN_MODELER=org.gcube.dataanalysis.ecoengine.modeling.SimpleModeler \ No newline at end of file diff --git a/src/org/gcube/dataanalysis/ecoengine/configuration/ALG_PROPS.java b/src/org/gcube/dataanalysis/ecoengine/configuration/ALG_PROPS.java index 39c4ea8..201193e 100644 --- a/src/org/gcube/dataanalysis/ecoengine/configuration/ALG_PROPS.java +++ b/src/org/gcube/dataanalysis/ecoengine/configuration/ALG_PROPS.java @@ -5,5 +5,6 @@ public enum ALG_PROPS { SPECIES_VS_CSQUARE_FROM_DATABASE, SPECIES_VS_CSQUARE_REMOTE_FROM_DATABASE, SPECIES_VS_CSQUARE, - PHENOMENON_VS_GEOINFO + PHENOMENON_VS_GEOINFO, + SPECIES_ENVELOPES } diff --git a/src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java b/src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java index 8fc5a5f..5db32bc 100644 --- a/src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java +++ b/src/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java @@ -32,6 +32,7 @@ public class AlgorithmConfiguration extends LexicalEngineConfiguration{ public static String algorithmsFile = "algorithms.properties"; public static String generatorsFile = "generators.properties"; public static String modelsFile = "models.properties"; + public static String modelersFile = "modelers.properties"; public static int chunkSize = 100000; public static int refreshResourcesTime = 10; diff --git a/src/org/gcube/dataanalysis/ecoengine/configuration/GENERATOR_WEIGHT.java b/src/org/gcube/dataanalysis/ecoengine/configuration/WEIGHT.java similarity index 75% rename from src/org/gcube/dataanalysis/ecoengine/configuration/GENERATOR_WEIGHT.java rename to src/org/gcube/dataanalysis/ecoengine/configuration/WEIGHT.java index 94d0060..db364c5 100644 --- a/src/org/gcube/dataanalysis/ecoengine/configuration/GENERATOR_WEIGHT.java +++ b/src/org/gcube/dataanalysis/ecoengine/configuration/WEIGHT.java @@ -1,6 +1,6 @@ package org.gcube.dataanalysis.ecoengine.configuration; -public enum GENERATOR_WEIGHT { +public enum WEIGHT { VERY_HIGH, HIGH, diff --git a/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java b/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java index de692ec..c668fb9 100644 --- a/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java +++ b/src/org/gcube/dataanalysis/ecoengine/interfaces/Generator.java @@ -4,7 +4,7 @@ import java.util.HashMap; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; -import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; +import org.gcube.dataanalysis.ecoengine.configuration.WEIGHT; public interface Generator { @@ -14,7 +14,7 @@ public interface Generator { public HashMap getInputParameters(); //gets the weight of the generator: according to this the generator will be placed in the execution order - public GENERATOR_WEIGHT getWeight(); + public WEIGHT getWeight(); public float getStatus(); diff --git a/src/org/gcube/dataanalysis/ecoengine/interfaces/Model.java b/src/org/gcube/dataanalysis/ecoengine/interfaces/Model.java index 995bc56..f98969d 100644 --- a/src/org/gcube/dataanalysis/ecoengine/interfaces/Model.java +++ b/src/org/gcube/dataanalysis/ecoengine/interfaces/Model.java @@ -1,14 +1,30 @@ package org.gcube.dataanalysis.ecoengine.interfaces; +import java.util.HashMap; + +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; + public interface Model { + + //defines the properties of this algorithm + public ALG_PROPS[] getProperties(); + + //defines the name of this model + public String getName(); + + //gets the description of the model + public String getDescription(); + + //set the input parameters for this generator + public HashMap getInputParameters(); + public float getVersion(); public void setVersion(float version); - public String getName(); - - public void init(Object Input, Model previousModel, Object Destination); + public void init(AlgorithmConfiguration Input, Model previousModel); public String getResourceLoad(); @@ -20,9 +36,9 @@ public interface Model { public String getOutputType(); - public void postprocess(Object Input, Model previousModel, Object Destination); + public void postprocess(AlgorithmConfiguration Input, Model previousModel); - public void train(Object Input, Model previousModel, Object Destination); + public void train(AlgorithmConfiguration Input, Model previousModel); // gets the type of the content inside the model: e.g. Table Model, Vectorial Model etc. public String getContentType(); diff --git a/src/org/gcube/dataanalysis/ecoengine/interfaces/Modeler.java b/src/org/gcube/dataanalysis/ecoengine/interfaces/Modeler.java index eb807bb..b83d9d9 100644 --- a/src/org/gcube/dataanalysis/ecoengine/interfaces/Modeler.java +++ b/src/org/gcube/dataanalysis/ecoengine/interfaces/Modeler.java @@ -1,10 +1,25 @@ package org.gcube.dataanalysis.ecoengine.interfaces; +import java.util.HashMap; + +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.WEIGHT; + public interface Modeler { + + public ALG_PROPS[] getSupportedModels(); + + //set the input parameters for this generator + public HashMap getInputParameters(); + + //gets the weight of the generator: according to this the generator will be placed in the execution order + public WEIGHT getWeight(); + public void setmodel(Model model); - public void model(Object Input, Model previousModel, Object Destination); + public void model(AlgorithmConfiguration Input, Model previousModel); public String getResourceLoad(); diff --git a/src/org/gcube/dataanalysis/ecoengine/modeling/SimpleModeler.java b/src/org/gcube/dataanalysis/ecoengine/modeling/SimpleModeler.java index 1d5f36c..df0eb60 100644 --- a/src/org/gcube/dataanalysis/ecoengine/modeling/SimpleModeler.java +++ b/src/org/gcube/dataanalysis/ecoengine/modeling/SimpleModeler.java @@ -1,5 +1,10 @@ package org.gcube.dataanalysis.ecoengine.modeling; +import java.util.HashMap; + +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.WEIGHT; import org.gcube.dataanalysis.ecoengine.interfaces.Model; import org.gcube.dataanalysis.ecoengine.interfaces.Modeler; @@ -7,10 +12,10 @@ public class SimpleModeler implements Modeler{ private Model innermodel; @Override - public void model(Object Input, Model previousModel, Object Destination) { - innermodel.init(Input, previousModel, Destination); - innermodel.train(Input, previousModel, Destination); - innermodel.postprocess(Input, previousModel, Destination); + public void model(AlgorithmConfiguration Input, Model previousModel) { + innermodel.init(Input, previousModel); + innermodel.train(Input, previousModel); + innermodel.postprocess(Input, previousModel); } @Override @@ -53,6 +58,22 @@ public class SimpleModeler implements Modeler{ innermodel.stop(); } + @Override + public ALG_PROPS[] getSupportedModels() { + ALG_PROPS[] props = {ALG_PROPS.SPECIES_ENVELOPES}; + return props; + } + + @Override + public HashMap getInputParameters() { + return innermodel.getInputParameters(); + } + + @Override + public WEIGHT getWeight() { + return WEIGHT.LOWEST; + } + diff --git a/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java b/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java index 2c92298..4c403a8 100644 --- a/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java +++ b/src/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java @@ -9,6 +9,7 @@ import java.util.concurrent.Executors; import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; @@ -61,36 +62,42 @@ public class ModelHSPEN implements Model { @Override public String getName() { - return ModelHSPEN.class.getName(); + return "HSPEN"; } @Override - public void init(Object Input, Model previousModel, Object Destination) { - AlgorithmConfiguration config = (AlgorithmConfiguration) Input; - outconfig = (AlgorithmConfiguration) Destination; - defaultDatabaseFile = config.getConfigPath() + defaultDatabaseFile; + public void init(AlgorithmConfiguration setup, Model previousModel) { + outconfig = setup; + defaultDatabaseFile = setup.getConfigPath() + defaultDatabaseFile; - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + AnalysisLogger.setLogger(setup.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); try { - connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, config); + String defaultDatabaseFile = setup.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile; + + setup.setDatabaseDriver(setup.getParam("DatabaseDriver")); + setup.setDatabaseUserName(setup.getParam("DatabaseUserName")); + setup.setDatabasePassword(setup.getParam("DatabasePassword")); + setup.setDatabaseURL(setup.getParam("DatabaseURL")); + + connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, setup); } catch (Exception e) { AnalysisLogger.getLogger().debug(e); e.printStackTrace(); } // initialize queries - dynamicAlterQuery = alterQuery.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); - dynamicDropTable = dropHspenTable.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); - dynamicCreateTable = createHspenTable.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); - dynamicPopulateNewHspen = populateNewHspen.replace("%HSPEN_ORIGIN%", config.getParam("EnvelopeTable")).replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); - dynamicSpeciesListQuery = speciesListQuery.replace("%HSPEN%", config.getParam("EnvelopeTable")); - dynamicHspenInformationQuery = hspenListQuery.replace("%HSPEN%", config.getParam("EnvelopeTable")); - currentHCAFTable = config.getParam("CsquarecodesTable"); - currentOccurrenceTable = config.getParam("OccurrenceCellsTable"); + dynamicAlterQuery = alterQuery.replace("%HSPEN%", outconfig.getParam("OuputEnvelopeTable")); + dynamicDropTable = dropHspenTable.replace("%HSPEN%", outconfig.getParam("OuputEnvelopeTable")); + dynamicCreateTable = createHspenTable.replace("%HSPEN%", outconfig.getParam("OuputEnvelopeTable")); + dynamicPopulateNewHspen = populateNewHspen.replace("%HSPEN_ORIGIN%", outconfig.getParam("EnvelopeTable")).replace("%HSPEN%", outconfig.getParam("OuputEnvelopeTable")); + dynamicSpeciesListQuery = speciesListQuery.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); + dynamicHspenInformationQuery = hspenListQuery.replace("%HSPEN%", outconfig.getParam("EnvelopeTable")); + currentHCAFTable = outconfig.getParam("CsquarecodesTable"); + currentOccurrenceTable = outconfig.getParam("OccurrenceCellsTable"); // Threads - numberOfthreads = config.getNumberOfResources(); + numberOfthreads = outconfig.getNumberOfResources(); // interrupt process interruptProcessing = false; @@ -172,7 +179,7 @@ public class ModelHSPEN implements Model { } @Override - public void train(Object Input, Model previousModel, Object Destination) { + public void train(AlgorithmConfiguration Input, Model previousModel) { long tstart = System.currentTimeMillis(); // INITIALIZATION try { @@ -340,7 +347,7 @@ public class ModelHSPEN implements Model { } @Override - public void postprocess(Object Input, Model previousModel, Object Destination) { + public void postprocess(AlgorithmConfiguration Input, Model previousModel) { } @@ -405,4 +412,30 @@ public class ModelHSPEN implements Model { return AlgorithmConfiguration.class.getName(); } + @Override + public ALG_PROPS[] getProperties() { + ALG_PROPS[] props = {ALG_PROPS.SPECIES_ENVELOPES}; + return props; + } + + @Override + public String getDescription() { + return "Hspen Algorithm"; + } + + @Override + public HashMap getInputParameters() { + HashMap parameters = new HashMap(); + parameters.put("EnvelopeTable", "hspen"); + parameters.put("CsquarecodesTable", "hcaf_d"); + parameters.put("OccurrenceCellsTable", "occurrencecells"); + parameters.put("CreateTable", "true"); + parameters.put("OuputEnvelopeTable", ""); + parameters.put("DatabaseUserName",""); + parameters.put("DatabasePassword",""); + parameters.put("DatabaseURL",""); + parameters.put("DatabaseDriver",""); + return parameters; + } + } diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java index 0c01c4f..081ab06 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java @@ -14,7 +14,7 @@ import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; -import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; +import org.gcube.dataanalysis.ecoengine.configuration.WEIGHT; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; @@ -326,8 +326,8 @@ public class LocalSimpleSplitGenerator implements Generator { } @Override - public GENERATOR_WEIGHT getWeight() { - return GENERATOR_WEIGHT.LOWEST; + public WEIGHT getWeight() { + return WEIGHT.LOWEST; } @Override diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java index d76fea5..fb1dbe1 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java @@ -16,7 +16,7 @@ import org.gcube.contentmanagement.graphtools.utils.MathFunctions; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; -import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; +import org.gcube.dataanalysis.ecoengine.configuration.WEIGHT; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; @@ -469,8 +469,8 @@ public class LocalSplitGenerator implements Generator { } @Override - public GENERATOR_WEIGHT getWeight() { - return GENERATOR_WEIGHT.LOWEST; + public WEIGHT getWeight() { + return WEIGHT.LOWEST; } @Override diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java b/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java index 5a5ee20..9652da6 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java @@ -6,7 +6,7 @@ import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; -import org.gcube.dataanalysis.ecoengine.configuration.GENERATOR_WEIGHT; +import org.gcube.dataanalysis.ecoengine.configuration.WEIGHT; import org.gcube.dataanalysis.ecoengine.connectors.RemoteGenerationManager; import org.gcube.dataanalysis.ecoengine.connectors.RemoteHspecInputObject; import org.gcube.dataanalysis.ecoengine.connectors.RemoteHspecOutputObject; @@ -172,8 +172,8 @@ public class RainyCloudGenerator implements Generator { } @Override - public GENERATOR_WEIGHT getWeight() { - return GENERATOR_WEIGHT.HIGH; + public WEIGHT getWeight() { + return WEIGHT.HIGH; } @Override diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java b/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java index dbdbeee..bf419de 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java @@ -56,70 +56,25 @@ public class GeneratorsFactory { public static List getProbabilityDistributionAlgorithms(String configPath) throws Exception{ - - Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.algorithmsFile); - List algs = new ArrayList(); - for (Object algName: p.keySet()){ - algs.add((String)algName); - } + List algs = ProcessorsFactory.getClasses(configPath + AlgorithmConfiguration.algorithmsFile); return algs; } public static List getAllGenerators(String configPath) throws Exception{ - Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.generatorsFile); - List gens = new ArrayList(); - for (Object genName: p.keySet()){ - gens.add((String)genName); - } + List gens = ProcessorsFactory.getClasses(configPath + AlgorithmConfiguration.generatorsFile); return gens; } - public static List getModels(String configPath) throws Exception{ - - Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.modelsFile); - List models = new ArrayList(); - for (Object modelName: p.keySet()){ - models.add((String)modelName); - } - return models; - } - public static HashMap getAlgorithmParameters(String configPath, String algorithmName) throws Exception{ - - Properties p = AlgorithmConfiguration.getProperties(configPath + AlgorithmConfiguration.algorithmsFile); - String algorithmclass = p.getProperty(algorithmName); - Object algclass = Class.forName(algorithmclass).newInstance(); - //if the algorithm is a generator itself then take it - if (algclass instanceof Generator){ - return ((Generator) algclass).getInputParameters(); - } - else - return ((SpatialProbabilityDistribution) algclass).getInputParameters(); - + + public static HashMap getAlgorithmParameters(String configPath, String algorithmName) throws Exception{ + HashMap inputs = ProcessorsFactory.getParameters(configPath + AlgorithmConfiguration.algorithmsFile, algorithmName); + return inputs; } - public static Generator getGenerator(AlgorithmConfiguration config) throws Exception { - //modify this class in order to take the right generator algorithm - try { - //initialize the logger - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); - //take the algorithm - String algorithm = config.getGenerator(); - if (algorithm == null) throw new Exception("GENERATOR NOT SPECIFIED"); - //take the algorithms list - Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.generatorsFile); - String algorithmclass = p.getProperty(algorithm); - Object algclass = Class.forName(algorithmclass).newInstance(); - Generator g = (Generator) algclass; - g.setConfiguration(config); - g.init(); - return g; - } catch (Exception e) { - e.printStackTrace(); - throw e; - } + return (Generator)ProcessorsFactory.getProcessor(config, config.getConfigPath() + AlgorithmConfiguration.generatorsFile); } diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java b/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java index 538b151..c7fa436 100644 --- a/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java +++ b/src/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java @@ -1,30 +1,111 @@ package org.gcube.dataanalysis.ecoengine.processing.factories; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Properties; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.Model; import org.gcube.dataanalysis.ecoengine.interfaces.Modeler; public class ModelersFactory { - public static Modeler getGenerator(AlgorithmConfiguration config) throws Exception { + + public static Modeler getModeler(AlgorithmConfiguration config) throws Exception { + Modeler modlr = (Modeler)ProcessorsFactory.getProcessor(config, config.getConfigPath() + AlgorithmConfiguration.generatorsFile); + return modlr; + } + + public static List getModels(String configPath) throws Exception{ + List models = ProcessorsFactory.getClasses(configPath + AlgorithmConfiguration.modelsFile); + return models; + } + + public static List getModelers(String configPath) throws Exception{ + List modelers = ProcessorsFactory.getClasses(configPath + AlgorithmConfiguration.modelersFile); + return modelers; + } + + + public static HashMap getModelParameters(String configPath, String algorithmName) throws Exception{ + HashMap inputs = ProcessorsFactory.getParameters(configPath + AlgorithmConfiguration.modelsFile, algorithmName); + return inputs; + } + + +public static List getModelers(AlgorithmConfiguration config) throws Exception { + + //modify this class in order to manage generators weight and match algorithm vs generators + List modelers = new ArrayList(); try { + //initialize the logger AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + //take the algorithm + String algorithm = config.getModel(); + //take the algorithms list Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.modelsFile); - String objectclass = p.getProperty(config.getModel() + "_MODELER"); - Modeler g = (Modeler) Class.forName(objectclass).newInstance(); - String modelclass = p.getProperty(config.getModel()); - Model m = (Model) Class.forName(modelclass).newInstance(); - g.setmodel(m); - - return g; + String algorithmclass = p.getProperty(algorithm); + Object algclass = Class.forName(algorithmclass).newInstance(); + //if the algorithm is a generator itself then execute it + if (algclass instanceof Modeler){ + Modeler g = (Modeler) algclass; + modelers.add(g); + } + else + { + Model mod = (Model) algclass; + //take alg's properties + ALG_PROPS[] algp = mod.getProperties(); + //take all generators + Properties pg = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.modelersFile); + //investigate on possible suitable modelers + for (Object modelerName:pg.values()){ + Modeler gen = (Modeler)Class.forName((String)modelerName).newInstance(); + ALG_PROPS[] supportedAlgs = gen.getSupportedModels(); + boolean genSuitable = false; + for (ALG_PROPS prop:algp){ + for (ALG_PROPS gprop:supportedAlgs){ + if (gprop == prop){ + genSuitable = true; + gen.setmodel(mod); + break; + } + } + } + //if suitable generator was found then add it at the right place in the list + if (genSuitable){ + addModeler2List(modelers,gen); + } + } + } + + return modelers; } catch (Exception e) { e.printStackTrace(); throw e; } + + + } + + //adds a generator to a sorted generators list + public static void addModeler2List(List modelers, Modeler mod){ + int i=0; + boolean inserted = false; + for (Modeler g: modelers){ + if (g.getWeight().compareTo(mod.getWeight())>0){ + modelers.add(i, mod); + inserted = true; + break; + } + i++; + } + if (!inserted) + modelers.add(mod); } } diff --git a/src/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java b/src/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java new file mode 100644 index 0000000..36eacda --- /dev/null +++ b/src/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java @@ -0,0 +1,80 @@ +package org.gcube.dataanalysis.ecoengine.processing.factories; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.Generator; +import org.gcube.dataanalysis.ecoengine.interfaces.Model; +import org.gcube.dataanalysis.ecoengine.interfaces.Modeler; +import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistribution; + +public class ProcessorsFactory { + + + public static List getClasses(String file) throws Exception{ + + Properties p = AlgorithmConfiguration.getProperties(file); + List algs = new ArrayList(); + for (Object algName: p.keySet()){ + algs.add((String)algName); + } + return algs; + } + + + public static HashMap getParameters(String file, String algorithmName) throws Exception{ + + Properties p = AlgorithmConfiguration.getProperties(file); + String algorithmclass = p.getProperty(algorithmName); + Object algclass = Class.forName(algorithmclass).newInstance(); + //if the algorithm is a generator itself then take it + if (algclass instanceof Generator){ + return ((Generator) algclass).getInputParameters(); + } + else if (algclass instanceof Modeler){ + return ((Modeler) algclass).getInputParameters(); + } + else if (algclass instanceof Model){ + return ((Model) algclass).getInputParameters(); + } + else + return ((SpatialProbabilityDistribution) algclass).getInputParameters(); + + } + + public static Object getProcessor(AlgorithmConfiguration config,String file) throws Exception { + //modify this class in order to take the right generator algorithm + try { + //initialize the logger + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + //take the algorithm + String algorithm = config.getGenerator(); + if (algorithm == null) throw new Exception("PROCESSOR NOT SPECIFIED"); + //take the algorithms list + Properties p = AlgorithmConfiguration.getProperties(file); + String algorithmclass = p.getProperty(algorithm); + Object algclass = Class.forName(algorithmclass).newInstance(); + if (algclass instanceof Generator){ + Generator g = (Generator) algclass; + g.setConfiguration(config); + g.init(); + return g; + } + else if (algclass instanceof Modeler){ + Modeler m = (Modeler) algclass; + return m; + } + else return null; + + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + +} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/TestsHSPENTraining.java b/src/org/gcube/dataanalysis/ecoengine/test/TestsHSPENTraining.java new file mode 100644 index 0000000..39cd568 --- /dev/null +++ b/src/org/gcube/dataanalysis/ecoengine/test/TestsHSPENTraining.java @@ -0,0 +1,90 @@ +package org.gcube.dataanalysis.ecoengine.test; + +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.Generator; +import org.gcube.dataanalysis.ecoengine.interfaces.Modeler; +import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory; + +public class TestsHSPENTraining { + /** + * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species + * + */ + +public static void main(String[] args) throws Exception { + + System.out.println("TEST 1"); + List modelers=ModelersFactory.getModelers(testConfig()); + train(modelers.get(0),testConfig()); + modelers = null; +} + + + private static void train(Modeler generator,AlgorithmConfiguration config) throws Exception { + + if (generator != null) { + TestsHSPENTraining tgs = new TestsHSPENTraining(); + ThreadCalculator tc = tgs.new ThreadCalculator(generator, config); + Thread t = new Thread(tc); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(1000); + } + System.out.println("FINAL STATUS: " + generator.getStatus()); + } else + AnalysisLogger.getLogger().trace("Generator Algorithm Not Supported"); + + } + + public class ThreadCalculator implements Runnable { + Modeler dg; + AlgorithmConfiguration config; + + public ThreadCalculator(Modeler dg,AlgorithmConfiguration config) { + this.dg = dg; + this.config = config; + } + + public void run() { + try { + + dg.model(config, null); + + } catch (Exception e) { + } + } + + } + + + private static AlgorithmConfiguration testConfig() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(2); + config.setModel("HSPEN"); + + config.setParam("OuputEnvelopeTable","hspen_trained"); + config.setParam("OccurrenceCellsTable","occurrencecells"); + config.setParam("EnvelopeTable","hspen_mini"); + config.setParam("CsquarecodesTable", "hcaf_d"); + config.setParam("CreateTable","true"); + + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); + config.setParam("DatabaseDriver","org.postgresql.Driver"); + + return config; + } + +} diff --git a/src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java b/src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java index c063b70..5bc5d69 100644 --- a/src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java +++ b/src/org/gcube/dataanalysis/ecoengine/test/TestsMetaInfo.java @@ -7,6 +7,7 @@ import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory; public class TestsMetaInfo { /** @@ -34,7 +35,15 @@ public static void main(String[] args) throws Exception { System.out.println("Gens: "+GeneratorsFactory.getAllGenerators("./cfg/")); System.out.println("\n***TEST 5 - Get All Models to be trained ***"); - System.out.println("Models: "+GeneratorsFactory.getModels("./cfg/")); + System.out.println("Models: "+ModelersFactory.getModels("./cfg/")); + + System.out.println("\n***TEST 6 - Get All Modelers ***"); + System.out.println("Models: "+ModelersFactory.getModelers("./cfg/")); + + System.out.println("\n***TEST 7- Get All Model parameters ***"); + map = ModelersFactory.getModelParameters("./cfg/","HSPEN"); + System.out.println("input for HSPEN algorithm: "+map); + }