From abd80189bae49b3c3cc753b49197972bf31f4425 Mon Sep 17 00:00:00 2001 From: Lucio Lelii Date: Fri, 8 Mar 2019 16:08:39 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@178520 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 5 - .../abstracts/GenericStandaloneGraph.java | 7 +- .../graphtools/core/StatisticsGenerator.java | 45 ++----- .../data/conversions/GraphConverter2D.java | 2 +- .../data/databases/CommonDBExtractor.java | 9 +- .../ExampleExternalConfigPostGressProd.java | 42 ------- .../examples/ExamplePostGressLocalRadar.java | 42 ------- .../examples/graphsTypes/ExampleImage.java | 113 ------------------ .../graphs/GaussianDistributionGraph.java | 12 +- .../graphtools/plotting/graphs/PieGraph.java | 14 +-- .../plotting/graphs/TimeSeriesGraph.java | 7 +- .../lexicalmatcher/analysis/core/Engine.java | 30 ++--- .../analysis/examples/Example1_Species.java | 32 ----- .../analysis/examples/Example2_Area.java | 32 ----- .../examples/Example3_SingleMatchShark.java | 48 -------- .../examples/Example4_SingleMatchMitella.java | 49 -------- .../examples/Example5_SingleMatchMitella.java | 48 -------- .../examples/ExampleGuessingExternalCfg.java | 64 ---------- .../analysis/guesser/data/CategoryScores.java | 9 +- .../guesser/data/CategoryScoresOld.java | 11 +- .../guesser/data/DBObjectTranslator.java | 41 ++++--- .../treeStructure/chunks/ChunkSet.java | 6 +- .../treeStructure/chunks/ReferenceChunk.java | 7 +- .../treeStructure/chunks/TimeSeriesChunk.java | 16 +-- .../treeStructure/graph/GraphDisplayer.java | 11 +- .../analysis/run/CategoryGuesser.java | 56 ++++----- .../analysis/run/StarGraphExtraction.java | 36 ------ .../analysis/test/ExampleGuesser.java | 52 -------- .../analysis/test/ExampleLexicalMatcher.java | 58 --------- .../test/TestExternalCfgProduction.java | 65 ---------- .../test/TestSingleExternalCfgProduction.java | 76 ------------ .../analysis/test/old/BenchMarkTest1.java | 58 --------- .../analysis/test/old/BenchMarkTest2.java | 54 --------- .../analysis/test/old/BenchMarkTest3.java | 31 ----- .../analysis/test/old/BenchMarkTest4.java | 31 ----- .../analysis/test/old/BenchMarkTest5.java | 31 ----- .../test/old/BenchMarkTestExternalCfg.java | 52 -------- .../test/old/BenchMarkTestFilterCategory.java | 38 ------ .../test/old/BenchMarkTestSingleton.java | 51 -------- .../test/old/BenchMarkTestTSCountry.java | 31 ----- .../test/old/BenchMarkTrainingSet.java | 88 -------------- .../BenchMarkTrainingSetScientificName.java | 33 ----- .../test/old/TestExternalCfgProduction.java | 64 ---------- .../old/TestSingleExternalCfgProduction.java | 71 ----------- .../lexicalmatcher/utils/AnalysisLogger.java | 49 ++++---- .../lexicalmatcher/utils/MathFunctions.java | 16 ++- .../ecoengine/clustering/DBScan.java | 64 +++++----- .../ecoengine/clustering/KMeans.java | 43 ++----- .../ecoengine/clustering/LOF.java | 55 ++++----- .../ecoengine/clustering/XMeansWrapper.java | 108 ++++------------- .../configuration/AlgorithmConfiguration.java | 12 +- .../connectors/RemoteGenerationManager.java | 23 ++-- .../evaluation/DiscrepancyAnalysis.java | 15 ++- .../DistributionQualityAnalysis.java | 68 ++--------- .../evaluation/HabitatRepresentativeness.java | 87 ++++---------- .../bioclimate/BioClimateAnalysis.java | 66 +++++----- .../bioclimate/InterpolateTables.java | 49 ++++---- .../ecoengine/interfaces/DataAnalysis.java | 10 +- .../ecoengine/models/FeedForwardNN.java | 36 +++--- .../ecoengine/models/ModelAquamapsNN.java | 30 +++-- .../ecoengine/models/ModelAquamapsNNNS.java | 20 ++-- .../ecoengine/models/ModelHSPEN.java | 46 +++---- .../aquamaps/AquamapsEnvelopeAlgorithm.java | 18 +-- .../cores/aquamaps/MaxMinGenerator.java | 41 ++++--- .../cores/neuralnetworks/Neural_Network.java | 16 ++- .../cores/pca/PrincipalComponentAnalysis.java | 71 +---------- .../models/testing/FeedForwardNNFile.java | 36 +++--- .../processing/LocalSimpleSplitGenerator.java | 40 ++++--- .../processing/LocalSplitGenerator.java | 66 +++++----- .../processing/RainyCloudGenerator.java | 18 ++- .../DynamicTransducerersFactory.java | 11 +- .../factories/GeneratorsFactory.java | 3 +- .../processing/factories/ModelersFactory.java | 3 - .../factories/ProcessorsFactory.java | 9 +- .../factories/TransducerersFactory.java | 11 +- .../signals/PeriodicityDetector.java | 71 +++++------ .../ecoengine/signals/SignalConverter.java | 38 +++--- .../ecoengine/signals/SignalProcessing.java | 69 ++++++----- .../ecoengine/signals/TimeSeries.java | 21 ++-- .../ecoengine/signals/ssa/SSAWorkflow.java | 12 +- .../signals/ssa/SingularSpectrumAnalysis.java | 19 +-- .../spatialdistributions/AquamapsNN.java | 19 +-- .../spatialdistributions/AquamapsNNNS.java | 9 +- .../AquamapsNNSuitable.java | 19 +-- .../AquamapsSuitable.java | 19 +-- .../BayesianDistribution.java | 25 ++-- .../spatialdistributions/DummyAlgorithm.java | 31 +++-- .../FeedForwardNeuralNetworkDistribution.java | 7 +- .../spatialdistributions/TestAlgorithm.java | 11 +- .../ecoengine/test/regression/Regressor.java | 3 - .../OccurrencePointsDuplicatesDeleter.java | 46 +++---- .../OccurrencePointsInSeaOnEarth.java | 21 ++-- .../transducers/OccurrencePointsMerger.java | 69 ++++++----- .../ecoengine/transducers/QueryExecutor.java | 20 ++-- .../transducers/TimeSeriesAnalysis.java | 64 +++++----- .../charts/AbstractChartsProducer.java | 20 ++-- ...uantitiesAttributesChartsTransducerer.java | 10 +- .../charts/TimeSeriesChartsTransducerer.java | 17 +-- .../utils/HspecDiscrepanciesCalculator.java | 30 ++--- .../utils/TablesDiscrepanciesCalculator.java | 24 ++-- .../utils/TrainingSetsGenerator.java | 13 +- .../ecoengine/utils/ZipTools.java | 104 ++++++++-------- 102 files changed, 1025 insertions(+), 2604 deletions(-) delete mode 100644 src/main/java/org/gcube/contentmanagement/graphtools/examples/ExampleExternalConfigPostGressProd.java delete mode 100644 src/main/java/org/gcube/contentmanagement/graphtools/examples/ExamplePostGressLocalRadar.java delete mode 100644 src/main/java/org/gcube/contentmanagement/graphtools/examples/graphsTypes/ExampleImage.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example1_Species.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example2_Area.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example3_SingleMatchShark.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example4_SingleMatchMitella.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example5_SingleMatchMitella.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/ExampleGuessingExternalCfg.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/StarGraphExtraction.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleGuesser.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleLexicalMatcher.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestExternalCfgProduction.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestSingleExternalCfgProduction.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest1.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest2.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest3.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest4.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest5.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestExternalCfg.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestFilterCategory.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestSingleton.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestTSCountry.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSet.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSetScientificName.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestExternalCfgProduction.java delete mode 100644 src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestSingleExternalCfgProduction.java diff --git a/pom.xml b/pom.xml index c690d18..80e1c17 100644 --- a/pom.xml +++ b/pom.xml @@ -62,11 +62,6 @@ commons-httpclient 3.1 - - commons-logging - commons-logging - 1.0.4 - dom4j dom4j diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/abstracts/GenericStandaloneGraph.java b/src/main/java/org/gcube/contentmanagement/graphtools/abstracts/GenericStandaloneGraph.java index 652a645..1a609ed 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/abstracts/GenericStandaloneGraph.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/abstracts/GenericStandaloneGraph.java @@ -11,7 +11,6 @@ import javax.imageio.ImageIO; import javax.swing.JPanel; import org.gcube.contentmanagement.graphtools.data.conversions.ImageTools; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.portlets.user.timeseries.charts.support.types.GraphData; import org.gcube.portlets.user.timeseries.charts.support.types.GraphGroups; import org.gcube.portlets.user.timeseries.charts.support.types.Point; @@ -21,6 +20,8 @@ import org.jfree.data.category.DefaultCategoryDataset; import org.jfree.data.general.Dataset; import org.jfree.ui.ApplicationFrame; import org.jfree.ui.RefineryUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* * Converts a GraphData into a graphicable structure DataSet @@ -28,6 +29,8 @@ import org.jfree.ui.RefineryUtilities; */ public abstract class GenericStandaloneGraph extends ApplicationFrame { + private static Logger logger = LoggerFactory.getLogger(GenericStandaloneGraph.class); + /** * */ @@ -108,7 +111,7 @@ public abstract class GenericStandaloneGraph extends ApplicationFrame { try{ ImageIO.write(bimage, "png", outputfile); }catch(Exception e){ - AnalysisLogger.getLogger().error("renderImages->Error in writing files ",e); + logger.error("renderImages->Error in writing files ",e); } i++; } diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/core/StatisticsGenerator.java b/src/main/java/org/gcube/contentmanagement/graphtools/core/StatisticsGenerator.java index c8ce539..2c02575 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/core/StatisticsGenerator.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/core/StatisticsGenerator.java @@ -10,12 +10,13 @@ import org.gcube.contentmanagement.graphtools.data.GraphSamplesTable; import org.gcube.contentmanagement.graphtools.data.conversions.GraphConverter2D; import org.gcube.contentmanagement.graphtools.data.databases.CommonDBExtractor; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.gcube.portlets.user.timeseries.charts.support.types.GraphData; import org.gcube.portlets.user.timeseries.charts.support.types.GraphGroups; import org.gcube.portlets.user.timeseries.charts.support.types.Point; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.RapidMiner; import com.rapidminer.example.ExampleSet; @@ -26,31 +27,8 @@ import com.thoughtworks.xstream.io.xml.DomDriver; public class StatisticsGenerator { - public static void main(String[] args) throws Exception { - String table = "ts_161efa00_2c32_11df_b8b3_aa10916debe6"; - String xDimension = "field5"; - String yDimension = "field6"; - String groupDimension = "field1"; - String speciesColumn = "field3"; - String filter1 = "Brown seaweeds"; - String filter2 = "River eels"; - StatisticsGenerator stg = new StatisticsGenerator(); - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - // database Parameters - conf.setDatabaseUserName("root"); - // conf.setDatabasePassword("password"); - conf.setDatabaseDriver("com.mysql.jdbc.Driver"); - conf.setDatabaseURL("jdbc:mysql://localhost/timeseries"); - conf.setDatabaseDialect("org.hibernate.dialect.MySQLDialect"); - conf.setDatabaseAutomaticTestTable("connectiontesttable"); - conf.setDatabaseIdleConnectionTestPeriod("3600"); - - // stg.init("./cfg/"); - stg.init("./cfg/", conf); - - stg.generateGraphs(3, table, xDimension, yDimension, groupDimension, speciesColumn, filter1, filter2); - } + private static Logger logger = LoggerFactory.getLogger(StatisticsGenerator.class); + SessionFactory referenceDBSession; CommonDBExtractor extractor; @@ -71,7 +49,6 @@ public class StatisticsGenerator { } public void init(String cfgPath, LexicalEngineConfiguration config) throws Exception { - AnalysisLogger.setLogger(cfgPath + "/" + LogFile); if (config == null) referenceDBSession = DatabaseFactory.initDBConnection(cfgPath + HibFile); else @@ -81,7 +58,7 @@ public class StatisticsGenerator { extractor = new CommonDBExtractor(referenceDBSession); - AnalysisLogger.getLogger().info("StatisticsGenerator->initialization complete"); + logger.info("StatisticsGenerator->initialization complete"); System.setProperty("rapidminer.init.operators", cfgPath + OperatorsFile); xStream = new XStream(new DomDriver()); RapidMiner.init(); @@ -112,8 +89,8 @@ public class StatisticsGenerator { Map samplesMap = extractor.getMultiDimTemporalTables(ColumnFilters, YRangeFilter, timeSeriesTable, xDimension, groupDimension, yDimension, speciesColumn, filters); - AnalysisLogger.getLogger().info("StatisticsGenerator-> samplesMap has been generated"); - AnalysisLogger.getLogger().trace(samplesMap.toString()); + logger.info("StatisticsGenerator-> samplesMap has been generated"); + logger.trace(samplesMap.toString()); // setup Absolute Sampling operator AbsoluteSampling asop = (AbsoluteSampling) OperatorService.createOperator("AbsoluteSampling"); asop.setParameter("sample_size", "" + maxElements); @@ -149,14 +126,14 @@ public class StatisticsGenerator { // get the points list from the graph samples table singlegraph = graphSamples.getGraph(); - AnalysisLogger.getLogger().trace("Regeneration\n" + graphSamples.toString()); + logger.trace("Regeneration\n" + graphSamples.toString()); } // reorder the elements of the points list // this steps performs re-enumeration and reordering of the rows after the sampling operations singlegraph = GraphConverter2D.reorder(singlegraph); - AnalysisLogger.getLogger().trace("Reordering\n" + singlegraph.toString()); + logger.trace("Reordering\n" + singlegraph.toString()); if ((singlegraph != null)&&(singlegraph.size()>0)) { // build up the GraphData for visualization GraphData grd = new GraphData(singlegraph, true); @@ -171,12 +148,12 @@ public class StatisticsGenerator { // add the points list graphgroups.addGraph("Distribution for " + key, grd); - AnalysisLogger.getLogger().trace("StatisticsGenerator-> graphgroup " + i + " generated with key: " + key); + logger.trace("StatisticsGenerator-> graphgroup " + i + " generated with key: " + key); i++; } } - AnalysisLogger.getLogger().info("StatisticsGenerator-> graphgroups have been generated"); + logger.info("StatisticsGenerator-> graphgroups have been generated"); return graphgroups; diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/data/conversions/GraphConverter2D.java b/src/main/java/org/gcube/contentmanagement/graphtools/data/conversions/GraphConverter2D.java index 79a0f8e..bcc4610 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/data/conversions/GraphConverter2D.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/data/conversions/GraphConverter2D.java @@ -5,7 +5,6 @@ import java.util.List; import org.gcube.contentmanagement.graphtools.abstracts.SamplesTable; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.portlets.user.timeseries.charts.support.types.GraphData; import org.gcube.portlets.user.timeseries.charts.support.types.GraphGroups; import org.gcube.portlets.user.timeseries.charts.support.types.Point; @@ -17,6 +16,7 @@ import org.gcube.portlets.user.timeseries.charts.support.types.ValueEntry; */ public class GraphConverter2D { + private static Point searchPoint(Point x, List> samples) { Point novelpoint = null; for (Point point : samples) { diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/data/databases/CommonDBExtractor.java b/src/main/java/org/gcube/contentmanagement/graphtools/data/databases/CommonDBExtractor.java index 502351e..8b13665 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/data/databases/CommonDBExtractor.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/data/databases/CommonDBExtractor.java @@ -10,12 +10,15 @@ import org.gcube.contentmanagement.graphtools.abstracts.SamplesTable; import org.gcube.contentmanagement.graphtools.core.filters.Filter; import org.gcube.contentmanagement.graphtools.data.BigSamplesTable; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.DataTypeRecognizer; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CommonDBExtractor implements GenericDBExtractor { + private static Logger logger = LoggerFactory.getLogger(CommonDBExtractor.class); + SessionFactory dbSession; public CommonDBExtractor(SessionFactory DbSession) { @@ -76,7 +79,7 @@ public class CommonDBExtractor implements GenericDBExtractor { HashMap typesMap = new HashMap(); String queryDesc = String.format(descriptionQuery, table); - AnalysisLogger.getLogger().trace("Query for Description: " + queryDesc); + logger.trace("Query for Description: " + queryDesc); List resultSet = DatabaseFactory.executeSQLQuery(queryDesc, dbSession); for (Object result : resultSet) { @@ -169,7 +172,7 @@ public class CommonDBExtractor implements GenericDBExtractor { query += String.format(staticOrderBy, xDimension); - AnalysisLogger.getLogger().trace("Query: " + query); + logger.trace("Query: " + query); List resultSet = DatabaseFactory.executeSQLQuery(query, dbSession); // xdim, group, quantity, yLabel diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/examples/ExampleExternalConfigPostGressProd.java b/src/main/java/org/gcube/contentmanagement/graphtools/examples/ExampleExternalConfigPostGressProd.java deleted file mode 100644 index 374bbc9..0000000 --- a/src/main/java/org/gcube/contentmanagement/graphtools/examples/ExampleExternalConfigPostGressProd.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.gcube.contentmanagement.graphtools.examples; - -import org.gcube.contentmanagement.graphtools.core.StatisticsGenerator; -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class ExampleExternalConfigPostGressProd { - - - public static void main(String[] args) throws Exception{ - String table = "ts_c4bdfaa0_6c16_11e0_bb1f_fb760af5afc7"; - String xDimension = "field4"; // the dates - String yDimension = "field6"; // values on Y - String groupDimension = "field1"; // group names - String speciesColumn = "field2"; // lines labels - - String filter1 = "ABW"; -// String filter2 = "Osteichthyes"; - StatisticsGenerator stg = new StatisticsGenerator(); - - AnalysisLogger.setLogger("./cfg/ALog.properties"); - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - //database Parameters - conf.setDatabaseUserName("gcube1"); - conf.setDatabasePassword("d4science"); - conf.setDatabaseDriver("org.postgresql.Driver"); - conf.setDatabaseURL("jdbc:postgresql://node28.p.d4science.research-infrastructures.eu/timeseries"); - -// conf.setDatabaseDialect("org.hibernate.dialect.MySQLDialect"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - conf.setDatabaseAutomaticTestTable("connectiontesttable"); - conf.setDatabaseIdleConnectionTestPeriod("3600"); - - stg.init("./cfg/",conf); - -// stg.generateGraphs(200, table, xDimension, yDimension, groupDimension, speciesColumn, filter1, filter2); - stg.generateGraphs(200, table, xDimension, yDimension, groupDimension, speciesColumn,filter1); - - System.out.println(); - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/examples/ExamplePostGressLocalRadar.java b/src/main/java/org/gcube/contentmanagement/graphtools/examples/ExamplePostGressLocalRadar.java deleted file mode 100644 index 4bec670..0000000 --- a/src/main/java/org/gcube/contentmanagement/graphtools/examples/ExamplePostGressLocalRadar.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.gcube.contentmanagement.graphtools.examples; - -import org.gcube.contentmanagement.graphtools.core.StatisticsGenerator; -import org.gcube.contentmanagement.graphtools.plotting.graphs.RadarGraph; -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.portlets.user.timeseries.charts.support.types.GraphGroups; - -public class ExamplePostGressLocalRadar { - - - public static void main(String[] args) throws Exception{ - - AnalysisLogger.setLogger("./cfg/ALog.properties"); - - String table = "ts_3bdaf790_edbe_11e0_93e3_f6a9821baa29"; - String xDimension = "field2"; // the dates - String yDimension = "field4"; // values on Y - String groupDimension = "field0"; // group names - String speciesColumn = "field3"; // lines labels - String filter2 = "Perciformes"; - String filter1 = "Boregadus"; - StatisticsGenerator stg = new StatisticsGenerator(); - - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - //database Parameters - conf.setDatabaseUserName("gcube"); - conf.setDatabasePassword("d4science2"); - conf.setDatabaseDriver("org.postgresql.Driver"); - conf.setDatabaseURL("jdbc:postgresql://dbtest.next.research-infrastructures.eu/timeseries"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - - stg.init("./cfg/",conf); - - GraphGroups gg = stg.generateGraphs(200, table, xDimension, yDimension, groupDimension, speciesColumn, filter1); - - RadarGraph radar = new RadarGraph(""); - radar.renderGraphGroup(gg); - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/examples/graphsTypes/ExampleImage.java b/src/main/java/org/gcube/contentmanagement/graphtools/examples/graphsTypes/ExampleImage.java deleted file mode 100644 index e45c2e2..0000000 --- a/src/main/java/org/gcube/contentmanagement/graphtools/examples/graphsTypes/ExampleImage.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.gcube.contentmanagement.graphtools.examples.graphsTypes; - -import java.applet.Applet; -import java.awt.Color; -import java.awt.Graphics; -import java.awt.Image; -import java.awt.image.BufferedImage; -import java.io.File; -import java.util.List; - -import javax.imageio.ImageIO; - -import org.gcube.contentmanagement.graphtools.core.StatisticsGenerator; -import org.gcube.contentmanagement.graphtools.data.conversions.ImageTools; -import org.gcube.contentmanagement.graphtools.plotting.graphs.GaussianDistributionGraph; -import org.gcube.contentmanagement.graphtools.plotting.graphs.ScatterGraphGeneric; -import org.gcube.contentmanagement.graphtools.plotting.graphs.ScatterGraphNumeric; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.portlets.user.timeseries.charts.support.types.GraphGroups; - -import com.thoughtworks.xstream.XStream; -import com.thoughtworks.xstream.io.xml.DomDriver; - - - -public class ExampleImage extends Applet { - - public void paint1(Graphics g) { - BufferedImage bgimg = loadImage(); - g.drawImage(bgimg, 0, 0, this); -// g.fillRect(0, 0, 10, 10); - - } - static int width = 320; - static int height = 280; - - private BufferedImage loadImage() { - String imgFileName = "C:/Users/coro/Desktop/WorkFolder/Workspace/StatisticsExtractor/weather-cloud.png"; - - BufferedImage img = null; - try { - img = ImageIO.read(new File(imgFileName)); - } catch (Exception e) { - e.printStackTrace(); - } - return img; - } - - public void init() { - - - setBackground( Color.LIGHT_GRAY ); - setSize(width, height); - } - - - -// public void paint(Graphics g){ - public static void main(String[] args){ - - try{ - - String table = "ts_161efa00_2c32_11df_b8b3_aa10916debe6"; - String xDimension = "field5"; - String yDimension = "field6"; - String groupDimension = "field1"; - String speciesColumn = "field3"; - String filter1 = "Brown seaweeds"; - - StatisticsGenerator stg = new StatisticsGenerator(); - stg.init("C:/Users/coro/Desktop/WorkFolder/Workspace/StatisticsExtractor/cfg/"); - GraphGroups gg = stg.generateGraphs(200, table, xDimension, yDimension, groupDimension, speciesColumn, filter1); - -// ScatterGraphNumeric series = new ScatterGraphNumeric(""); - -// GaussianDistributionGraph series = new GaussianDistributionGraph(""); - ScatterGraphGeneric series = new ScatterGraphGeneric(""); - - series.renderImages("./saved",width,height,gg); - - AnalysisLogger.getLogger().debug("finished"); - - System.exit(0); - - /* OLD CODE - List image = series.renderGraphGroupImage(width,height,gg); - - Image singleimage = image.get(1); - - - BufferedImage bimage = ImageTools.toBufferedImage(singleimage); - - XStream xStream = new XStream(new DomDriver()); - - String xmlimage = xStream.toXML(singleimage); - - System.out.println(xmlimage); - - File outputfile = new File("saved.png"); - - ImageIO.write(bimage, "png", outputfile); - */ - - }catch(Exception e){ - - e.printStackTrace(); - - } - - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/GaussianDistributionGraph.java b/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/GaussianDistributionGraph.java index 80926d2..40337da 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/GaussianDistributionGraph.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/GaussianDistributionGraph.java @@ -49,7 +49,6 @@ import java.util.Map; import org.gcube.contentmanagement.graphtools.abstracts.GenericStandaloneGraph; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.portlets.user.timeseries.charts.support.types.GraphData; import org.gcube.portlets.user.timeseries.charts.support.types.GraphGroups; import org.gcube.portlets.user.timeseries.charts.support.types.Point; @@ -63,12 +62,13 @@ import org.jfree.data.function.NormalDistributionFunction2D; import org.jfree.data.general.Dataset; import org.jfree.data.general.DatasetUtilities; import org.jfree.data.xy.XYSeriesCollection; - -import com.thoughtworks.xstream.XStream; -import com.thoughtworks.xstream.io.xml.DomDriver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GaussianDistributionGraph extends GenericStandaloneGraph { + private static Logger logger = LoggerFactory.getLogger(GaussianDistributionGraph.class); + private static final long serialVersionUID = 1L; public GaussianDistributionGraph(String title) { @@ -173,7 +173,7 @@ public class GaussianDistributionGraph extends GenericStandaloneGraph { if (variance==0) variance = 0.1; - AnalysisLogger.getLogger().debug("mean "+mean+" variance "+variance); + logger.debug("mean "+mean+" variance "+variance); //build up normal distribution and add to the series NormalDistributionFunction2D normaldistributionfunction2d = new NormalDistributionFunction2D(mean, variance); //make the representation a bit longer @@ -213,7 +213,7 @@ public class GaussianDistributionGraph extends GenericStandaloneGraph { if (variance==0) variance = 0.1; - AnalysisLogger.getLogger().debug("mean "+mean+" variance "+variance); + logger.debug("mean "+mean+" variance "+variance); //build up normal distribution and add to the series NormalDistributionFunction2D normaldistributionfunction2d = new NormalDistributionFunction2D(mean, variance); normalsList.add(normaldistributionfunction2d); diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/PieGraph.java b/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/PieGraph.java index 15817bc..f702221 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/PieGraph.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/PieGraph.java @@ -44,7 +44,6 @@ package org.gcube.contentmanagement.graphtools.plotting.graphs; import java.util.List; import org.gcube.contentmanagement.graphtools.abstracts.GenericStandaloneGraph; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.portlets.user.timeseries.charts.support.types.GraphData; import org.gcube.portlets.user.timeseries.charts.support.types.Point; import org.jfree.chart.ChartFactory; @@ -53,22 +52,19 @@ import org.jfree.data.category.DefaultCategoryDataset; import org.jfree.data.general.Dataset; import org.jfree.data.general.DefaultPieDataset; import org.jfree.util.TableOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PieGraph extends GenericStandaloneGraph { + private static Logger logger = LoggerFactory.getLogger(PieGraph.class); + private static final long serialVersionUID = 1L; public PieGraph(String title) { super(title); } - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } protected Dataset generateDataset() { DefaultPieDataset dataset = new DefaultPieDataset(); @@ -142,7 +138,7 @@ public class PieGraph extends GenericStandaloneGraph { double value = pointslist.get(x).getEntries().get(y).getValue().doubleValue(); value = (value/max)*100; String ylabel = pointslist.get(x).getEntries().get(y).getLabel(); - AnalysisLogger.getLogger().info(xlabel + ":" + ylabel +"->" + value); + logger.info(xlabel + ":" + ylabel +"->" + value); dataset.setValue(xlabel + ":" + ylabel, value); } diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/TimeSeriesGraph.java b/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/TimeSeriesGraph.java index 7add501..41b14e6 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/TimeSeriesGraph.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/plotting/graphs/TimeSeriesGraph.java @@ -49,7 +49,6 @@ import java.util.List; import org.gcube.contentmanagement.graphtools.abstracts.GenericStandaloneGraph; import org.gcube.contentmanagement.graphtools.utils.DateGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.portlets.user.timeseries.charts.support.types.GraphData; import org.gcube.portlets.user.timeseries.charts.support.types.Point; import org.jfree.chart.ChartFactory; @@ -65,9 +64,13 @@ import org.jfree.data.time.TimeSeries; import org.jfree.data.time.TimeSeriesCollection; import org.jfree.data.xy.XYDataset; import org.jfree.ui.RectangleInsets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TimeSeriesGraph extends GenericStandaloneGraph { + private static Logger logger = LoggerFactory.getLogger(TimeSeriesGraph.class); + private static final long serialVersionUID = 1L; private static final String formatYear= "yyyy"; private static final String formatMonth= "MM-yyyy"; @@ -206,7 +209,7 @@ public class TimeSeriesGraph extends GenericStandaloneGraph { else if (granularity.equals(DateGuesser.MONTH)) timeseriesformat = formatMonth; else if (granularity.equals(DateGuesser.DAY)) timeseriesformat = formatDay; - AnalysisLogger.getLogger().debug("TimeSeriesGraph-> granularity "+granularity+" format "+timeseriesformat); + logger.debug("TimeSeriesGraph-> granularity "+granularity+" format "+timeseriesformat); xyseries.add(new Day(new Date(cal.getTimeInMillis())),value); diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/core/Engine.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/core/Engine.java index ad18190..541f1e8 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/core/Engine.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/core/Engine.java @@ -14,13 +14,16 @@ import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.treeStructure import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.treeStructure.chunks.SingletonChunkSet; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.treeStructure.chunks.TimeSeriesChunk; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.treeStructure.chunks.TimeSeriesChunkSet; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.gcube.contentmanagement.lexicalmatcher.utils.MathFunctions; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Engine { + private static Logger logger = LoggerFactory.getLogger(Engine.class); + private String ConfigurationFileNameLocal = "lexicalguesser/hibernate.cfg.xml"; private SessionFactory referenceDBSession; @@ -85,7 +88,7 @@ public class Engine { tsChunkSet = new TimeSeriesChunkSet(config.TimeSeriesChunksToTake, config.chunkSize, unknownSeriesName, unknownSeriesColumn,config, this); } catch (Exception e) { e.printStackTrace(); - AnalysisLogger.getLogger().error("Engine->calcLike-> ERROR could not retrieve time series chunks " + e.getLocalizedMessage()); + logger.error("Engine->calcLike-> ERROR could not retrieve time series chunks " + e.getLocalizedMessage()); } // if we took the ts chunk set correctly perform calculation if (tsChunkSet != null) { @@ -108,7 +111,7 @@ public class Engine { tsChunk.compareToReferenceChunk(scoresTable, refChunk); } catch (Exception e) { e.printStackTrace(); - AnalysisLogger.getLogger().error("Engine->calcLike-> ERROR could not compare time series chunk with reference chunk " + e.getLocalizedMessage()); + logger.error("Engine->calcLike-> ERROR could not compare time series chunk with reference chunk " + e.getLocalizedMessage()); } // take another chunk in the reference chunk set refChunk = refChunkSet.nextChunk(); @@ -150,7 +153,7 @@ public class Engine { ThreadCalculator tc = new ThreadCalculator(tsChunk, refChunkSet,index); Thread t = new Thread(tc); t.start(); -// AnalysisLogger.getLogger().info("ThreadCalculator<-go "+index); +// logger.info("ThreadCalculator<-go "+index); } @@ -178,8 +181,7 @@ public class Engine { } } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().error("Engine->calcLike-> ERROR could not retrieve time series chunks " + e.getLocalizedMessage()); + logger.error("Engine->calcLike-> ERROR could not retrieve time series chunks ",e); } // if we took the ts chunk set correctly perform calculation if (tsChunkSet != null) { @@ -189,7 +191,7 @@ public class Engine { TimeSeriesChunk tsChunk = (TimeSeriesChunk)tsChunkSet.nextChunk(); - AnalysisLogger.getLogger().debug("tsChunk is null "+(tsChunk != null)); + logger.debug("tsChunk is null "+(tsChunk != null)); // for all ts chunks while (tsChunk != null) { @@ -240,7 +242,7 @@ public class Engine { tsChunk.compareToReferenceChunk(scoresTable, refChunk,columnFilter); } catch (Exception e) { e.printStackTrace(); - AnalysisLogger.getLogger().error("Engine->calcLike-> ERROR could not compare time series chunk with reference chunk " + e.getLocalizedMessage()); + logger.error("Engine->calcLike-> ERROR could not compare time series chunk with reference chunk " + e.getLocalizedMessage()); } //if the TimeSeries chunk states the processing must be interrupted, don't perform other comparisons @@ -266,8 +268,8 @@ public class Engine { score = categoryScore.getScore(bestColumn,singletonMatch); } - AnalysisLogger.getLogger().trace("Engine->UpdateScores-> \tBEST SUITABLE COLUMN IS: " + bestColumn); - AnalysisLogger.getLogger().trace("Engine->UpdateScores-> \tBEST SCORE IS: " + score); + logger.trace("Engine->UpdateScores-> \tBEST SUITABLE COLUMN IS: " + bestColumn); + logger.trace("Engine->UpdateScores-> \tBEST SCORE IS: " + score); // order this column if (score > config.categoryDiscardThreshold) { @@ -314,11 +316,11 @@ public class Engine { index++; } - // AnalysisLogger.getLogger().info("chechAndAddColumns -> column to add "+column+" category "+categoryName+" with value "+score+" previous "+(bestScore - 0.5 * bestScore)); + // logger.info("chechAndAddColumns -> column to add "+column+" category "+categoryName+" with value "+score+" previous "+(bestScore - 0.5 * bestScore)); bestColumns.add(index,column); bestScores.add(index,score); bestCategories.add(index,categoryName); - // AnalysisLogger.getLogger().info("chechAndAddColumns -> "+bestCategories); + // logger.info("chechAndAddColumns -> "+bestCategories); } } } @@ -337,10 +339,10 @@ public class Engine { } public void run() { -// AnalysisLogger.getLogger().info("ThreadCalculator->started "+index); +// logger.info("ThreadCalculator->started "+index); makeComparisonsTSChunk2RefChunks(tsChunk, refChunksSet); threadActivity[index]=false; -// AnalysisLogger.getLogger().info("ThreadCalculator>-finished "+index); +// logger.info("ThreadCalculator>-finished "+index); } } diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example1_Species.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example1_Species.java deleted file mode 100644 index 77e2f00..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example1_Species.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.examples; - - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class Example1_Species { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; - String column = "field1"; - String correctFamily = "SPECIES"; - String correctColumn = "SCIENTIFIC_NAME"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example2_Area.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example2_Area.java deleted file mode 100644 index 2755d07..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example2_Area.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.examples; - - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class Example2_Area { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; - String column = "field3"; - String correctFamily = "AREA"; - String correctColumn = "NAME_EN"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example3_SingleMatchShark.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example3_SingleMatchShark.java deleted file mode 100644 index 44cb831..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example3_SingleMatchShark.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.examples; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - - -public class Example3_SingleMatchShark { - - public static void main(String[] args) { - - try { - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String singleton = "shark"; - String family = "species"; - String column = "name_en"; - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - - //CHANGE THIS TO ENHANCE THE RECALL - conf.setEntryAcceptanceThreshold(30); - conf.setReferenceChunksToTake(-1); - conf.setTimeSeriesChunksToTake(-1); - conf.setUseSimpleDistance(false); - - guesser.runGuesser(configPath, singleton, conf, family,column ); - ArrayList detailedResults = guesser.getDetailedMatches(); - - AnalysisLogger.getLogger().warn("Detailed Match on Name :"+singleton); - - CategoryGuesser.showResults(detailedResults); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example4_SingleMatchMitella.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example4_SingleMatchMitella.java deleted file mode 100644 index 92c4945..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example4_SingleMatchMitella.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.examples; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - - -public class Example4_SingleMatchMitella { - - public static void main(String[] args) { - - try { - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String singleton = "Mitella pollicipes"; -// String singleton = "policipes"; - String family = "species"; - String column = "scientific_name"; - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - - //CHANGE THIS TO ENHANCE THE RECALL - conf.setEntryAcceptanceThreshold(30); - conf.setReferenceChunksToTake(-1); - conf.setTimeSeriesChunksToTake(-1); - conf.setUseSimpleDistance(false); - - guesser.runGuesser(configPath, singleton, conf, family,column ); - ArrayList detailedResults = guesser.getDetailedMatches(); - - AnalysisLogger.getLogger().warn("Detailed Match on Name :"+singleton); - - CategoryGuesser.showResults(detailedResults); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example5_SingleMatchMitella.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example5_SingleMatchMitella.java deleted file mode 100644 index 2394228..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example5_SingleMatchMitella.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.examples; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - - -public class Example5_SingleMatchMitella { - - public static void main(String[] args) { - - try { - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String singleton = "Mirella policepes"; - String family = "species"; - String column = "scientific_name"; - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - - //CHANGE THIS TO ENHANCE THE RECALL - conf.setEntryAcceptanceThreshold(30); - conf.setReferenceChunksToTake(-1); - conf.setTimeSeriesChunksToTake(-1); - conf.setUseSimpleDistance(false); - - guesser.runGuesser(configPath, singleton, conf, family,column ); - ArrayList detailedResults = guesser.getDetailedMatches(); - - AnalysisLogger.getLogger().warn("Detailed Match on Name :"+singleton); - - CategoryGuesser.showResults(detailedResults); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/ExampleGuessingExternalCfg.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/ExampleGuessingExternalCfg.java deleted file mode 100644 index 9645dc8..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/ExampleGuessingExternalCfg.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.examples; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; - -public class ExampleGuessingExternalCfg { - - public static void main(String[] args) { - - try { - - String configPath = "./"; - CategoryGuesser guesser = new CategoryGuesser(); - - - //bench 1 - System.out.println("----------------------BENCH 1-------------------------"); - String seriesName = "import_532bba80_1c8f_11df_a4ee_87804054691e"; - String column = "field2"; - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - conf.setCategoryDiscardDifferencialThreshold(10); - conf.setCategoryDiscardThreshold(0); - conf.setChunkSize(25); - conf.setEntryAcceptanceThreshold(50); - conf.setNumberOfThreadsToUse(2); - conf.setRandomTake(true); - conf.setReferenceChunksToTake(20); - conf.setTimeSeriesChunksToTake(1); - conf.setUseSimpleDistance(false); - - //database Parameters - conf.setDatabaseUserName("root"); - conf.setDatabasePassword("ash_ash80"); - conf.setDatabaseDriver("com.mysql.jdbc.Driver"); - conf.setDatabaseURL("jdbc:mysql://localhost/timeseries"); - conf.setDatabaseDialect("org.hibernate.dialect.MySQLDialect"); - conf.setDatabaseAutomaticTestTable("connectiontesttable"); - conf.setDatabaseIdleConnectionTestPeriod("3600"); - - //reference parameters - conf.setReferenceTable("reference_table"); - conf.setReferenceColumn("table_name"); - conf.setIdColumn("id"); - conf.setNameHuman("name_human"); - conf.setDescription("description"); - - guesser.init(conf); - - guesser.runGuesser(seriesName, column, conf); - ArrayList results = guesser.getClassification(); - CategoryGuesser.showResults(results); - - System.out.println("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScores.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScores.java index 285b790..5d27e70 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScores.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScores.java @@ -7,13 +7,16 @@ import java.util.ArrayList; import java.util.HashMap; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.MathFunctions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; //score relative to a certain category and column public class CategoryScores { + private static Logger logger = LoggerFactory.getLogger(CategoryScores.class); + // column names vs percentage private HashMap columnsScore; @@ -152,7 +155,7 @@ public class CategoryScores { try { score = columnsScore.get(column); } catch (Exception e) { - AnalysisLogger.getLogger().error("ERROR in getting SCORE " + e.getLocalizedMessage()); + logger.error("ERROR in getting SCORE ",e); } if (bestscore.compareTo(score) < 0) { bestscore = score; @@ -175,7 +178,7 @@ public class CategoryScores { try { score = columnsScore.get(column); } catch (Exception e) { - AnalysisLogger.getLogger().error("ERROR in getting SCORE " + e.getLocalizedMessage()); + logger.error("ERROR in getting SCORE ",e); } // find best place where to put column diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScoresOld.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScoresOld.java index 4f39044..f93dc23 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScoresOld.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/CategoryScoresOld.java @@ -6,12 +6,15 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; //score relative to a certain category and column public class CategoryScoresOld { + private static Logger logger = LoggerFactory.getLogger(CategoryScoresOld.class); + // column names vs percentage private HashMap columnsScore; @@ -52,7 +55,7 @@ public class CategoryScoresOld { if (percentage == null) percentage = BigDecimal.ZERO; - AnalysisLogger.getLogger().trace("getScore -> Score for "+columnName+": " + percentage + " vs " + maximumElements); + logger.trace("getScore -> Score for "+columnName+": " + percentage + " vs " + maximumElements); percentage = percentage.divide(maximumElements, 2, BigDecimal.ROUND_DOWN); } catch (ArithmeticException e) { percentage = BigDecimal.ZERO; @@ -78,7 +81,7 @@ public class CategoryScoresOld { try { score = columnsScore.get(column); } catch (Exception e) { - AnalysisLogger.getLogger().error("ERROR in getting SCORE " + e.getLocalizedMessage()); + logger.error("ERROR in getting SCORE ",e); } if (bestscore.compareTo(score) < 0) { bestscore = score; @@ -101,7 +104,7 @@ public class CategoryScoresOld { try { score = columnsScore.get(column); } catch (Exception e) { - AnalysisLogger.getLogger().error("ERROR in getting SCORE " + e.getLocalizedMessage()); + logger.error("ERROR in getting SCORE ",e); } // find best place where to put column diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/DBObjectTranslator.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/DBObjectTranslator.java index 4441a32..f6fb129 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/DBObjectTranslator.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/DBObjectTranslator.java @@ -5,16 +5,15 @@ import java.util.ArrayList; import java.util.List; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.DataTypeRecognizer; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DBObjectTranslator { - public static void main(String[] args) { - - } - + private static Logger logger = LoggerFactory.getLogger(DBObjectTranslator.class); + public ArrayList relations; public ArrayList categories; @@ -44,7 +43,7 @@ public class DBObjectTranslator { BigInteger resultcount = (BigInteger) result; totalEntries = totalEntries.add(resultcount); count = resultcount; - AnalysisLogger.getLogger().trace("DBObjectTranslator->calculateTotalEntries: Time Series " + timeSeriesName + " total " + totalEntries); + logger.trace("DBObjectTranslator->calculateTotalEntries: Time Series " + timeSeriesName + " total " + totalEntries); } catch (Exception e) { } } @@ -56,7 +55,7 @@ public class DBObjectTranslator { // String query = "SELECT distinct "+timeSeriesColumn+" FROM "+timeSeriesName+" r limit "+min+","+numberOfElements; String query = "SELECT distinct " + timeSeriesColumn + " FROM " + timeSeriesName + " r limit " + numberOfElements + " offset " + min; - AnalysisLogger.getLogger().trace("DBObjectTranslator->query: " + query); + logger.trace("DBObjectTranslator->query: " + query); List resultSet = DatabaseFactory.executeSQLQuery(query, dbSession); ArrayList column = new ArrayList(); @@ -69,14 +68,14 @@ public class DBObjectTranslator { column.add(value); - // AnalysisLogger.getLogger().debug("DBObjectTranslator->retrieveColumnRange: Column Element Added " + value); + // logger.debug("DBObjectTranslator->retrieveColumnRange: Column Element Added " + value); } catch (Exception e) { e.printStackTrace(); - AnalysisLogger.getLogger().trace("DBObjectTranslator->retrieveTimeSeriesEntries: Error in adding entry :" + e.getLocalizedMessage()); + logger.trace("DBObjectTranslator->retrieveTimeSeriesEntries: Error in adding entry :" + e.getLocalizedMessage()); } } - AnalysisLogger.getLogger().trace("DBObjectTranslator->retrieveColumnRange: Column " + column.toString()); + logger.trace("DBObjectTranslator->retrieveColumnRange: Column " + column.toString()); return column; } @@ -105,7 +104,7 @@ public class DBObjectTranslator { // String query = "SELECT DISTINCT * FROM " + timeSeriesName + " r where id>=" + min.toString() + " and id<=" + max.toString(); // String query = "SELECT DISTINCT * FROM " + timeSeriesName + " r limit "+min+","+numberOfElements; String query = "SELECT DISTINCT * FROM " + timeSeriesName + " r limit " + numberOfElements + " offset " + min; - AnalysisLogger.getLogger().trace("DBObjectTranslator->retrieveEntries: query " + query); + logger.trace("DBObjectTranslator->retrieveEntries: query " + query); List resultSet = DatabaseFactory.executeSQLQuery(query, dbSession); @@ -126,15 +125,15 @@ public class DBObjectTranslator { } // add entry currentEntries.add(entry); - // AnalysisLogger.getLogger().debug("DBObjectTranslator->retrieveEntries: Entry Added " + entry.toString()); + // logger.debug("DBObjectTranslator->retrieveEntries: Entry Added " + entry.toString()); } catch (Exception e) { // e.printStackTrace(); - AnalysisLogger.getLogger().trace("DBObjectTranslator->retrieveEntries: Error in adding entry :" + e.getLocalizedMessage()); + logger.trace("DBObjectTranslator->retrieveEntries: Error in adding entry :" + e.getLocalizedMessage()); } } } -// AnalysisLogger.getLogger().trace("DBObjectTranslator->retrieveEntries: Entries " + currentEntries); +// logger.trace("DBObjectTranslator->retrieveEntries: Entries " + currentEntries); return currentEntries; } @@ -152,7 +151,7 @@ public class DBObjectTranslator { } if (re != null) { relations.add(re); - AnalysisLogger.getLogger().trace("DBObjectTranslator->buildRelationsEdges: add relation " + re.toString()); + logger.trace("DBObjectTranslator->buildRelationsEdges: add relation " + re.toString()); } } } @@ -180,7 +179,7 @@ public class DBObjectTranslator { } if (cat != null) { categories.add(cat); - AnalysisLogger.getLogger().trace("DBObjectTranslator->buildCategories: add category " + cat.toString()); + logger.trace("DBObjectTranslator->buildCategories: add category " + cat.toString()); } } } @@ -208,7 +207,7 @@ public class DBObjectTranslator { Category to = getCategoryfromIndex(re.getTo()); re.setCategoryFrom(from.getName()); re.setCategoryTo(to.getName()); - AnalysisLogger.getLogger().trace("DBObjectTranslator->populateRelationWithCategories: modified Relation " + re.toString()); + logger.trace("DBObjectTranslator->populateRelationWithCategories: modified Relation " + re.toString()); } } @@ -225,7 +224,7 @@ public class DBObjectTranslator { BigInteger resultcount = (BigInteger) result; re.setWeigth(resultcount); totalRelationElements = totalRelationElements.add(resultcount); - AnalysisLogger.getLogger().trace("DBObjectTranslator->calculateRelationWeights: Relation " + re.getName() + " weight " + re.getWeigth()); + logger.trace("DBObjectTranslator->calculateRelationWeights: Relation " + re.getName() + " weight " + re.getWeigth()); } catch (Exception e) { } } @@ -246,7 +245,7 @@ public class DBObjectTranslator { BigInteger resultcount = (BigInteger) result; cat.setNumberOfElements(resultcount); totalCatElements = totalCatElements.add(resultcount); - AnalysisLogger.getLogger().trace("DBObjectTranslator->calculateCategoriesWeights: Category " + cat.getName() + " weight " + cat.getNumberOfElements() + " total " + totalCatElements); + logger.trace("DBObjectTranslator->calculateCategoriesWeights: Category " + cat.getName() + " weight " + cat.getNumberOfElements() + " total " + totalCatElements); } catch (Exception e) { } } @@ -256,7 +255,7 @@ public class DBObjectTranslator { public void buildCategoriesStructure(SessionFactory dbSession, String referenceTable, String referenceColumn, String idColumn, String nameHuman, String description) { buildCategories(dbSession, referenceTable, referenceColumn, idColumn, nameHuman, description); calculateCategoriesWeights(dbSession); - AnalysisLogger.getLogger().trace("DBObjectTranslator->buildWholeStructure: Total Categories Elements " + totalCatElements + " Total Relation Elements " + totalRelationElements); + logger.trace("DBObjectTranslator->buildWholeStructure: Total Categories Elements " + totalCatElements + " Total Relation Elements " + totalRelationElements); } public void buildWholeStructure(SessionFactory dbSession, String referenceTable, String referenceColumn, String idColumn, String nameHuman, String description) { @@ -267,7 +266,7 @@ public class DBObjectTranslator { calculateRelationWeights(dbSession); calculateCategoriesWeights(dbSession); - AnalysisLogger.getLogger().trace("DBObjectTranslator->buildWholeStructure: Total Categories Elements " + totalCatElements + " Total Relation Elements " + totalRelationElements); + logger.trace("DBObjectTranslator->buildWholeStructure: Total Categories Elements " + totalCatElements + " Total Relation Elements " + totalRelationElements); } } diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ChunkSet.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ChunkSet.java index ed849ad..a45ea17 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ChunkSet.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ChunkSet.java @@ -7,11 +7,13 @@ import java.util.ArrayList; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.Engine; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.MathFunctions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class ChunkSet { + private static Logger logger = LoggerFactory.getLogger(ChunkSet.class); protected String seriesName; protected String seriesColumn; @@ -52,7 +54,7 @@ public abstract class ChunkSet { public void generateChunkSet() throws Exception{ - AnalysisLogger.getLogger().trace("ChunkSet->generateChunkSet-> \tGenerating Chunk Set for " + seriesName+ " "+seriesColumn); + logger.trace("ChunkSet->generateChunkSet-> \tGenerating Chunk Set for " + seriesName+ " "+seriesColumn); int numberOfChunks = calculateNumberOfCycles(); //generate chunks to be processed chunkSet = MathFunctions.generateRandoms(maxNumberOfChunks, 0, numberOfChunks); diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ReferenceChunk.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ReferenceChunk.java index 1f2ad15..1d2930a 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ReferenceChunk.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/ReferenceChunk.java @@ -7,12 +7,13 @@ import java.util.ArrayList; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.Engine; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.DBObjectTranslator; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Entry; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ReferenceChunk extends Chunk{ - + private static Logger logger = LoggerFactory.getLogger(ReferenceChunk.class); private String categoryName; private String categoryTableName; @@ -27,7 +28,7 @@ public class ReferenceChunk extends Chunk{ categoryName = CategoryName; categoryTableName = CategoryTableName; startPoint = StartPoint; - AnalysisLogger.getLogger().trace("ReferenceChunk-> \t\tTOOK CATEGORY CHUNK FOR CATEGORY: " + categoryName+" - index : "+startPoint); + logger.trace("ReferenceChunk-> \t\tTOOK CATEGORY CHUNK FOR CATEGORY: " + categoryName+" - index : "+startPoint); } diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/TimeSeriesChunk.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/TimeSeriesChunk.java index 86c816a..7a304d8 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/TimeSeriesChunk.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/chunks/TimeSeriesChunk.java @@ -12,12 +12,14 @@ import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.DBObjectTranslator; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Entry; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DistanceCalculator; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TimeSeriesChunk extends Chunk{ + private static Logger logger = LoggerFactory.getLogger(TimeSeriesChunk.class); private ArrayList columnEntries; private String columnType; @@ -49,7 +51,7 @@ public class TimeSeriesChunk extends Chunk{ columnEntries = dbo.retrieveTimeSeriesEntries(sess, timeSeriesName, timeSeriesColumn, start, ChunkSize); if (ColumnType==null){ columnType = DataTypeRecognizer.guessType(columnEntries); - AnalysisLogger.getLogger().trace("TimeSeriesChunk-> GUESSED TYPE " + columnType + " FOR COLUMN "+timeSeriesColumn); + logger.trace("TimeSeriesChunk-> GUESSED TYPE " + columnType + " FOR COLUMN "+timeSeriesColumn); } mustInterrupt = false; config = Config; @@ -62,7 +64,7 @@ public class TimeSeriesChunk extends Chunk{ columnEntries.add(singletonString); if (ColumnType==null){ columnType = DataTypeRecognizer.guessType(columnEntries); - AnalysisLogger.getLogger().trace("TimeSeriesChunk-> GUESSED TYPE " + columnType + " FOR SINGLETON "+singletonString); + logger.trace("TimeSeriesChunk-> GUESSED TYPE " + columnType + " FOR SINGLETON "+singletonString); } mustInterrupt = false; config = Config; @@ -108,17 +110,17 @@ public class TimeSeriesChunk extends Chunk{ // perform calculation only if the column type is the same if (types.get(referenceColumn).equals(columnType)&&((ColumnFilter==null)||(ColumnFilter.equalsIgnoreCase(referenceColumn)))) { -// AnalysisLogger.getLogger().debug("CategoryOrderedList->checkAllEntriesOnEntireCategory-> REFERENCE COLUMN "+referenceColumn+" HAS TYPE "+types.get(referenceColumn)); +// logger.debug("CategoryOrderedList->checkAllEntriesOnEntireCategory-> REFERENCE COLUMN "+referenceColumn+" HAS TYPE "+types.get(referenceColumn)); // take the attribute value of the entry String attribute = attributes.get(referenceColumn); // calculate the distance between the unknown entry and the attribute DistanceCalculator d = new DistanceCalculator(); double percentage = d.CD(config.useSimpleDistance, timeSeriesElement, attribute, isSingleton, isSingleton) * 100f; -// AnalysisLogger.getLogger().debug("CategoryOrderedList->checkUnkEntriesOnEntireCategory-> Percentage between " +timeSeriesElement + " and " + attribute + " is: "+percentage ); +// logger.debug("CategoryOrderedList->checkUnkEntriesOnEntireCategory-> Percentage between " +timeSeriesElement + " and " + attribute + " is: "+percentage ); // if they are similar if (percentage > config.entryAcceptanceThreshold) { // if (catChunk.getCategoryName().equals("COUNTRY_OLD")) - AnalysisLogger.getLogger().trace("TimeSeriesChunk->compareToCategoryChunk-> \t\tPercentage between " + timeSeriesElement + " vs. " + attribute + " is: " + percentage+" in "+catChunk.getCategoryName()+":"+referenceColumn); + logger.trace("TimeSeriesChunk->compareToCategoryChunk-> \t\tPercentage between " + timeSeriesElement + " vs. " + attribute + " is: " + percentage+" in "+catChunk.getCategoryName()+":"+referenceColumn); categoryScores.incrementScore(referenceColumn, (float)percentage,anotherReference); @@ -138,7 +140,7 @@ public class TimeSeriesChunk extends Chunk{ detailedResults.add(index, new SingleResult(attribute, null, percentage,null,"0")); } else{ - AnalysisLogger.getLogger().trace("TimeSeriesChunk->compareToCategoryChunk-> "+categoryScores.showScores()); + logger.trace("TimeSeriesChunk->compareToCategoryChunk-> "+categoryScores.showScores()); } //if exact match is reached, exit if ((percentage==100)&&(isSingleton)) diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/graph/GraphDisplayer.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/graph/GraphDisplayer.java index 8df91c1..a0df788 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/graph/GraphDisplayer.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/treeStructure/graph/GraphDisplayer.java @@ -5,7 +5,6 @@ import java.awt.Color; import java.awt.Dimension; import java.awt.Rectangle; import java.awt.geom.Rectangle2D; -import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -13,13 +12,17 @@ import java.util.Random; import javax.swing.JApplet; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.jgraph.JGraph; import org.jgraph.graph.DefaultGraphCell; import org.jgraph.graph.GraphConstants; import org.jgrapht.ext.JGraphModelAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GraphDisplayer extends JApplet { + + private static Logger logger = LoggerFactory.getLogger(GraphDisplayer.class); + private static final Color DEFAULT_BG_COLOR = Color.decode("#FAFBFF"); private static final Dimension DEFAULT_SIZE = new Dimension(530, 320); @@ -64,7 +67,7 @@ public class GraphDisplayer extends JApplet { } public void init() { - AnalysisLogger.getLogger().debug("INIZIALIZZATO!"); + logger.debug("INIZIALIZZATO!"); JGraph jgraph = new JGraph(m_jgAdapter); @@ -72,7 +75,7 @@ public class GraphDisplayer extends JApplet { getContentPane().add(jgraph); resize(DEFAULT_SIZE); - AnalysisLogger.getLogger().debug("RESIZED!"); + logger.debug("RESIZED!"); } public void generateGraph() { diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/CategoryGuesser.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/CategoryGuesser.java index 76ed6e3..e03bc7f 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/CategoryGuesser.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/CategoryGuesser.java @@ -13,27 +13,26 @@ import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.DBObject import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.TSObjectTransformer; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.interfaces.Reference; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CategoryGuesser { - /** - * @param args - * @throws Exception - */ + private static Logger logger = LoggerFactory.getLogger(CategoryGuesser.class); + private final static int MAXRESULTS = 10; public static void showResults(ArrayList results) { - AnalysisLogger.getLogger().warn("CLASSIFICATION RESULT:\n"); + logger.warn("CLASSIFICATION RESULT:\n"); int i = 1; for (SingleResult result : results) { if (result.getColumn() != null) - AnalysisLogger.getLogger().warn(i + ": " + result.getCategory() + " - " + result.getColumn() + " ; SCORE: " + result.getStringScore() + "%"); + logger.warn(i + ": " + result.getCategory() + " - " + result.getColumn() + " ; SCORE: " + result.getStringScore() + "%"); else - AnalysisLogger.getLogger().warn(i + ": " + result.getCategory() + " ; SCORE: " + result.getStringScore() + "%"); + logger.warn(i + ": " + result.getCategory() + " ; SCORE: " + result.getStringScore() + "%"); i++; } @@ -57,7 +56,7 @@ public class CategoryGuesser { String result = results.toString(); showResults(results); - AnalysisLogger.getLogger().info("CLASSIFICATION RESULT " + result + " " + CategoryGuesser.resultString(result, correctFamily, correctColumn)); + logger.info("CLASSIFICATION RESULT " + result + " " + CategoryGuesser.resultString(result, correctFamily, correctColumn)); if (CategoryGuesser.CheckCompleteResult(result, correctFamily, correctColumn)) columnscore++; @@ -70,7 +69,7 @@ public class CategoryGuesser { double percColumn = ((double) columnscore / (double) attempts) * 100; double percFamily = ((double) familyscore / (double) attempts) * 100; - AnalysisLogger.getLogger().info("->ACCURACY ON FAMILY " + correctFamily + ":" + percFamily + " ACCURACY ON COLUMN " + correctColumn + ":" + percColumn); + logger.info("->ACCURACY ON FAMILY " + correctFamily + ":" + percFamily + " ACCURACY ON COLUMN " + correctColumn + ":" + percColumn); } public static String resultString(String result, String family, String column) { @@ -142,9 +141,8 @@ public class CategoryGuesser { public void init(String categoryFilter, String columnFilter, LexicalEngineConfiguration externalConfig) throws Exception { String cfgFileCompletePath = cfgFile; - AnalysisLogger.setLogger(LogFile); - AnalysisLogger.getLogger().trace("******************INITIALIZING******************"); + logger.trace("******************INITIALIZING******************"); config = new LexicalEngineConfiguration(); config.configureByStream(cfgFileCompletePath); @@ -159,7 +157,7 @@ public class CategoryGuesser { DBObjectTranslator dbo = new DBObjectTranslator(); if (col == null) { - AnalysisLogger.getLogger().trace("******************Order Category******************"); + logger.trace("******************Order Category******************"); if (externalConfig == null) externalConfig = new LexicalEngineConfiguration(); @@ -169,7 +167,7 @@ public class CategoryGuesser { dbo.buildCategoriesStructure(dbSession, externalConfig.getReferenceTable(), externalConfig.getReferenceColumn(), externalConfig.getIdColumn(), externalConfig.getNameHuman(), externalConfig.getDescription()); col = TSObjectTransformer.transform2List(dbo, config, categoryFilter); - AnalysisLogger.getLogger().trace("***************End Ordering********************"); + logger.trace("***************End Ordering********************"); originalCol = col.generateNovelList(); } else { col = originalCol.generateNovelList(); @@ -181,7 +179,6 @@ public class CategoryGuesser { public void initSingleMatcher(LexicalEngineConfiguration externalConfig, String ColumnFilter) throws Exception { String cfgFileCompletePath = cfgFile; - AnalysisLogger.setLogger(LogFile); config = new LexicalEngineConfiguration(); config.configureByStream(cfgFileCompletePath); @@ -220,9 +217,8 @@ public class CategoryGuesser { public void runGuesser(String seriesName, String columnName, LexicalEngineConfiguration externalConfig, String CategoryFilter, String ColumnFilter, String SingletonString) throws Exception { SessionFactory dbSession = null; String cfgFileCompletePath = cfgFile; - AnalysisLogger.setLogger(LogFile); - AnalysisLogger.getLogger().debug("Guessing Table " + seriesName + " column " + columnName); + logger.debug("Guessing Table " + seriesName + " column " + columnName); if (externalConfig != null) { config = new LexicalEngineConfiguration(); config.configureByStream(cfgFileCompletePath); @@ -247,7 +243,7 @@ public class CategoryGuesser { //modification of 10/10/11 calculate structure each time // if (col == null) { - AnalysisLogger.getLogger().trace("******************Order Category******************"); + logger.trace("******************Order Category******************"); if (externalConfig !=null){ ArrayList externalcategories = externalConfig.getCategories(); if ((externalcategories!=null) && (externalcategories.size()>0)){ @@ -259,7 +255,7 @@ public class CategoryGuesser { } col = TSObjectTransformer.transform2List(dbo, config, CategoryFilter); - AnalysisLogger.getLogger().trace("***************End Ordering********************"); + logger.trace("***************End Ordering********************"); originalCol = col.generateNovelList(); /* } else { @@ -267,7 +263,7 @@ public class CategoryGuesser { } */ - AnalysisLogger.getLogger().warn("Starting Calculation...wait..."); + logger.warn("Starting Calculation...wait..."); long t0 = System.currentTimeMillis(); @@ -284,16 +280,16 @@ public class CategoryGuesser { checkingResults = getClassification(); while ((checkingResults == null || checkingResults.size() == 0) && (triesCounter < maxTriesClassification)) { - AnalysisLogger.getLogger().warn("..another processing pass is required. Attempt number " + (triesCounter + 1)); + logger.warn("..another processing pass is required. Attempt number " + (triesCounter + 1)); triesCounter++; float differencialThr = config.getCategoryDiscardDifferencialThreshold(); float acceptanceThr = config.getEntryAcceptanceThreshold(); // reduce the thresholds of 10 points and recalculate config.setCategoryDiscardDifferencialThreshold(Math.max(differencialThr - 20, 0)); config.setEntryAcceptanceThreshold(Math.max(acceptanceThr - 20, 0)); - AnalysisLogger.getLogger().trace("Performing next processing pass"); + logger.trace("Performing next processing pass"); runGuesser(seriesName, columnName, externalConfig, CategoryFilter, ColumnFilter, SingletonString); - AnalysisLogger.getLogger().debug("End processing pass"); + logger.debug("End processing pass"); // if (oneshotMode) // checkingResults = getClassification(); @@ -306,7 +302,7 @@ public class CategoryGuesser { long t1 = System.currentTimeMillis() - t0; - AnalysisLogger.getLogger().warn("...End Calculation in " + t1 + "ms"); + logger.warn("...End Calculation in " + t1 + "ms"); triesCounter = 0; lastResults=checkingResults; @@ -416,7 +412,7 @@ public class CategoryGuesser { // multiply for impotance BigInteger catElements = col.getScoresTable().get(processor.bestCategories.get(i)).getCategoryElements(); - // AnalysisLogger.getLogger().warn("\t elements "+catElements+" sum "+sumElements); + // logger.warn("\t elements "+catElements+" sum "+sumElements); double weight = new BigDecimal(catElements).divide(sumElements, 2, BigDecimal.ROUND_HALF_UP).doubleValue(); @@ -429,7 +425,7 @@ public class CategoryGuesser { else if (weight < 0.05) weight = 0.05; - AnalysisLogger.getLogger().warn("WEIGHT FOR CATEGORY " + processor.bestCategories.get(i) + "-" + processor.bestColumns.get(i) + " : " + weight + " SCORE " + score); + logger.warn("WEIGHT FOR CATEGORY " + processor.bestCategories.get(i) + "-" + processor.bestColumns.get(i) + " : " + weight + " SCORE " + score); // recalculate weights score = score * weight; @@ -441,21 +437,21 @@ public class CategoryGuesser { subscores.add(score); } - // AnalysisLogger.getLogger().warn("MAX SCORE "+maxscore); + // logger.warn("MAX SCORE "+maxscore); for (int i = 0; i < size; i++) { // double score = processor.bestScores.get(i); double score = subscores.get(i); - // AnalysisLogger.getLogger().warn("SCORE FOR CATEGORY "+processor.bestCategories.get(i)+" -COLUMN : "+processor.bestColumns.get(i)+" - "+score); + // logger.warn("SCORE FOR CATEGORY "+processor.bestCategories.get(i)+" -COLUMN : "+processor.bestColumns.get(i)+" - "+score); // normalizing percentages!!! score = (score / (maxscore + ((size > 1) ? 1 : 0))) * 100; - // AnalysisLogger.getLogger().warn("SCORE FOR CATEGORY "+processor.bestCategories.get(i)+" -COLUMN : "+processor.bestColumns.get(i)+" - "+score); + // logger.warn("SCORE FOR CATEGORY "+processor.bestCategories.get(i)+" -COLUMN : "+processor.bestColumns.get(i)+" - "+score); if (score > config.categoryDiscardDifferencialThreshold) { - // AnalysisLogger.getLogger().warn("SCORE "+score); + // logger.warn("SCORE "+score); // insert into the right place int index = results.size(); int j = 0; diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/StarGraphExtraction.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/StarGraphExtraction.java deleted file mode 100644 index df43655..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/StarGraphExtraction.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.run; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.DBObjectTranslator; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.TSObjectTransformer; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; -import org.hibernate.SessionFactory; - -public class StarGraphExtraction { - - /** - * @param args - */ - public static void main(String[] args) { - try { - RunMain(); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } - - private final static String ConfigurationFileNameLocal = "hibernate.cfg.xml"; - - public static void RunMain() throws Exception{ - AnalysisLogger.setLogger("./ALog.properties"); - - //configurazione DB - inizializzo la sessione e mi connetto - SessionFactory dbSession = DatabaseFactory.initDBConnection(ConfigurationFileNameLocal); - DBObjectTranslator dbo = new DBObjectTranslator(); - dbo.buildWholeStructure(dbSession,null,null,null,null,null); - TSObjectTransformer.transform2Graph(dbo); - - } -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleGuesser.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleGuesser.java deleted file mode 100644 index fefd0da..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleGuesser.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; - -public class ExampleGuesser { - - public static void main(String[] args) { - - try { - String seriesName = "IMPORT_ecd2e3a0_ee90_11e0_be9e_90f3621758ee"; - String column = "field4"; - - - CategoryGuesser guesser = new CategoryGuesser(); - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - ArrayList categories = new ArrayList(); - //human name, index, table name, description - categories.add(new Category("COUNTRY_OLD","39c98800-dd3c-11e0-b8d1-d1e2e7ba4f9d","rdf39c98800dd3c11e0b8d1d1e2e7ba4f9d","country")); - categories.add(new Category("CONTINENT_OLD","1d5d51f0-dd42-11e0-b8d3-d1e2e7ba4f9d","rdf1d5d51f0dd4211e0b8d3d1e2e7ba4f9d","continent reference data")); - categories.add(new Category("SPECIES_OLD","0a7fb500-dd3d-11e0-b8d1-d1e2e7ba4f9d","rdf0a7fb500dd3d11e0b8d1d1e2e7ba4f9d","species")); - categories.add(new Category("CodeListCountry","4c8d93a0-edc2-11e0-93e4-f6a9821baa29","rdf4c8d93a0edc211e093e4f6a9821baa29","Country")); - categories.add(new Category("CL_DIVISION","1140bdf0-dd2c-11e0-9220-ae17b3db32b7","rdf1140bdf0dd2c11e09220ae17b3db32b7","undefined")); - categories.add(new Category("CL_ASFIS_TAX","f87360f0-d9f9-11e0-ba05-d9adb0db767c","rdff87360f0d9f911e0ba05d9adb0db767c","undefined")); - conf.setCategories(categories); - - //database Parameters - conf.setDatabaseUserName("gcube"); - conf.setDatabasePassword("d4science2"); - conf.setDatabaseDriver("org.postgresql.Driver"); - conf.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - - guesser.runGuesser(seriesName, column, conf); - ArrayList classification = guesser.getLastResults(); - - //only for debug - guesser.showResults(classification); - - guesser.shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleLexicalMatcher.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleLexicalMatcher.java deleted file mode 100644 index 0781028..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/ExampleLexicalMatcher.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class ExampleLexicalMatcher { - - public static void main(String[] args) { - - try { - - String singleton = "Faroe Island"; - String family = "COUNTRY_OLD"; - String column = "field6"; - - CategoryGuesser guesser = new CategoryGuesser(); - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - - ArrayList categories = new ArrayList(); - //human name, index, table name, description - categories.add(new Category("COUNTRY_OLD","39c98800-dd3c-11e0-b8d1-d1e2e7ba4f9d","rdf39c98800dd3c11e0b8d1d1e2e7ba4f9d","country")); - categories.add(new Category("CONTINENT_OLD","1d5d51f0-dd42-11e0-b8d3-d1e2e7ba4f9d","rdf1d5d51f0dd4211e0b8d3d1e2e7ba4f9d","continent reference data")); - categories.add(new Category("SPECIES_OLD","0a7fb500-dd3d-11e0-b8d1-d1e2e7ba4f9d","rdf0a7fb500dd3d11e0b8d1d1e2e7ba4f9d","species")); - categories.add(new Category("CodeListCountry","4c8d93a0-edc2-11e0-93e4-f6a9821baa29","rdf4c8d93a0edc211e093e4f6a9821baa29","Country")); - categories.add(new Category("CL_DIVISION","1140bdf0-dd2c-11e0-9220-ae17b3db32b7","rdf1140bdf0dd2c11e09220ae17b3db32b7","undefined")); - categories.add(new Category("CL_ASFIS_TAX","f87360f0-d9f9-11e0-ba05-d9adb0db767c","rdff87360f0d9f911e0ba05d9adb0db767c","undefined")); - conf.setCategories(categories); - - //CHANGE THIS TO ENHANCE THE RECALL - conf.setEntryAcceptanceThreshold(30); - conf.setReferenceChunksToTake(-1); - conf.setTimeSeriesChunksToTake(-1); - conf.setUseSimpleDistance(false); - - //database Parameters - conf.setDatabaseUserName("gcube"); - conf.setDatabasePassword("d4science2"); - conf.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - - guesser.runGuesser(singleton, conf, family,column ); - ArrayList detailedResults = guesser.getDetailedMatches(); - AnalysisLogger.getLogger().warn("Detailed Match on Name :"+singleton); - - CategoryGuesser.showResults(detailedResults); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestExternalCfgProduction.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestExternalCfgProduction.java deleted file mode 100644 index 28fc72a..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestExternalCfgProduction.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test; - -import java.util.ArrayList; -import java.util.Properties; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class TestExternalCfgProduction { - - public static void main(String[] args) { - - try { - int attempts = 1; - -// new Properties().load(ClassLoader.getSystemResourceAsStream("lexicalguesser/lexicalGuesser.properties")); - - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - - String seriesName = "IMPORT_ecd2e3a0_ee90_11e0_be9e_90f3621758ee"; - String column = "field4"; - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - conf.setReferenceTable("codelist1733371938"); - conf.setReferenceColumn("ifield14"); - conf.setNameHuman("ifield1"); - conf.setIdColumn("ifield0"); - conf.setDescription("ifield2"); - - - ArrayList categories = new ArrayList(); - //human name, index, table name, description - - categories.add(new Category("COUNTRY_OLD","39c98800-dd3c-11e0-b8d1-d1e2e7ba4f9d","rdf39c98800dd3c11e0b8d1d1e2e7ba4f9d","country")); - categories.add(new Category("CONTINENT_OLD","1d5d51f0-dd42-11e0-b8d3-d1e2e7ba4f9d","rdf1d5d51f0dd4211e0b8d3d1e2e7ba4f9d","continent reference data")); - categories.add(new Category("SPECIES_OLD","0a7fb500-dd3d-11e0-b8d1-d1e2e7ba4f9d","rdf0a7fb500dd3d11e0b8d1d1e2e7ba4f9d","species")); - categories.add(new Category("CodeListCountry","4c8d93a0-edc2-11e0-93e4-f6a9821baa29","rdf4c8d93a0edc211e093e4f6a9821baa29","Country")); - categories.add(new Category("CL_DIVISION","1140bdf0-dd2c-11e0-9220-ae17b3db32b7","rdf1140bdf0dd2c11e09220ae17b3db32b7","undefined")); - categories.add(new Category("CL_ASFIS_TAX","f87360f0-d9f9-11e0-ba05-d9adb0db767c","rdff87360f0d9f911e0ba05d9adb0db767c","undefined")); - - conf.setCategories(categories); - - //database Parameters - conf.setDatabaseUserName("gcube"); - conf.setDatabasePassword("d4science2"); - conf.setDatabaseDriver("org.postgresql.Driver"); - conf.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - - guesser.runGuesser(seriesName, column, conf); - guesser.showResults(guesser.getClassification()); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestSingleExternalCfgProduction.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestSingleExternalCfgProduction.java deleted file mode 100644 index 405cae5..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestSingleExternalCfgProduction.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test; - -import java.util.ArrayList; -import java.util.List; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class TestSingleExternalCfgProduction { - - public static void main(String[] args) { - - try { - - CategoryGuesser guesser = new CategoryGuesser(); - - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String singleton = "Faroe Island"; - - String family = "COUNTRY_OLD"; - String column = "field6"; - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - /* - conf.setReferenceTable("codelist1733371938"); - conf.setReferenceColumn("ifield14"); - conf.setNameHuman("ifield1"); - conf.setIdColumn("ifield0"); - conf.setDescription("ifield2"); - */ - - ArrayList categories = new ArrayList(); - //human name, index, table name, description - categories.add(new Category("COUNTRY_OLD","39c98800-dd3c-11e0-b8d1-d1e2e7ba4f9d","rdf39c98800dd3c11e0b8d1d1e2e7ba4f9d","country")); - categories.add(new Category("CONTINENT_OLD","1d5d51f0-dd42-11e0-b8d3-d1e2e7ba4f9d","rdf1d5d51f0dd4211e0b8d3d1e2e7ba4f9d","continent reference data")); - categories.add(new Category("SPECIES_OLD","0a7fb500-dd3d-11e0-b8d1-d1e2e7ba4f9d","rdf0a7fb500dd3d11e0b8d1d1e2e7ba4f9d","species")); - categories.add(new Category("CodeListCountry","4c8d93a0-edc2-11e0-93e4-f6a9821baa29","rdf4c8d93a0edc211e093e4f6a9821baa29","Country")); - categories.add(new Category("CL_DIVISION","1140bdf0-dd2c-11e0-9220-ae17b3db32b7","rdf1140bdf0dd2c11e09220ae17b3db32b7","undefined")); - - conf.setCategories(categories); - - //CHANGE THIS TO ENHANCE THE RECALL - conf.setEntryAcceptanceThreshold(30); - conf.setReferenceChunksToTake(-1); - conf.setTimeSeriesChunksToTake(-1); - conf.setUseSimpleDistance(false); - - //database Parameters - conf.setDatabaseUserName("gcube"); - conf.setDatabasePassword("d4science2"); -// conf.setDatabaseDriver("org.postgresql.Driver"); - conf.setDatabaseURL("jdbc:postgresql://localhost/testdb"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - - guesser.runGuesser(singleton, conf, family,column ); - - ArrayList detailedResults = guesser.getDetailedMatches(); - - AnalysisLogger.getLogger().warn("Detailed Match on Name :"+singleton); - - CategoryGuesser.showResults(detailedResults); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest1.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest1.java deleted file mode 100644 index 2c05181..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest1.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTest1 { - - public static void main(String[] args) { - - try { - int attempts = 1; - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592"; - String column = "field1"; - String correctFamily = "country"; - String correctColumn = "name_en"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - //bench 2 - AnalysisLogger.getLogger().warn("----------------------BENCH 2-------------------------"); - seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592"; - column = "field2"; - correctFamily = "area"; - correctColumn = "name_en"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 2-----------------------\n"); - - //bench 3 - AnalysisLogger.getLogger().warn("----------------------BENCH 3-------------------------"); - seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592"; - column = "field4"; - correctFamily = "species"; - correctColumn = "scientific_name"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 3-----------------------\n"); - - //bench 4 - AnalysisLogger.getLogger().warn("----------------------BENCH 4-------------------------"); - seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592"; - column = "field3"; - correctFamily = "species"; - correctColumn = "scientific_name"; -// CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 4-----------------------\n"); - - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest2.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest2.java deleted file mode 100644 index ca70087..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest2.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTest2 { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; - String column = "field1"; - String correctFamily = "SPECIES"; - String correctColumn = "SCIENTIFIC_NAME"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - - - - //bench 2 - AnalysisLogger.getLogger().warn("----------------------BENCH 2-------------------------"); - seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; - column = "field2"; - correctFamily = "COUNTRY"; - correctColumn = "ISO_3_CODE"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 2-----------------------\n"); - - - //bench 4 - AnalysisLogger.getLogger().warn("----------------------BENCH 4-------------------------"); - seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; - column = "field3"; - correctFamily = "AREA"; - correctColumn = "NAME_EN"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 4-----------------------\n"); - - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest3.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest3.java deleted file mode 100644 index f940995..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest3.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTest3 { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; - String column = "field1"; - String correctFamily = "SPECIES"; - String correctColumn = "SCIENTIFIC_NAME"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest4.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest4.java deleted file mode 100644 index 1743a11..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest4.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTest4 { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; - String column = "field3"; - String correctFamily = "AREA"; - String correctColumn = "NAME_EN"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest5.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest5.java deleted file mode 100644 index 6eb5805..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest5.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTest5 { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_532bba80_1c8f_11df_a4ee_87804054691e"; - String column = "field2"; - String correctFamily = "ISSCAAP GROUP"; - String correctColumn = "NAME_EN"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestExternalCfg.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestExternalCfg.java deleted file mode 100644 index 5d6f87a..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestExternalCfg.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTestExternalCfg { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_532bba80_1c8f_11df_a4ee_87804054691e"; - String column = "field2"; - String correctFamily = "ISSCAAP GROUP"; - String correctColumn = "NAME_EN"; - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - conf.setCategoryDiscardDifferencialThreshold(5); - conf.setCategoryDiscardThreshold(0); - conf.setChunkSize(25); - conf.setEntryAcceptanceThreshold(50); - conf.setNumberOfThreadsToUse(2); - conf.setRandomTake(true); - conf.setReferenceChunksToTake(20); - conf.setTimeSeriesChunksToTake(1); - conf.setUseSimpleDistance(false); - - //database Parameters - conf.setDatabaseUserName("root"); -// conf.setDatabasePassword("password"); - conf.setDatabaseDriver("com.mysql.jdbc.Driver"); - conf.setDatabaseURL("jdbc:mysql://localhost/timeseries"); - conf.setDatabaseDialect("org.hibernate.dialect.MySQLDialect"); - conf.setDatabaseAutomaticTestTable("connectiontesttable"); - conf.setDatabaseIdleConnectionTestPeriod("3600"); - - CategoryGuesser.AccuracyCalc(conf,guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestFilterCategory.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestFilterCategory.java deleted file mode 100644 index 9c48b56..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestFilterCategory.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - - -public class BenchMarkTestFilterCategory { - - public static void main(String[] args) { - - try { - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "ref_order"; - String column = "scientific_name"; - String correctFamily = "order"; - String correctColumn = "scientific_name"; - - guesser.runGuesser(seriesName, column, null, correctFamily, correctColumn); - ArrayList results = guesser.getClassification(); - - CategoryGuesser.showResults(results); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestSingleton.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestSingleton.java deleted file mode 100644 index f2f5e03..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestSingleton.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - - -public class BenchMarkTestSingleton { - - public static void main(String[] args) { - - try { - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String singleton = "sarda sarda"; -// String singleton = "Mitella pollicipes"; -// String singleton = "policipes"; -// String singleton = ""; - String family = "catalog life"; - String column = "scientific_name"; - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - - //CHANGE THIS TO ENHANCE THE RECALL - conf.setEntryAcceptanceThreshold(30); - conf.setReferenceChunksToTake(-1); - conf.setTimeSeriesChunksToTake(-1); - conf.setUseSimpleDistance(false); - - guesser.runGuesser(singleton, conf, family,column ); - ArrayList detailedResults = guesser.getDetailedMatches(); - - AnalysisLogger.getLogger().warn("Detailed Match on Name :"+singleton); - - CategoryGuesser.showResults(detailedResults); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestTSCountry.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestTSCountry.java deleted file mode 100644 index 9ab8f92..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestTSCountry.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTestTSCountry { - - public static void main(String[] args) { - - try { - int attempts = 1; - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592"; - String column = "field1"; - String correctFamily = "country"; - String correctColumn = "name_en"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSet.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSet.java deleted file mode 100644 index 87277e3..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSet.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTrainingSet { - - -public static void main(String[] args) { - - try { - String configPath ="."; - int attempts = 1; - CategoryGuesser guesser = new CategoryGuesser(); - - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "ref_commission"; - String column = "name_en"; - String correctFamily = "commission"; - String correctColumn = "name_en"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - - AnalysisLogger.getLogger().warn("----------------------BENCH 2-------------------------"); - seriesName = "ref_species"; - column = "scientific_name"; - correctFamily = "species"; - correctColumn = "scientific_name"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 2-----------------------\n"); - - - AnalysisLogger.getLogger().warn("----------------------BENCH 3-------------------------"); - seriesName = "ref_area"; - column = "name_en"; - correctFamily = "area"; - correctColumn = "name_en"; -// CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 3-----------------------\n"); - - - AnalysisLogger.getLogger().warn("----------------------BENCH 4-------------------------"); - seriesName = "ref_ocean"; - column = "name_en"; - correctFamily = "ocean"; - correctColumn = "name_en"; -// CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 4-----------------------\n"); - - - AnalysisLogger.getLogger().warn("----------------------BENCH 5-------------------------"); - seriesName = "ref_geo_region"; - column = "name_en"; - correctFamily = "geo region"; - correctColumn = "name_en"; -// CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 5-----------------------\n"); - - - AnalysisLogger.getLogger().warn("----------------------BENCH 6-------------------------"); - seriesName = "ref_fa_region"; - column = "name_en"; - correctFamily = "fa region"; - correctColumn = "name_en"; -// CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 6-----------------------\n"); - - - AnalysisLogger.getLogger().warn("----------------------BENCH 7-------------------------"); - seriesName = "ref_order"; - column = "scientific_name"; - correctFamily = "order"; - correctColumn = "scientific_name"; -// CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 7-----------------------\n"); - - - - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSetScientificName.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSetScientificName.java deleted file mode 100644 index abcd0de..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSetScientificName.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class BenchMarkTrainingSetScientificName { - - -public static void main(String[] args) { - - try { - String configPath ="."; - int attempts = 1; - CategoryGuesser guesser = new CategoryGuesser(); - - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String seriesName = "ref_species"; - String column = "scientific_name"; - String correctFamily = "species"; - String correctColumn = "scientific_name"; - CategoryGuesser.AccuracyCalc(guesser, configPath, seriesName, column, attempts, correctFamily, correctColumn); - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestExternalCfgProduction.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestExternalCfgProduction.java deleted file mode 100644 index 2a4ddd4..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestExternalCfgProduction.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class TestExternalCfgProduction { - - public static void main(String[] args) { - - try { - int attempts = 1; - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); -// String seriesName = "rdmc366dfe0ddf511e086b1b1c5d6fb1c27"; - String seriesName = "IMPORT_ecd2e3a0_ee90_11e0_be9e_90f3621758ee"; - - String column = "field4"; - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - /* - conf.setCategoryDiscardDifferencialThreshold(5); - conf.setCategoryDiscardThreshold(0); - conf.setChunkSize(25); - conf.setEntryAcceptanceThreshold(50); - conf.setNumberOfThreadsToUse(2); - conf.setRandomTake(true); - conf.setReferenceChunksToTake(20); - conf.setTimeSeriesChunksToTake(1); - conf.setUseSimpleDistance(false); - */ - - //database Parameters - conf.setDatabaseUserName("utente"); - conf.setDatabasePassword("d4science"); -// conf.setDatabaseDriver("org.postgresql.Driver"); - conf.setDatabaseURL("jdbc:postgresql://dbtest.next.research-infrastructures.eu/timeseries"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - conf.setDatabaseAutomaticTestTable("connectiontesttable"); - conf.setDatabaseIdleConnectionTestPeriod("3600"); - conf.setReferenceTable("codelist1733371938"); - conf.setReferenceColumn("ifield14"); - conf.setNameHuman("ifield1"); - conf.setIdColumn("ifield0"); - conf.setDescription("ifield2"); - guesser.runGuesser(seriesName, column, conf); - guesser.showResults(guesser.getClassification()); -// AnalysisLogger.getLogger().warn(); - - - - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestSingleExternalCfgProduction.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestSingleExternalCfgProduction.java deleted file mode 100644 index 211b82f..0000000 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestSingleExternalCfgProduction.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.gcube.contentmanagement.lexicalmatcher.analysis.test.old; - -import java.util.ArrayList; - -import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; -import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; - -public class TestSingleExternalCfgProduction { - - public static void main(String[] args) { - - try { - - - String configPath = "."; - CategoryGuesser guesser = new CategoryGuesser(); - //bench 1 - AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); - String singleton = "Faroe Island"; -// String singleton = "Mitella pollicipes"; -// String singleton = "policipes"; -// String singleton = ""; -// String family = "rdf0a7fb500dd3d11e0b8d1d1e2e7ba4f9d"; - - String family = "COUNTRY_OLD"; - String column = "field6"; - - LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); - - //CHANGE THIS TO ENHANCE THE RECALL - conf.setEntryAcceptanceThreshold(30); - conf.setReferenceChunksToTake(-1); - conf.setTimeSeriesChunksToTake(-1); - conf.setUseSimpleDistance(false); - //database Parameters - conf.setDatabaseUserName("utente"); - conf.setDatabasePassword("d4science"); -// conf.setDatabaseDriver("org.postgresql.Driver"); - conf.setDatabaseURL("jdbc:postgresql://dbtest.next.research-infrastructures.eu/timeseries"); - conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect"); - conf.setDatabaseAutomaticTestTable("connectiontesttable"); - conf.setDatabaseIdleConnectionTestPeriod("3600"); - - conf.setReferenceTable("codelist1733371938"); - conf.setReferenceColumn("ifield14"); - conf.setNameHuman("ifield1"); - conf.setIdColumn("ifield0"); - conf.setDescription("ifield2"); - - guesser.initSingleMatcher(conf,column ); - - guesser.runGuesser(singleton, null, family,column ); - - ArrayList detailedResults = guesser.getDetailedMatches(); - - AnalysisLogger.getLogger().warn("Detailed Match on Name :"+singleton); - - CategoryGuesser.showResults(detailedResults); - - AnalysisLogger.getLogger().warn("--------------------END BENCH 1-----------------------\n"); - - - } catch (Exception e) { - e.printStackTrace(); - } - - } - -} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/AnalysisLogger.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/AnalysisLogger.java index 91acbc2..aab8bac 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/AnalysisLogger.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/AnalysisLogger.java @@ -1,23 +1,26 @@ -package org.gcube.contentmanagement.lexicalmatcher.utils; - -import org.apache.log4j.Logger; - -@Deprecated -public class AnalysisLogger { - - private static Logger logger = Logger.getLogger("AnalysisLogger"); - //private static Logger hibernateLogger = LoggerFactory.getLogger("HibernateLogger"); - - public static Logger getLogger() { - return logger; - } - - // in ingresso vuole il path al file di config del log4j - public static void setLogger(String path) { - - } - - public static void printStackTrace(Exception e) { - logger.error("error ",e); - } -} +package org.gcube.contentmanagement.lexicalmatcher.utils; + +import org.apache.log4j.Logger; + +@Deprecated +public class AnalysisLogger { + + private static Logger logger = Logger.getLogger("AnalysisLogger"); + //private static Logger hibernateLogger = LoggerFactory.getLogger("HibernateLogger"); + + @Deprecated + public static Logger getLogger() { + return logger; + } + + // in ingresso vuole il path al file di config del log4j + @Deprecated + public static void setLogger(String path) { + + } + + @Deprecated + public static void printStackTrace(Exception e) { + logger.error("error ",e); + } +} diff --git a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/MathFunctions.java b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/MathFunctions.java index 0015b0e..92c4508 100644 --- a/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/MathFunctions.java +++ b/src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/MathFunctions.java @@ -3,16 +3,14 @@ package org.gcube.contentmanagement.lexicalmatcher.utils; import java.math.BigInteger; import java.util.ArrayList; +import org.gcube.contentmanagement.graphtools.abstracts.GenericStandaloneGraph; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class MathFunctions { - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } - + private static Logger logger = LoggerFactory.getLogger(MathFunctions.class); + //increments a percentage o mean calculation when a lot of elements are present public static float incrementPerc(float perc, float quantity, int N){ @@ -78,7 +76,7 @@ public class MathFunctions { } } - AnalysisLogger.getLogger().trace("MathFunctions-> generateRandoms " + randomsSet.toString()); + logger.trace("MathFunctions-> generateRandoms " + randomsSet.toString()); return randomsSet; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/DBScan.java b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/DBScan.java index 8426a3c..c0c8aba 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/DBScan.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/DBScan.java @@ -3,7 +3,6 @@ package org.gcube.dataanalysis.ecoengine.clustering; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; @@ -13,7 +12,6 @@ import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; -import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.DatabaseParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; @@ -23,6 +21,8 @@ import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; import org.gcube.dataanalysis.ecoengine.utils.Transformations; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.example.Attribute; import com.rapidminer.example.Attributes; @@ -36,6 +36,8 @@ import com.rapidminer.tools.OperatorService; public class DBScan implements Clusterer{ + private static Logger logger = LoggerFactory.getLogger(DBScan.class); + protected AlgorithmConfiguration config; protected String epsilon; protected String minPoints; @@ -170,20 +172,20 @@ public class DBScan implements Clusterer{ status = 0; if ((config!=null) && (initrapidminer)) config.initRapidMiner(); - AnalysisLogger.getLogger().debug("Initialized Rapid Miner "); - AnalysisLogger.getLogger().debug("Initializing Database Connection"); + logger.debug("Initialized Rapid Miner "); + logger.debug("Initializing Database Connection"); dbHibConnection=DatabaseUtils.initDBSession(config); //create the final table try{ - AnalysisLogger.getLogger().debug("dropping table "+OccurrencePointsClusterTable); + logger.debug("dropping table "+OccurrencePointsClusterTable); String dropStatement = DatabaseUtils.dropTableStatement(OccurrencePointsClusterTable); - AnalysisLogger.getLogger().debug("dropping table "+dropStatement); + logger.debug("dropping table "+dropStatement); DatabaseFactory.executeSQLUpdate(dropStatement, dbHibConnection); }catch(Exception e){ - AnalysisLogger.getLogger().debug("Could not drop table "+OccurrencePointsClusterTable); + logger.debug("Could not drop table "+OccurrencePointsClusterTable); } //create Table - AnalysisLogger.getLogger().debug("Creating table "+OccurrencePointsClusterTable); + logger.debug("Creating table "+OccurrencePointsClusterTable); String [] features = FeaturesColumnNames.split(AlgorithmConfiguration.getListSeparator()); String columns = ""; @@ -195,18 +197,18 @@ public class DBScan implements Clusterer{ String createStatement = "create table "+OccurrencePointsClusterTable+" ( "+columns+")"; // String createStatement = new DatabaseUtils(dbHibConnection).buildCreateStatement(OccurrencePointsTable,OccurrencePointsClusterTable); - AnalysisLogger.getLogger().debug("Statement: "+createStatement); + logger.debug("Statement: "+createStatement); DatabaseFactory.executeSQLUpdate(createStatement, dbHibConnection); //add two columns one for cluster and another for outliers - AnalysisLogger.getLogger().debug("Adding Columns"); + logger.debug("Adding Columns"); DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(OccurrencePointsClusterTable, clusterColumn, clusterColumnType), dbHibConnection); DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(OccurrencePointsClusterTable, outliersColumn, outliersColumnType), dbHibConnection); - AnalysisLogger.getLogger().debug("Getting Samples"); + logger.debug("Getting Samples"); //build samples try{ getSamples(); }catch(Throwable e){ - AnalysisLogger.getLogger().debug("Error getting samples for clustering: "+e.getLocalizedMessage()); + logger.debug("Error getting samples for clustering: "+e.getLocalizedMessage()); } status = 10f; } @@ -242,7 +244,7 @@ public class DBScan implements Clusterer{ // limit = (int)(11d*Math.pow(N,2d/(k+1))); // limit =(int) ((double)N/(1.3d)); - AnalysisLogger.getLogger().debug("Clustering limit: "+limit); + logger.debug("Clustering limit: "+limit); List samples = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(OccurrencePointsTable, FeaturesColumnNames, "limit "+limit), dbHibConnection); @@ -272,9 +274,9 @@ public class DBScan implements Clusterer{ ir++; } - AnalysisLogger.getLogger().debug("Building Sample Set For Miner"); + logger.debug("Building Sample Set For Miner"); produceSamples(samplesVector); - AnalysisLogger.getLogger().debug("Obtained "+samplesVector.length+" chunks"); + logger.debug("Obtained "+samplesVector.length+" chunks"); } public void produceSamples(double[][] sampleVectors) throws Exception{ @@ -292,7 +294,7 @@ public class DBScan implements Clusterer{ throw new Exception("DBScan: Error incomplete parameters"); } - AnalysisLogger.getLogger().debug("DBScan: Settin up the cluster"); + logger.debug("DBScan: Settin up the cluster"); //take elements and produce example set com.rapidminer.operator.clustering.clusterer.DBScan clusterer = (com.rapidminer.operator.clustering.clusterer.DBScan) OperatorService.createOperator("DBScanClustering"); clusterer.setParameter("local_random_seed", "-1"); @@ -303,11 +305,11 @@ public class DBScan implements Clusterer{ IOContainer innerInput = new IOContainer(points); - AnalysisLogger.getLogger().debug("DBScan: Clustering..."); + logger.debug("DBScan: Clustering..."); long ti= System.currentTimeMillis(); IOContainer output = clusterer.apply(innerInput); - AnalysisLogger.getLogger().debug("DBScan: ...ELAPSED CLUSTERING TIME: "+(System.currentTimeMillis()-ti)); - AnalysisLogger.getLogger().debug("DBScan: ...Clustering Finished in "+(System.currentTimeMillis()-t00)); + logger.debug("DBScan: ...ELAPSED CLUSTERING TIME: "+(System.currentTimeMillis()-ti)); + logger.debug("DBScan: ...Clustering Finished in "+(System.currentTimeMillis()-t00)); status = 70f; IOObject[] outputvector = output.getIOObjects(); @@ -315,7 +317,7 @@ public class DBScan implements Clusterer{ BuildClusterTable(outputvector); }catch(Exception e){ - AnalysisLogger.getLogger().debug("ERROR "+e.getLocalizedMessage()); + logger.debug("ERROR "+e.getLocalizedMessage()); e.printStackTrace(); throw e; } @@ -332,11 +334,11 @@ public class DBScan implements Clusterer{ ExampleSet es = (ExampleSet) outputvector[1]; String columnsNames =FeaturesColumnNames+","+clusterColumn+","+outliersColumn; int minpoints = Integer.parseInt(minPoints); - AnalysisLogger.getLogger().debug("Analyzing Cluster ->"+" minpoints"+minpoints); + logger.debug("Analyzing Cluster ->"+" minpoints"+minpoints); int nClusters = innermodel.getClusters().size(); float statusstep = ((100f-status)/ (float)(nClusters+1)); - AnalysisLogger.getLogger().debug("Start Write On DB"); + logger.debug("Start Write On DB"); for (Cluster c : innermodel.getClusters()){ StringBuffer bufferRows = new StringBuffer(); //take cluster id @@ -344,7 +346,7 @@ public class DBScan implements Clusterer{ boolean outlier = false; //take cluster element indexes int npoints = c.getExampleIds().size(); - AnalysisLogger.getLogger().debug("Analyzing Cluster ->"+id+" with "+npoints); + logger.debug("Analyzing Cluster ->"+id+" with "+npoints); if (npoints<=minpoints) outlier=true; @@ -375,20 +377,20 @@ public class DBScan implements Clusterer{ } k++; -// AnalysisLogger.getLogger().trace("DBScan: Classification : "+towrite+"->"+id+" is outlier?"+outlier); +// logger.trace("DBScan: Classification : "+towrite+"->"+id+" is outlier?"+outlier); } if (bufferRows.length()>0){ -// AnalysisLogger.getLogger().debug("DBScan: Inserting Buffer "+DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows)); - AnalysisLogger.getLogger().debug("Writing into DB"); +// logger.debug("DBScan: Inserting Buffer "+DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows)); + logger.debug("Writing into DB"); DatabaseFactory.executeSQLUpdate(DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows),dbHibConnection); - AnalysisLogger.getLogger().debug("Finished with writing into DB"); + logger.debug("Finished with writing into DB"); }else - AnalysisLogger.getLogger().debug("Nothing to write in the buffer"); + logger.debug("Nothing to write in the buffer"); float instatus = status + statusstep; status = Math.min(95f, instatus); - AnalysisLogger.getLogger().debug("Status: "+status); + logger.debug("Status: "+status); } } @@ -397,11 +399,11 @@ public class DBScan implements Clusterer{ @Override public void shutdown() { try{ - AnalysisLogger.getLogger().debug("Closing DB Connection "); + logger.debug("Closing DB Connection "); if (dbHibConnection!=null) dbHibConnection.close(); }catch(Exception e){ - AnalysisLogger.getLogger().debug("Could not shut down connection"); + logger.debug("Could not shut down connection"); } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/KMeans.java b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/KMeans.java index 768488b..b1eb041 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/KMeans.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/KMeans.java @@ -3,7 +3,7 @@ package org.gcube.dataanalysis.ecoengine.clustering; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.contentmanagement.graphtools.abstracts.GenericStandaloneGraph; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -11,11 +11,12 @@ import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; -import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.DatabaseParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.operator.IOContainer; import com.rapidminer.operator.IOObject; @@ -23,38 +24,12 @@ import com.rapidminer.tools.OperatorService; public class KMeans extends DBScan{ + private static Logger logger = LoggerFactory.getLogger(KMeans.class); + private String kk; private String maxRuns; private String maxOptimizations; - public static void main(String[] args) throws Exception{ - long t0 = System.currentTimeMillis(); - - AlgorithmConfiguration config = new AlgorithmConfiguration(); - config.setConfigPath("./cfg/"); - config.setPersistencePath("./"); - config.setParam("OccurrencePointsTable","presence_basking_cluster"); - config.setParam("FeaturesColumnNames","centerlat"+AlgorithmConfiguration.getListSeparator()+"centerlong"); - config.setParam("OccurrencePointsClusterTable","occCluster_kmeans"); - config.setParam("k","50"); - config.setParam("max_runs","10"); - config.setParam("max_optimization_steps","10"); - config.setParam("min_points","2"); - - config.setParam("DatabaseUserName","gcube"); - config.setParam("DatabasePassword","d4science2"); - config.setParam("DatabaseURL","jdbc:postgresql://146.48.87.169/testdb"); - config.setParam("DatabaseDriver","org.postgresql.Driver"); - - KMeans cluster = new KMeans(); - cluster.setConfiguration(config); - cluster.init(); - cluster.compute(); - - System.out.println("ELAPSED "+(System.currentTimeMillis()-t0)); - - } - @Override public void setConfiguration(AlgorithmConfiguration config) { @@ -79,7 +54,7 @@ public class KMeans extends DBScan{ throw new Exception("KMeans: Error incomplete parameters"); } - AnalysisLogger.getLogger().debug("KMeans: Settin up the cluster"); + logger.debug("KMeans: Settin up the cluster"); //take elements and produce example set com.rapidminer.operator.clustering.clusterer.KMeans kmeans = (com.rapidminer.operator.clustering.clusterer.KMeans) OperatorService.createOperator("KMeans"); @@ -93,11 +68,11 @@ public class KMeans extends DBScan{ IOContainer innerInput = new IOContainer(points); - AnalysisLogger.getLogger().debug("KMeans: Clustering..."); + logger.debug("KMeans: Clustering..."); long ti= System.currentTimeMillis(); IOContainer output = kmeans.apply(innerInput); - AnalysisLogger.getLogger().debug("KMEANS: ...ELAPSED CLUSTERING TIME: "+(System.currentTimeMillis()-ti)); - AnalysisLogger.getLogger().debug("KMeans: ...Clustering Finished"); + logger.debug("KMEANS: ...ELAPSED CLUSTERING TIME: "+(System.currentTimeMillis()-ti)); + logger.debug("KMeans: ...Clustering Finished"); status = 70f; IOObject[] outputvector = output.getIOObjects(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/LOF.java b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/LOF.java index 3741b49..a23a8c5 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/LOF.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/LOF.java @@ -1,11 +1,9 @@ package org.gcube.dataanalysis.ecoengine.clustering; import java.lang.reflect.Field; -import java.security.acl.LastOwnerException; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -20,26 +18,21 @@ import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.DynamicEnum; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.example.Attribute; -import com.rapidminer.example.Attributes; -import com.rapidminer.example.Example; -import com.rapidminer.example.ExampleSet; import com.rapidminer.example.set.SimpleExampleSet; -import com.rapidminer.example.table.AttributeFactory; import com.rapidminer.example.table.DataRow; -import com.rapidminer.example.table.DoubleArrayDataRow; -import com.rapidminer.example.table.ExampleTable; import com.rapidminer.example.table.MemoryExampleTable; import com.rapidminer.operator.IOContainer; import com.rapidminer.operator.IOObject; -import com.rapidminer.operator.clustering.Cluster; -import com.rapidminer.operator.clustering.ClusterModel; -import com.rapidminer.tools.Ontology; import com.rapidminer.tools.OperatorService; public class LOF extends DBScan { + private static Logger logger = LoggerFactory.getLogger(LOF.class); + String minimal_points_lower_bound = "1"; String minimal_points_upper_bound = "10"; String lof_threshold = "2"; @@ -64,20 +57,20 @@ public class LOF extends DBScan { status = 0; if ((config!=null) && (initrapidminer)) config.initRapidMiner(); - AnalysisLogger.getLogger().debug("Initialized Rapid Miner "); - AnalysisLogger.getLogger().debug("Initializing Database Connection"); + logger.debug("Initialized Rapid Miner "); + logger.debug("Initializing Database Connection"); dbHibConnection=DatabaseUtils.initDBSession(config); //create the final table try{ - AnalysisLogger.getLogger().debug("dropping table "+OccurrencePointsClusterTable); + logger.debug("dropping table "+OccurrencePointsClusterTable); String dropStatement = DatabaseUtils.dropTableStatement(OccurrencePointsClusterTable); - AnalysisLogger.getLogger().debug("dropping table "+dropStatement); + logger.debug("dropping table "+dropStatement); DatabaseFactory.executeSQLUpdate(dropStatement, dbHibConnection); }catch(Exception e){ - AnalysisLogger.getLogger().debug("Could not drop table "+OccurrencePointsClusterTable); + logger.debug("Could not drop table "+OccurrencePointsClusterTable); } //create Table - AnalysisLogger.getLogger().debug("Creating table "+OccurrencePointsClusterTable); + logger.debug("Creating table "+OccurrencePointsClusterTable); String [] features = FeaturesColumnNames.split(AlgorithmConfiguration.getListSeparator()); String columns = ""; @@ -89,13 +82,13 @@ public class LOF extends DBScan { String createStatement = "create table "+OccurrencePointsClusterTable+" ( "+columns+")"; // String createStatement = new DatabaseUtils(dbHibConnection).buildCreateStatement(OccurrencePointsTable,OccurrencePointsClusterTable); - AnalysisLogger.getLogger().debug("Statement: "+createStatement); + logger.debug("Statement: "+createStatement); DatabaseFactory.executeSQLUpdate(createStatement, dbHibConnection); //add two columns one for cluster and another for outliers - AnalysisLogger.getLogger().debug("Adding Columns"); + logger.debug("Adding Columns"); DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(OccurrencePointsClusterTable, lofcolumn, lofcolumntype), dbHibConnection); DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(OccurrencePointsClusterTable, outliersColumn, outliersColumnType), dbHibConnection); - AnalysisLogger.getLogger().debug("Getting Samples"); + logger.debug("Getting Samples"); //build samples getSamples(); status = 10f; @@ -127,7 +120,7 @@ public class LOF extends DBScan { throw new Exception("LOF: Error incomplete parameters"); } status = 10f; - AnalysisLogger.getLogger().debug("LOF: Settin up the cluster"); + logger.debug("LOF: Settin up the cluster"); // take elements and produce example set com.rapidminer.operator.preprocessing.outlier.LOFOutlierOperator clusterer = (com.rapidminer.operator.preprocessing.outlier.LOFOutlierOperator) OperatorService.createOperator("LOFOutlierDetection"); clusterer.setParameter("minimal_points_lower_bound", minimal_points_lower_bound); @@ -136,11 +129,11 @@ public class LOF extends DBScan { IOContainer innerInput = new IOContainer(points); - AnalysisLogger.getLogger().debug("LOF: Clustering..."); + logger.debug("LOF: Clustering..."); long ti = System.currentTimeMillis(); IOContainer output = clusterer.apply(innerInput); - AnalysisLogger.getLogger().debug("LOF: ...ELAPSED CLUSTERING TIME: " + (System.currentTimeMillis() - ti)); - AnalysisLogger.getLogger().debug("LOF: ...Clustering Finished"); + logger.debug("LOF: ...ELAPSED CLUSTERING TIME: " + (System.currentTimeMillis() - ti)); + logger.debug("LOF: ...Clustering Finished"); status = 70f; IOObject[] outputvector = output.getIOObjects(); @@ -165,7 +158,7 @@ public class LOF extends DBScan { double lofthr = 2; if (lof_threshold!=null) try{lofthr = Double.parseDouble(lof_threshold);}catch(Exception e){}; - AnalysisLogger.getLogger().debug("LOF: using lof threshold :"+lofthr); + logger.debug("LOF: using lof threshold :"+lofthr); for (int i = 0; i < numofrows; i++) { @@ -188,7 +181,7 @@ public class LOF extends DBScan { bufferRows.append(","); } - AnalysisLogger.getLogger().debug("LOF: Finished in retrieving and building output to write"); + logger.debug("LOF: Finished in retrieving and building output to write"); String columnsNames = FeaturesColumnNames + ","+lofcolumn+","+outliersColumn; @@ -196,15 +189,15 @@ public class LOF extends DBScan { if (bufferRows.length() > 0) { - AnalysisLogger.getLogger().debug("Writing into DB"); -// AnalysisLogger.getLogger().debug("Query to execute: "+DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows)); + logger.debug("Writing into DB"); +// logger.debug("Query to execute: "+DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows)); DatabaseFactory.executeSQLUpdate(DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows), dbHibConnection); - AnalysisLogger.getLogger().debug("Finished with writing into DB"); + logger.debug("Finished with writing into DB"); } else - AnalysisLogger.getLogger().debug("Nothing to write in the buffer"); + logger.debug("Nothing to write in the buffer"); status = 100; - AnalysisLogger.getLogger().debug("Status: " + status); + logger.debug("Status: " + status); } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/XMeansWrapper.java b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/XMeansWrapper.java index 994cc1a..96adbe5 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/XMeansWrapper.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/clustering/XMeansWrapper.java @@ -1,12 +1,10 @@ package org.gcube.dataanalysis.ecoengine.clustering; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.InputStream; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -14,22 +12,23 @@ import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; -import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.DatabaseParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import weka.clusterers.ClusterEvaluation; import weka.core.DenseInstance; import weka.core.Instance; import weka.core.Instances; -import weka.core.converters.ArffSaver; import weka.core.converters.CSVLoader; public class XMeansWrapper extends DBScan { + private static Logger logger = LoggerFactory.getLogger(XMeansWrapper.class); + private String maxIterations; private String minClusters; private String maxClusters; @@ -38,22 +37,7 @@ public class XMeansWrapper extends DBScan { super(); initrapidminer=false; } - public static void main1(String[] args) throws Exception { - args = new String[2]; - args[0] = "input.csv"; - args[1] = "c:/tmp/output.arff"; - // load CSV - CSVLoader loader = new CSVLoader(); - loader.setSource(new File(args[0])); - Instances data = loader.getDataSet(); - - // save ARFF - ArffSaver saver = new ArffSaver(); - saver.setInstances(data); - saver.setFile(new File(args[1])); - // saver.setDestination(new File(args[1])); - saver.writeBatch(); - } + public class CSV2Arff { /** @@ -62,56 +46,6 @@ public class XMeansWrapper extends DBScan { } - public static void main(String[] args) throws Exception { - XMeans xmeans = new XMeans(); - // xmeans.setInputCenterFile(new File("./clusterinput.arf")); - // String[] options = {"-I 10","-M 1000","-J 1000","-L 2","-H 50","-B 1.0","-use-kdtree no","-N clusterinput.arf","-O clusterout.txt","-U 3"}; - // String[] options = {"-I 10","-M 1000","-J 1000","-L 2","-H 50","-B 1.0","-use-kdtree no","-t clusterinput.arf","-O clusterout.txt","-U 3"}; - // String optionsS = "-t c:/tmp/output.arff -O c:/tmp/clusterout.arff"; - String optionsS = "-t c:/tmp/output.arff"; - String[] options = optionsS.split(" "); - String elements = "ciao,tutti\n5.1,3.5\n4.9,3.0\n4.7,3.2\n4.6,3.1\n5.0,3.6\n5.4,3.9\n4.6,3.4\n5.0,3.4\n4.4,2.9\n4.9,3.1\n"; - - // xmeans.setInputCenterFile(new File("./clusterinput.arf")); - - CSVLoader loader = new CSVLoader(); - InputStream tis = new ByteArrayInputStream(elements.getBytes("UTF-8")); - loader.setSource(tis); - Instances id = loader.getDataSet(); - System.out.println("ids: "+id.numInstances()); - System.exit(0); - xmeans.buildClusterer(id); - - // xmeans.main(options); - // ClusterEvaluation.evaluateClusterer(xmeans, options); - /* - * String[] opts = xmeans.getOptions(); for (int i=0;i" + di.toString(0)); - System.out.println(" " + di.toString(1)); - - // System.out.println(i); - - System.out.println("-------------------------------"); - } - - System.out.println(xmeans.m_Bic); - - // System.out.println(xmeans.clusterInstance(instance)); - int[] ii = xmeans.m_ClusterAssignments; - for (int ix : ii) - System.out.print(ix + " "); - - // xmeans.main(options); - } @Override public List getInputParameters() { @@ -172,7 +106,7 @@ public class XMeansWrapper extends DBScan { if ((samplesVector != null) && (samplesVector.length > 0)) { - AnalysisLogger.getLogger().debug("XMeans: Setting up the cluster"); + logger.debug("XMeans: Setting up the cluster"); CSVLoader loader = new CSVLoader(); StringBuffer sb = new StringBuffer(); @@ -198,22 +132,22 @@ public class XMeansWrapper extends DBScan { xmeans.setMinNumClusters(Integer.parseInt(minClusters)); xmeans.setMaxNumClusters(Integer.parseInt(maxClusters)); xmeans.buildClusterer(id); - AnalysisLogger.getLogger().debug("XMEANS: ...ELAPSED CLUSTERING TIME: "+(System.currentTimeMillis()-ti)); + logger.debug("XMEANS: ...ELAPSED CLUSTERING TIME: "+(System.currentTimeMillis()-ti)); status = 50f; // do clustering - AnalysisLogger.getLogger().debug("XMeans: Clustering ..."); + logger.debug("XMeans: Clustering ..."); Instances is = xmeans.getClusterCenters(); int nClusters = is.numInstances(); // take results - AnalysisLogger.getLogger().debug("XMeans: Found "+nClusters+" Centroids"); + logger.debug("XMeans: Found "+nClusters+" Centroids"); for (Instance i : is) { DenseInstance di = (DenseInstance) i; int nCluster = di.numAttributes(); for (int k = 0; k < nCluster; k++) { - AnalysisLogger.getLogger().debug(di.toString(k)); + logger.debug(di.toString(k)); } - AnalysisLogger.getLogger().debug("-------------------------------"); + logger.debug("-------------------------------"); } int[] clusteringAssignments = xmeans.m_ClusterAssignments; @@ -223,11 +157,11 @@ public class XMeansWrapper extends DBScan { counters[cluster]++; } - AnalysisLogger.getLogger().debug("XMeans: Building Table"); + logger.debug("XMeans: Building Table"); BuildClusterTable(clusteringAssignments, counters); } else - AnalysisLogger.getLogger().debug("XMeans: Warning - Empty Training Set"); + logger.debug("XMeans: Warning - Empty Training Set"); }catch(Exception e){ throw e; } @@ -242,13 +176,13 @@ public class XMeansWrapper extends DBScan { String columnsNames = FeaturesColumnNames + "," + clusterColumn + "," + outliersColumn; int minpoints = Integer.parseInt(minPoints); - AnalysisLogger.getLogger().debug("Analyzing Cluster ->" + " minpoints " + minpoints); + logger.debug("Analyzing Cluster ->" + " minpoints " + minpoints); StringBuffer bufferRows = new StringBuffer(); int nrows = samplesVector.length; int ncols = samplesVector[0].length; - AnalysisLogger.getLogger().debug("Analyzing Cluster ->" + "Building Rows to Insert"); + logger.debug("Analyzing Cluster ->" + "Building Rows to Insert"); for (int k = 0; k < nrows; k++) { bufferRows.append("("); @@ -269,18 +203,18 @@ public class XMeansWrapper extends DBScan { //TO-DO: insert row at chunks - AnalysisLogger.getLogger().debug("Analyzing Cluster ->" + "Inserting rows"); + logger.debug("Analyzing Cluster ->" + "Inserting rows"); if (bufferRows.length() > 0) { - AnalysisLogger.getLogger().debug("XMeans: Writing into DB"); - AnalysisLogger.getLogger().debug(DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows)); + logger.debug("XMeans: Writing into DB"); + logger.debug(DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows)); DatabaseFactory.executeSQLUpdate(DatabaseUtils.insertFromBuffer(OccurrencePointsClusterTable, columnsNames, bufferRows), dbHibConnection); - AnalysisLogger.getLogger().debug("XMeans: Finished with writing into DB"); + logger.debug("XMeans: Finished with writing into DB"); } else - AnalysisLogger.getLogger().debug("XMeans: Nothing to write in the buffer"); + logger.debug("XMeans: Nothing to write in the buffer"); status = 95f; - AnalysisLogger.getLogger().debug("XMeans: Status: " + status); + logger.debug("XMeans: Status: " + status); } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java b/src/main/java/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java index e17fc8b..21af1a2 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/configuration/AlgorithmConfiguration.java @@ -7,14 +7,17 @@ import java.util.List; import java.util.Properties; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.RapidMiner; public class AlgorithmConfiguration extends LexicalEngineConfiguration implements Serializable{ + private static Logger logger = LoggerFactory.getLogger(AlgorithmConfiguration.class); + private static final long serialVersionUID = 1L; public static Properties getProperties(String absoluteFilePath) { @@ -24,7 +27,7 @@ public class AlgorithmConfiguration extends LexicalEngineConfiguration implement fis = new FileInputStream(absoluteFilePath); props.load(fis); } catch (Exception e) { - AnalysisLogger.getLogger().error("error getting properties",e); + logger.error("error getting properties",e); } finally { try { fis.close(); @@ -179,7 +182,7 @@ public class AlgorithmConfiguration extends LexicalEngineConfiguration implement try { connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, Input); } catch (Exception e) { - AnalysisLogger.getLogger().error("ERROR initializing connection",e); + logger.error("ERROR initializing connection",e); } return connection; } @@ -188,8 +191,7 @@ public class AlgorithmConfiguration extends LexicalEngineConfiguration implement public void initRapidMiner(){ System.setProperty("rapidminer.init.operators", configPath+ AlgorithmConfiguration.RapidMinerOperatorsFile); RapidMiner.init(); - AnalysisLogger.setLogger(getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); - AnalysisLogger.getLogger().info("Rapid Miner initialized"); + logger.info("Rapid Miner initialized"); } public String getGcubeUserName() { diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/connectors/RemoteGenerationManager.java b/src/main/java/org/gcube/dataanalysis/ecoengine/connectors/RemoteGenerationManager.java index 2e7a9bc..da4a9d6 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/connectors/RemoteGenerationManager.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/connectors/RemoteGenerationManager.java @@ -1,10 +1,13 @@ package org.gcube.dataanalysis.ecoengine.connectors; import org.gcube.contentmanagement.graphtools.utils.HttpRequest; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RemoteGenerationManager { + private static Logger logger = LoggerFactory.getLogger(RemoteGenerationManager.class); + private final String submissionMethod = "submit"; private final String statusMethod = "status/"; private String submissionID; @@ -20,24 +23,24 @@ public class RemoteGenerationManager { public void submitJob(RemoteHspecInputObject rhio) throws Exception{ - AnalysisLogger.getLogger().warn("RemoteGenerationManager: retrieving job information"); + logger.warn("RemoteGenerationManager: retrieving job information"); RemoteHspecOutputObject rhoo = null; username = rhio.userName; try{ rhoo = (RemoteHspecOutputObject)HttpRequest.postJSonData(endpoint+submissionMethod, rhio, RemoteHspecOutputObject.class); - AnalysisLogger.getLogger().trace("RemoteGenerationManager: job information retrieved"); + logger.trace("RemoteGenerationManager: job information retrieved"); }catch(Exception e){ e.printStackTrace(); - AnalysisLogger.getLogger().trace("RemoteGenerationManager: ERROR - job information NOT retrieved"); + logger.trace("RemoteGenerationManager: ERROR - job information NOT retrieved"); throw e; } if ((rhoo!=null) && (rhoo.id!=null)){ - AnalysisLogger.getLogger().warn("RemoteGenerationManager: job ID retrieved "); + logger.warn("RemoteGenerationManager: job ID retrieved "); submissionID = rhoo.id; } else{ - AnalysisLogger.getLogger().warn("RemoteGenerationManager: ERROR - job ID NOT retrieved "+rhoo.error); - throw new Exception(); + logger.warn("RemoteGenerationManager: ERROR - job ID NOT retrieved "+rhoo.error); + throw new Exception("RemoteGenerationManager: ERROR - job ID NOT retrieved "+rhoo.error); } } @@ -48,8 +51,7 @@ public class RemoteGenerationManager { double completion = Double.parseDouble(rhoo.completion); return completion; }catch(Exception e){ - e.printStackTrace(); - AnalysisLogger.getLogger().warn("RemoteGenerationManager: ERROR - cannot retrieve information from remote site ",e); + logger.warn("RemoteGenerationManager: ERROR - cannot retrieve information from remote site ",e); } return 0; } @@ -60,8 +62,7 @@ public class RemoteGenerationManager { try{ rhoo = (RemoteHspecOutputObject)HttpRequest.getJSonData(endpoint+statusMethod+submissionID, null ,RemoteHspecOutputObject.class); }catch(Exception e){ - e.printStackTrace(); - AnalysisLogger.getLogger().warn("RemoteGenerationManager: ERROR - cannot retrieve information from remote site ",e); + logger.warn("RemoteGenerationManager: ERROR - cannot retrieve information from remote site ",e); } return rhoo; diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DiscrepancyAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DiscrepancyAnalysis.java index 210095c..80e2a1a 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DiscrepancyAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DiscrepancyAnalysis.java @@ -5,7 +5,6 @@ import java.util.LinkedHashMap; import java.util.List; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; @@ -15,6 +14,8 @@ import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DiscrepancyAnalysis extends DataAnalysis { @@ -29,6 +30,8 @@ public class DiscrepancyAnalysis extends DataAnalysis { "on a.%1$s=b.%2$s) as sel where firstprob<>secondprob"; */ + private static Logger logger = LoggerFactory.getLogger(DiscrepancyAnalysis.class); + static String discrepancyQuery = "select * from (select distinct a.%1$s as csquareone,b.%2$s as csquaretwo,a.%3$s as firstprob,b.%4$s as secondprob from " + "(select %1$s , avg(%3$s) as %3$s from (select distinct * from %5$s order by %1$s limit %7$s) as aa group by %1$s) as a " + "left join " + @@ -110,7 +113,7 @@ public class DiscrepancyAnalysis extends DataAnalysis { kthreshold = Double.parseDouble(kthresholdString); }catch(Exception e){} - AnalysisLogger.getLogger().trace("Using Cohen's Kappa Threshold: "+kthreshold); + logger.trace("Using Cohen's Kappa Threshold: "+kthreshold); int maxCompElements = maxElements; if (maxSamples!=null && maxSamples.length()>0){ @@ -143,10 +146,10 @@ public class DiscrepancyAnalysis extends DataAnalysis { return output; } - AnalysisLogger.getLogger().trace("Number Of Elements to take: "+numberofvectors); + logger.trace("Number Of Elements to take: "+numberofvectors); String query = String.format(discrepancyQuery, FirstTableCsquareColumn, SecondTableCsquareColumn, FirstTableProbabilityColumn, SecondTableProbabilityColumn, FirstTable, SecondTable,""+numberofvectors); - AnalysisLogger.getLogger().debug("Discrepancy Calculation - Query to perform :" + query); + logger.debug("Discrepancy Calculation - Query to perform :" + query); List takePoints = DatabaseFactory.executeSQLQuery(query, connection); super.processedRecords = 0; @@ -164,9 +167,9 @@ public class DiscrepancyAnalysis extends DataAnalysis { if (maxdiscrepancyPoint==null) maxdiscrepancyPoint="-"; - AnalysisLogger.getLogger().debug("Discrepancy Calculation - Kappa values: " + "agreementA1B1 "+agreementA1B1 +" agreementA1B0 " + agreementA1B0 +" agreementA0B1 "+agreementA0B1+" agreementA0B0 "+agreementA0B0); + logger.debug("Discrepancy Calculation - Kappa values: " + "agreementA1B1 "+agreementA1B1 +" agreementA1B0 " + agreementA1B0 +" agreementA0B1 "+agreementA0B1+" agreementA0B0 "+agreementA0B0); double kappa = MathFunctions.cohensKappaForDichotomy(agreementA1B1, agreementA1B0, agreementA0B1, agreementA0B0); - AnalysisLogger.getLogger().debug("Discrepancy Calculation - Calculated Cohen's Kappa:" + kappa); + logger.debug("Discrepancy Calculation - Calculated Cohen's Kappa:" + kappa); output = new LinkedHashMap(); output.put("MEAN", "" + MathFunctions.roundDecimal(mean,2)); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DistributionQualityAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DistributionQualityAnalysis.java index 7beee7b..0187f08 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DistributionQualityAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/DistributionQualityAnalysis.java @@ -8,8 +8,6 @@ import java.util.List; import java.util.Map; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; @@ -19,6 +17,8 @@ import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.example.Attribute; import com.rapidminer.example.Attributes; @@ -33,6 +33,8 @@ import com.rapidminer.tools.math.ROCDataGenerator; public class DistributionQualityAnalysis extends DataAnalysis { + private static Logger logger = LoggerFactory.getLogger(DistributionQualityAnalysis.class); + static String getProbabilititesQuery = "select count(*) from (select distinct * from %1$s as a join %2$s as b on a.%3$s=b.%4$s and b.%5$s %6$s %7$s) as aa"; static String getNumberOfElementsQuery = "select count(*) from %1$s"; static String getValuesQuery = "select %5$s as distribprob (select distinct * from %1$s as a join %2$s as b on a.%3$s=b.%4$s) as b"; @@ -110,7 +112,7 @@ public class DistributionQualityAnalysis extends DataAnalysis { private int calculateCaughtPoints(String casesTable, String distributionTable, String casesTableKeyColumn, String distributionTableKeyColumn, String distributionTableProbabilityColumn, String operator, String threshold) { String query = String.format(getProbabilititesQuery, casesTable, distributionTable, casesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn, operator, threshold); - AnalysisLogger.getLogger().trace("Compare - Query to perform for caught cases:" + query); + logger.trace("Compare - Query to perform for caught cases:" + query); List caughtpoints = DatabaseFactory.executeSQLQuery(query, connection); int points = Integer.parseInt("" + caughtpoints.get(0)); return points; @@ -120,7 +122,7 @@ public class DistributionQualityAnalysis extends DataAnalysis { String query = String.format(getValuesQuery, casesTable, distributionTable, casesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn); - AnalysisLogger.getLogger().trace("Compare - Query to perform for caught cases:" + query); + logger.trace("Compare - Query to perform for caught cases:" + query); List caughtpoints = DatabaseFactory.executeSQLQuery(query, connection); int size = 0; if (caughtpoints != null) @@ -143,12 +145,12 @@ public class DistributionQualityAnalysis extends DataAnalysis { try { acceptanceThreshold = Float.parseFloat(config.getParam("PositiveThreshold")); } catch (Exception e) { - AnalysisLogger.getLogger().debug("ERROR : " + e.getLocalizedMessage()); + logger.debug("ERROR : " + e.getLocalizedMessage()); } try { rejectionThreshold = Float.parseFloat(config.getParam("NegativeThreshold")); } catch (Exception e) { - AnalysisLogger.getLogger().debug("ERROR : " + e.getLocalizedMessage()); + logger.debug("ERROR : " + e.getLocalizedMessage()); } String positiveCasesTable = config.getParam("PositiveCasesTable"); @@ -270,60 +272,6 @@ public class DistributionQualityAnalysis extends DataAnalysis { System.out.println(key + ":" + results.get(key)); } } - - public static void main(String[] args) { - - AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile); - /* - * double [] pos = new double [4]; // pos[0] = 1d; pos[1] = 0.8d;pos[2]=0.7;pos[3]=0.9; // pos[0] = 1d; pos[1] = 1d;pos[2]=1;pos[3]=1; // pos[0] = 0.3d; pos[1] = 0.7d;pos[2]=0.1;pos[3]=0.9; - * - * - * double [] neg = new double [4]; // neg[0] = 0d; neg[1] = 0.3d;neg[2]=0.4;neg[3]=0.6; // neg[0] = 0d; neg[1] = 0.0d;neg[2]=0.0;neg[3]=0.0; // neg[0] = 0.7d; neg[1] = 0.3d;neg[2]=0.9;neg[3]=0.1; - * - * DistributionQualityAnalysis quality = new DistributionQualityAnalysis(); double auc = quality.calculateAUC(pos, neg); System.out.println("AUC: "+auc); - * - * int n = 100; double[] posRandom = new double[n]; double[] negRandom = new double[n]; - * - * for (int i=0;i caughtpoints = DatabaseFactory.executeSQLQuery(query, connection); int size = 0; if (caughtpoints != null) @@ -122,28 +125,28 @@ public class HabitatRepresentativeness extends DataAnalysis { private void calcHRS(String projectingAreaTable, String projectingAreaFeaturesOptionalCondition, String FeaturesColumns, String positiveCasesTable, String negativeCasesTable,int numberOfElements) throws Exception{ innerstatus = 0f; int numberOfElementsToTake = Operations.calcNumOfRepresentativeElements(numberOfElements, minimumNumberToTake); - AnalysisLogger.getLogger().trace("HRS: TAKING "+numberOfElementsToTake+" POINTS ON "+numberOfElements+" FROM THE AREA UNDER ANALYSIS"); + logger.trace("HRS: TAKING "+numberOfElementsToTake+" POINTS ON "+numberOfElements+" FROM THE AREA UNDER ANALYSIS"); // 1 - take the right number of points double[][] areaPoints = getPoints(projectingAreaTable, projectingAreaFeaturesOptionalCondition, FeaturesColumns, numberOfElementsToTake); - AnalysisLogger.getLogger().trace("HRS: AREA POINTS MATRIX GENERATED"); + logger.trace("HRS: AREA POINTS MATRIX GENERATED"); innerstatus = 10f; Operations operations = new Operations(); // 2 - standardize the matrix areaPoints = operations.standardize(areaPoints); - AnalysisLogger.getLogger().trace("HRS: MATRIX HAS BEEN STANDARDIZED"); + logger.trace("HRS: MATRIX HAS BEEN STANDARDIZED"); innerstatus = 20f; // 3 - calculate PCA PrincipalComponentAnalysis pca = new PrincipalComponentAnalysis(); pca.calcPCA(areaPoints); - AnalysisLogger.getLogger().trace("HRS: PCA HAS BEEN TRAINED"); + logger.trace("HRS: PCA HAS BEEN TRAINED"); innerstatus = 30f; // 4 - get the pca components for all the vector double[][] pcaComponents = pca.getComponentsMatrix(areaPoints); - AnalysisLogger.getLogger().trace("HRS: PCA COMPONENT CALCULATED"); + logger.trace("HRS: PCA COMPONENT CALCULATED"); innerstatus = 40f; // 5 - calculate the frequency distributions for all the pca: each row will be a frequency distribution for a pca component associated to uniform divisions of the range calcFrequenciesDistributionsForComponents(pcaComponents); - AnalysisLogger.getLogger().trace("HRS: FREQUENCIES FOR COMPONENTS CALCULATED"); + logger.trace("HRS: FREQUENCIES FOR COMPONENTS CALCULATED"); innerstatus = 50f; // 6 - take positive points and negative points - eventually merge them double[][] positivePoints = null; @@ -153,14 +156,14 @@ public class HabitatRepresentativeness extends DataAnalysis { if ((negativeCasesTable!=null) && (negativeCasesTable.length()>0)) negativePoints = getPoints(negativeCasesTable, "", FeaturesColumns, numberOfElementsToTake); double[][] habitatPoints = Transformations.mergeMatrixes(positivePoints, negativePoints); - AnalysisLogger.getLogger().trace("HRS: HABITAT POINTS BUILT FROM POSITIVE AND NEGATIVE POINTS"); + logger.trace("HRS: HABITAT POINTS BUILT FROM POSITIVE AND NEGATIVE POINTS"); innerstatus = 60f; // 7 - Standardize the points respect to previous means and variances habitatPoints = operations.standardize(habitatPoints, operations.means, operations.variances); - AnalysisLogger.getLogger().trace("HRS: HABITAT POINTS HAVE BEEN STANDARDIZED RESPECT TO PREVIOUS MEANS AND VARIANCES"); + logger.trace("HRS: HABITAT POINTS HAVE BEEN STANDARDIZED RESPECT TO PREVIOUS MEANS AND VARIANCES"); // 8 - calculate the pca components for habitat double[][] habitatPcaComponents = pca.getComponentsMatrix(habitatPoints); - AnalysisLogger.getLogger().trace("HRS: HABITAT POINTS HAVE BEEN TRANSFORMED BY PCA"); + logger.trace("HRS: HABITAT POINTS HAVE BEEN TRANSFORMED BY PCA"); innerstatus = 70f; // 9 - calculate frequencies distributions for each component, respect to previous intervals int components = habitatPcaComponents[0].length; @@ -177,13 +180,13 @@ public class HabitatRepresentativeness extends DataAnalysis { currentHRSVector[i] = Operations.sumVector(absdifference); } - AnalysisLogger.getLogger().trace("HRS: HRS VECTOR HAS BEEN CALCULATED"); + logger.trace("HRS: HRS VECTOR HAS BEEN CALCULATED"); innerstatus = 90f; // 11 - obtain hrsScore by weighted sum of hrs respect to inverse eigenvalues - too variable, substituted with the sum of the scores // currentHRSScore = Operations.scalarProduct(currentHRSVector, pca.getInverseNormalizedEigenvalues()); currentHRSScore = Operations.sumVector(currentHRSVector); - AnalysisLogger.getLogger().trace("HRS: HRS SCORE HAS BEEN CALCULATED"); + logger.trace("HRS: HRS SCORE HAS BEEN CALCULATED"); innerstatus = 100f; } private double meanHRS ; @@ -206,7 +209,7 @@ public class HabitatRepresentativeness extends DataAnalysis { for (int i=0;i0"); - config.setParam("FeaturesColumns", "depthmean,depthmax,depthmin, sstanmean,sbtanmean,salinitymean,salinitybmean, primprodmean,iceconann,landdist,oceanarea"); - config.setParam("PositiveCasesTable", "presence_data_baskingshark"); - config.setParam("NegativeCasesTable", "absence_data_baskingshark_random"); -// config.setParam("NegativeCasesTable", "absence_data_baskingshark2"); - - HabitatRepresentativeness hsrcalc = new HabitatRepresentativeness(); - hsrcalc.setConfiguration(config); - hsrcalc.init(); - - HashMap output = hsrcalc.analyze(); - for (String param:output.keySet()){ - System.out.println(param+":"+output.get(param)); - } - /* - double[][] matrix = new double[7][2]; - double[] row1 = { 2d, 3d }; - double[] row2 = { 3d, 4d }; - double[] row3 = { 4d, 5d }; - double[] row4 = { 5d, 6d }; - double[] row5 = { 2d, 3d }; - double[] row6 = { 2d, 5d }; - double[] row7 = { 3d, 4d }; - - matrix[0] = row1; - matrix[1] = row2; - matrix[2] = row3; - matrix[3] = row4; - matrix[4] = row5; - matrix[5] = row6; - matrix[6] = row7; -*/ - // Operations operations = new Operations(); - // matrix = operations.standardize(matrix); - // hsrcalc.calcFrequenciesDistributionsForComponents(matrix); - -// double[][] bigmat = Transformations.mergeMatrixes(null, matrix); - - System.out.println("FINISHED"); - } - @Override public StatisticalType getOutput() { PrimitiveType p = new PrimitiveType(Map.class.getName(), PrimitiveType.stringMap2StatisticalMap(output), PrimitiveTypes.MAP, "AnalysisResult","Habitat Representativeness Score"); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java index b8253fd..8ea63bb 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java @@ -8,25 +8,22 @@ import java.util.UUID; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; -import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; -import org.gcube.dataanalysis.ecoengine.interfaces.Evaluator; -import org.gcube.dataanalysis.ecoengine.models.cores.aquamaps.Hspen; import org.gcube.dataanalysis.ecoengine.processing.factories.EvaluatorsFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.Operations; -import org.gcube.dataanalysis.ecoengine.utils.Tuple; import org.hibernate.SessionFactory; import org.jfree.data.category.DefaultCategoryDataset; - -import com.mchange.v1.util.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BioClimateAnalysis { + private static Logger logger = LoggerFactory.getLogger(BioClimateAnalysis.class); + private static String countHighProbabilityCells = "select count(*) from %1$s where probability>%2$s"; private static String countSeaCells = "select count(*) from %1$s as a join %2$s as b on a.oceanarea>0 and a.csquarecode=b.csquarecode and ((a.iceconannb.iceconann+%3$s) or " + "(a.salinitymeanb.salinitymean+%3$s) or (a.sstanmeanb.sstanmean+%3$s))"; private static String iceLeakage = "select count(*) from %1$s as a join %2$s as b on a.csquarecode=b.csquarecode and (a.iceconann0"; @@ -180,7 +177,7 @@ public class BioClimateAnalysis { } - AnalysisLogger.getLogger().trace("Produced All Images"); + logger.trace("Produced All Images"); } @@ -251,7 +248,7 @@ public class BioClimateAnalysis { public void speciesEvolutionAnalysis(String[] hspenTables, String[] hspenTableNames, String parameterName, String condition) throws Exception { try { referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); status = 0f; int numbOfTables = (hspenTables != null) ? hspenTables.length : 0; @@ -265,7 +262,7 @@ public class BioClimateAnalysis { if (condition != null && (condition.length() > 0)) condition = "and " + condition; // take the spectrum of depths - AnalysisLogger.getLogger().trace("Range query: " + String.format(takeRangeOfParam, parameterName, hspenTables[0], condition)); + logger.trace("Range query: " + String.format(takeRangeOfParam, parameterName, hspenTables[0], condition)); List paramrange = DatabaseFactory.executeSQLQuery(String.format(takeRangeOfParam, parameterName, hspenTables[0], condition), referencedbConnection); @@ -301,12 +298,12 @@ public class BioClimateAnalysis { if (prevpmax != pmax) { // take the number of elements for this range String countSpeciesQuery = String.format(countNumberOfSpeciesPerRange, hspenTables[i], prevpmax, pmax, parameterName); - AnalysisLogger.getLogger().trace("count elements query: " + countSpeciesQuery); + logger.trace("count elements query: " + countSpeciesQuery); List elementsInRange = DatabaseFactory.executeSQLQuery(countSpeciesQuery, referencedbConnection); int nelements = (elementsInRange == null) ? 0 : Integer.parseInt("" + elementsInRange.get(0)); - AnalysisLogger.getLogger().trace("Number of elements for " + hspenTables[i] + " in (" + prevpmax + " - " + pmax + ")" + " : " + nelements); + logger.trace("Number of elements for " + hspenTables[i] + " in (" + prevpmax + " - " + pmax + ")" + " : " + nelements); // take the chart for this range String chartName = parameterName + " envelope for interval (" + prevpmax + " ; " + pmax + ")"; @@ -355,7 +352,7 @@ public class BioClimateAnalysis { public void speciesGeographicEvolutionAnalysis2(String[] hspecTables, String[] hspecTablesNames, float threshold) throws Exception { try { referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); status = 0f; int numbOfTables = (hspecTables != null) ? hspecTables.length : 0; @@ -375,9 +372,9 @@ public class BioClimateAnalysis { String code$ = "" + code; String query = String.format(countProbabilityPerArea, hspecTables[i], selectionCriteria[j], code$); query = query.replace("#THRESHOLD#", "" + threshold); - AnalysisLogger.getLogger().trace("Executing query for counting probabilities: " + query); + logger.trace("Executing query for counting probabilities: " + query); List counts = DatabaseFactory.executeSQLQuery(query, referencedbConnection); - AnalysisLogger.getLogger().trace("Query Executed"); + logger.trace("Query Executed"); int countPerArea = (counts == null) ? 0 : Integer.parseInt("" + counts.get(0)); String chartName = "Hspec (prob>0.8) for " + criteriaNames[j] + "_" + code$; // put the code and the value in the timeseries associated to the feature name @@ -417,7 +414,7 @@ public class BioClimateAnalysis { public void speciesGeographicEvolutionAnalysis(String[] hspecTables, String[] hspecTablesNames, float threshold) throws Exception { try { referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); status = 0f; int numbOfTables = (hspecTables != null) ? hspecTables.length : 0; @@ -434,26 +431,26 @@ public class BioClimateAnalysis { try { DatabaseFactory.executeSQLUpdate("drop table " + tmpanalysisTable, referencedbConnection); } catch (Exception ee) { - AnalysisLogger.getLogger().trace("table " + tmpanalysisTable + " does not exist"); + logger.trace("table " + tmpanalysisTable + " does not exist"); } String preparationQuery = "create table " + tmpanalysisTable + " as select a.faoaream, lme,count(*) from %1$s as a where a.probability > #THRESHOLD# group by faoaream,lme;"; preparationQuery = String.format(preparationQuery, hspecTables[i]); preparationQuery = preparationQuery.replace("#THRESHOLD#", "" + threshold); - AnalysisLogger.getLogger().trace("Executing query for counting probabilities: " + preparationQuery); + logger.trace("Executing query for counting probabilities: " + preparationQuery); DatabaseFactory.executeSQLUpdate(preparationQuery, referencedbConnection); - AnalysisLogger.getLogger().trace("Query Executed"); + logger.trace("Query Executed"); // for each criterion to apply: fao area, lme etc. for (int j = 0; j < criteriaNames.length; j++) { String criteriaQuery = String.format("select %1$s,sum(count) from " + tmpanalysisTable + " %2$s group by %1$s;", selectionCriteria[j], criteriaFilters[j]); - AnalysisLogger.getLogger().trace("Executing query for counting probabilities: " + criteriaQuery); + logger.trace("Executing query for counting probabilities: " + criteriaQuery); List codeSums = DatabaseFactory.executeSQLQuery(criteriaQuery, referencedbConnection); for (Object codeSum : codeSums) { String code$ = "" + ((Object[]) codeSum)[0]; int countPerArea = (((Object[]) codeSum)[1] == null) ? 0 : Integer.parseInt("" + ((Object[]) codeSum)[1]); - AnalysisLogger.getLogger().trace("Analyzing " + selectionCriteria[j] + " with code " + code$ + " count " + countPerArea); + logger.trace("Analyzing " + selectionCriteria[j] + " with code " + code$ + " count " + countPerArea); String chartName = "Hspec (prob>0.8) for " + criteriaNames[j] + "_" + code$; // put the code and the value in the timeseries associated to the feature name @@ -479,7 +476,7 @@ public class BioClimateAnalysis { DatabaseFactory.executeSQLUpdate("drop table " + tmpanalysisTable, referencedbConnection); } catch (Exception ee) { ee.printStackTrace(); - AnalysisLogger.getLogger().trace("table " + tmpanalysisTable + " does not exist"); + logger.trace("table " + tmpanalysisTable + " does not exist"); } status = status + statusstep; @@ -507,7 +504,7 @@ public class BioClimateAnalysis { public void geographicEvolutionAnalysis(String[] hcafTable, String[] hcafTableNames) throws Exception { try { referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); doHcafAn = true; doHspecAn = true; status = 0f; @@ -528,7 +525,7 @@ public class BioClimateAnalysis { // for each quantity to display: ice concentration for (int k = 0; k < quantitiesNames.length; k++) { String query = String.format(takeAvgSelection, quantitiesNames[k], selectionCriteria[j], hcafTable[i], criteriaFilters[j]); - AnalysisLogger.getLogger().debug("Query to be executed : " + query); + logger.debug("Query to be executed : " + query); // take couples (avg,code) List quantityCriterion = DatabaseFactory.executeSQLQuery(query, referencedbConnection); // for each row @@ -586,7 +583,7 @@ public class BioClimateAnalysis { public void globalEvolutionAnalysis(String[] hcafTable, String[] hspecTables, String[] hcafTablesNames, String[] hspecTableNames, String probabilityColumn, String csquareColumn, float threshold) throws Exception { try { referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); doHcafAn = true; doHspecAn = true; @@ -620,7 +617,7 @@ public class BioClimateAnalysis { avgIce[i] = avgValue(hcafTable[i], FIELD.iceconann.name()); avgSST[i] = avgValue(hcafTable[i], FIELD.sstanmean.name()); avgSalinity[i] = avgValue(hcafTable[i], FIELD.salinitymean.name()); - AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "): " + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); + logger.trace("(" + hcafTable[i] + "): " + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); } if (doHspecAn) { @@ -631,10 +628,10 @@ public class BioClimateAnalysis { // discrepancies[i] = MathFunctions.roundDecimal(calcOverDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5); discrepancies[i] = highProbabilityCells[i]-highProbabilityCells[i-1]; } - AnalysisLogger.getLogger().trace("(" + hspecTables[i] + "): DISCREPANCY " + discrepancies[i] + " HIGH PROB CELLS " + highProbabilityCells[i]); + logger.trace("(" + hspecTables[i] + "): DISCREPANCY " + discrepancies[i] + " HIGH PROB CELLS " + highProbabilityCells[i]); } - // AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "," + hspecTables[i] + "): HIGH PROB CELLS " + highProbabilityCells[i] + " DISCREPANCY " + discrepancies[i] + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); + // logger.trace("(" + hcafTable[i] + "," + hspecTables[i] + "): HIGH PROB CELLS " + highProbabilityCells[i] + " DISCREPANCY " + discrepancies[i] + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); status = status + statusstep; } @@ -661,7 +658,6 @@ public class BioClimateAnalysis { if (!persistencePath.endsWith("/")) this.temporaryDirectory += "/"; - AnalysisLogger.setLogger(configPath + AlgorithmConfiguration.defaultLoggerFile); config = new LexicalEngineConfiguration(); config.setDatabaseURL(databaseURL); config.setDatabaseUserName(databaseUserName); @@ -671,10 +667,10 @@ public class BioClimateAnalysis { } public int calcHighProbabilityCells(String hspec, double probabilty) throws Exception { - AnalysisLogger.getLogger().trace("Calculating High Prob Cells: "+String.format(countHighProbabilityCells, hspec, probabilty)); + logger.trace("Calculating High Prob Cells: "+String.format(countHighProbabilityCells, hspec, probabilty)); List countage = DatabaseFactory.executeSQLQuery(String.format(countHighProbabilityCells, hspec, probabilty), referencedbConnection); int count = Integer.parseInt("" + countage.get(0)); - AnalysisLogger.getLogger().trace("Calc High Prob Cells: " + count); + logger.trace("Calc High Prob Cells: " + count); return count; } @@ -710,18 +706,18 @@ public class BioClimateAnalysis { List nelementsQ1 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(firstTable), referencedbConnection); int nelements = Integer.parseInt("" + nelementsQ1.get(0)); - AnalysisLogger.getLogger().trace("Number Of elements1: " + nelementsQ1); + logger.trace("Number Of elements1: " + nelementsQ1); List nelementsQ2 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(secondTable), referencedbConnection); int nelements2 = Integer.parseInt("" + nelementsQ2.get(0)); - AnalysisLogger.getLogger().trace("Number Of elements2: " + nelementsQ1); + logger.trace("Number Of elements2: " + nelementsQ1); List sumFirst = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(firstTable, probabilityColumnName), referencedbConnection); double sum1 = Double.parseDouble("" + sumFirst.get(0)); - AnalysisLogger.getLogger().trace("Sum1: " + sum1); + logger.trace("Sum1: " + sum1); List sumSecond = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(secondTable, probabilityColumnName), referencedbConnection); double sum2 = Double.parseDouble("" + sumSecond.get(0)); - AnalysisLogger.getLogger().trace("Sum2: " + sum1); + logger.trace("Sum2: " + sum1); double d = (double) (sum2 - sum1) / (double) (nelements + nelements2); return d; diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/InterpolateTables.java b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/InterpolateTables.java index 5e1985b..e8e2e60 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/InterpolateTables.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/InterpolateTables.java @@ -4,18 +4,18 @@ import java.io.File; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; -import java.util.UUID; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.DataTypeRecognizer; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.Operations; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * checks if two tables are equal checks numbers at the second decimal position @@ -24,6 +24,8 @@ public class InterpolateTables { // connection setup + private static Logger logger = LoggerFactory.getLogger(InterpolateTables.class); + protected String temporaryDirectory; // selection query public static String selectElementsQuery = "select %1$s from %2$s order by %3$s"; @@ -53,9 +55,8 @@ public class InterpolateTables { if (!persistencePath.endsWith("/")) this.temporaryDirectory += "/"; - AnalysisLogger.setLogger(configPath + AlgorithmConfiguration.defaultLoggerFile); - - AnalysisLogger.getLogger().debug("Initialization complete: persistence path " + persistencePath); + + logger.debug("Initialization complete: persistence path " + persistencePath); config = new LexicalEngineConfiguration(); config.setDatabaseURL(databaseURL); @@ -71,16 +72,16 @@ public class InterpolateTables { interpolatedTables = new String[2]; interpolatedTables[0] = table1; interpolatedTables[1] = table2; - AnalysisLogger.getLogger().debug("NO TABLES TO PRODUCE"); + logger.debug("NO TABLES TO PRODUCE"); } else { referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); status = 0f; - AnalysisLogger.getLogger().debug("Interpolating from " + table1 + " to " + table2); + logger.debug("Interpolating from " + table1 + " to " + table2); DatabaseUtils utils = new DatabaseUtils(referencedbConnection); // analyze table and take information about it String createTableStatement = utils.buildCreateStatement(table1, "%1$s"); - AnalysisLogger.getLogger().debug("Create Statement for table " + table1 + ": " + createTableStatement); + logger.debug("Create Statement for table " + table1 + ": " + createTableStatement); int numberOfColumns = utils.getColumnDecriptions().size(); // initialize the map of columns to write List> outputFiles = new ArrayList>(); @@ -97,19 +98,19 @@ public class InterpolateTables { String javatype = DataTypeRecognizer.transformTypeFromDB(gotColumnType); String takeF = DatabaseUtils.getOrderedElements(table1, utils.getPrimaryKey(), gotColumn); String takeS = DatabaseUtils.getOrderedElements(table2, utils.getPrimaryKey(), gotColumn); - AnalysisLogger.getLogger().debug("Taking First column->" + takeF); - AnalysisLogger.getLogger().debug("Taking Second column->" + takeS); + logger.debug("Taking First column->" + takeF); + logger.debug("Taking Second column->" + takeS); List takeFirstColumn = DatabaseFactory.executeSQLQuery(takeF, referencedbConnection); List takeSecondColumn = DatabaseFactory.executeSQLQuery(takeS, referencedbConnection); - AnalysisLogger.getLogger().debug("First column elements size->" + takeFirstColumn.size()); - AnalysisLogger.getLogger().debug("Second column elements size->" + takeSecondColumn.size()); + logger.debug("First column elements size->" + takeFirstColumn.size()); + logger.debug("Second column elements size->" + takeSecondColumn.size()); // only if data are of numeric type, perform calculation if (javatype.equals(BigDecimal.class.getName())) { - AnalysisLogger.getLogger().debug("interpolating -> " + gotColumn); + logger.debug("interpolating -> " + gotColumn); List> interpolations = interpolateColumns(takeFirstColumn, takeSecondColumn, intervals, gotColumnType, function); @@ -119,10 +120,10 @@ public class InterpolateTables { // for each column to substitute List columnToSub = interpolations.get(i); if (columnToSub.size() > 0) { - AnalysisLogger.getLogger().debug("UPDATE TABLE " + tableInterp + " ON COLUMN " + gotColumn); + logger.debug("UPDATE TABLE " + tableInterp + " ON COLUMN " + gotColumn); addColumnToTable(outputFiles.get(i - 1), columnToSub, true); } else { - AnalysisLogger.getLogger().debug("DOESN'T CHANGE TABLE " + tableInterp + " COLUMN " + gotColumn); + logger.debug("DOESN'T CHANGE TABLE " + tableInterp + " COLUMN " + gotColumn); addColumnToTable(outputFiles.get(i - 1), takeFirstColumn, true); } } @@ -138,7 +139,7 @@ public class InterpolateTables { } status = 60f; - AnalysisLogger.getLogger().debug("WRITING ALL THE BUFFERS"); + logger.debug("WRITING ALL THE BUFFERS"); writeAllStringBuffersToFiles(table1, outputFiles, function, startYear, endYear); statusstep = 40f / (float) producedfiles.length; @@ -153,27 +154,27 @@ public class InterpolateTables { /*create Table from file*/ /* OLD CODE FOR LOCAL DB String copyFileQuery = DatabaseUtils.copyFileToTableStatement(temporaryDirectory + producedfiles[i].getName(), filename); - AnalysisLogger.getLogger().debug("CREATING TABLE->" + filename); + logger.debug("CREATING TABLE->" + filename); DatabaseFactory.executeSQLUpdate(String.format(createTableStatement, filename), referencedbConnection); - AnalysisLogger.getLogger().debug("FULFILLING TABLE->" + filename + ": " + copyFileQuery); + logger.debug("FULFILLING TABLE->" + filename + ": " + copyFileQuery); DatabaseFactory.executeSQLUpdate(copyFileQuery, referencedbConnection); */ - AnalysisLogger.getLogger().debug("CREATING TABLE->" + filename); + logger.debug("CREATING TABLE->" + filename); DatabaseFactory.executeSQLUpdate(String.format(createTableStatement, filename), referencedbConnection); - AnalysisLogger.getLogger().debug("COPYING TABLE->" + filename); + logger.debug("COPYING TABLE->" + filename); DatabaseUtils.createRemoteTableFromFile(producedfiles[i].getAbsolutePath(),filename,";",false,config.getDatabaseUserName(),config.getDatabasePassword(),config.getDatabaseURL()); status = Math.min(status + statusstep, 99); } - AnalysisLogger.getLogger().debug("DELETING ALL TEMPORARY FILES"); + logger.debug("DELETING ALL TEMPORARY FILES"); for (int i = 0; i < producedfiles.length; i++) { producedfiles[i].delete(); } interpolatedTables[interpolatedTables.length - 1] = table2; - AnalysisLogger.getLogger().debug("ALL TABLES HAVE BEEN PRODUCED"); + logger.debug("ALL TABLES HAVE BEEN PRODUCED"); }//end else control on the number of intervals } catch (Exception e) { e.printStackTrace(); @@ -253,7 +254,7 @@ public class InterpolateTables { double[] interpolation = null; if (firstNum != secondNum) { if (interping) { - AnalysisLogger.getLogger().debug("Interpolating ... "); + logger.debug("Interpolating ... "); interping = false; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/DataAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/DataAnalysis.java index 9a71dca..53e682b 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/DataAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/DataAnalysis.java @@ -4,7 +4,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; @@ -12,6 +11,8 @@ import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Implements a mono-thread data analysis process @@ -21,6 +22,8 @@ import org.hibernate.SessionFactory; */ public abstract class DataAnalysis implements Evaluator{ + private static Logger logger = LoggerFactory.getLogger(DataAnalysis.class); + protected ResourceFactory resourceManager; protected int processedRecords; protected float status; @@ -112,7 +115,6 @@ public abstract class DataAnalysis implements Evaluator{ } public void init(boolean initRapidMiner) throws Exception { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); // init db connection connection = AlgorithmConfiguration.getConnectionFromConfig(config); if (initRapidMiner) @@ -126,10 +128,10 @@ public abstract class DataAnalysis implements Evaluator{ public void shutdown() { try { - AnalysisLogger.getLogger().debug("Closing DB connections"); + logger.debug("Closing DB connections"); connection.close(); } catch (Exception e) { - AnalysisLogger.getLogger().debug("Error in closing DB connections "+e.getLocalizedMessage()); + logger.debug("Error in closing DB connections "+e.getLocalizedMessage()); } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/FeedForwardNN.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/FeedForwardNN.java index 84dcf9d..266b035 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/FeedForwardNN.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/FeedForwardNN.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; @@ -21,9 +20,12 @@ import org.gcube.dataanalysis.ecoengine.interfaces.Model; import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FeedForwardNN extends ModelAquamapsNN{ + private static Logger logger = LoggerFactory.getLogger(FeedForwardNN.class); @Override public String getName() { @@ -95,13 +97,12 @@ public class FeedForwardNN extends ModelAquamapsNN{ @Override public void init(AlgorithmConfiguration config, Model previousModel) { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + // init the database try { connection = DatabaseUtils.initDBSession(config); } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().trace("ERROR initializing connection"); + logger.trace("ERROR initializing connection", e); } fileName = config.getPersistencePath()+Neural_Network.generateNNName(config.getParam(Reference), config.getParam(UserName), config.getParam(ModelName)); trainingDataSet = config.getParam(TrainingDataSet); @@ -147,7 +148,7 @@ public class FeedForwardNN extends ModelAquamapsNN{ try { // take all features input vectors String query = String.format(takeElementsQuery, trainingColumn+","+dbcolumns,trainingDataSet,trainingColumn); - AnalysisLogger.getLogger().debug("Query to execute: "+query); + logger.debug("Query to execute: "+query); List features = DatabaseFactory.executeSQLQuery(query, connection); int numbOfFeatures = features.size(); @@ -155,12 +156,12 @@ public class FeedForwardNN extends ModelAquamapsNN{ List maxmin = DatabaseFactory.executeSQLQuery("select max("+trainingColumn+"), min("+trainingColumn+") from "+trainingDataSet, connection); maxfactor = Double.parseDouble(""+((Object[])maxmin.get(0))[0]); minfactor = Double.parseDouble(""+((Object[])maxmin.get(0))[1]); - AnalysisLogger.getLogger().debug("Calculated max: "+maxfactor+" min: "+minfactor); + logger.debug("Calculated max: "+maxfactor+" min: "+minfactor); // setup Neural Network int numberOfInputNodes = dbcolumnsList.length; int numberOfOutputNodes = 1; - AnalysisLogger.getLogger().debug("Training the ANN with "+numbOfFeatures+" training data and "+numberOfInputNodes+" inputs"); + logger.debug("Training the ANN with "+numbOfFeatures+" training data and "+numberOfInputNodes+" inputs"); if (layersNeurons!=null){ int[] innerLayers = Neural_Network.setupInnerLayers(layersNeurons); nn = new Neural_Network(numberOfInputNodes, numberOfOutputNodes, innerLayers, Neural_Network.ACTIVATIONFUNCTION.SIGMOID); @@ -172,10 +173,10 @@ public class FeedForwardNN extends ModelAquamapsNN{ nn.minfactor=minfactor; nn.setThreshold(learningThr); nn.setCycles(maxiter); - AnalysisLogger.getLogger().debug("network parameters: M: "+maxfactor+", m: "+minfactor+", lt: "+learningThr+", it: "+maxiter); - AnalysisLogger.getLogger().debug("topology: "+nn.griglia.length+"X"+nn.griglia[0].length); + logger.debug("network parameters: M: "+maxfactor+", m: "+minfactor+", lt: "+learningThr+", it: "+maxiter); + logger.debug("topology: "+nn.griglia.length+"X"+nn.griglia[0].length); - AnalysisLogger.getLogger().debug("Features preprocessing"); + logger.debug("Features preprocessing"); double[][] in = new double[numbOfFeatures][]; double[][] out = new double[numbOfFeatures][]; // build NN input @@ -185,21 +186,20 @@ public class FeedForwardNN extends ModelAquamapsNN{ in[i] = Neural_Network.preprocessObjects(Arrays.copyOfRange((Object[]) features.get(i), 1, feats.length)); out[i] = Neural_Network.preprocessObjects(Arrays.copyOfRange((Object[]) features.get(i), 0, 1)); //apply reduction factor -// AnalysisLogger.getLogger().debug("Output Transformed from "+out[i][0]); +// logger.debug("Output Transformed from "+out[i][0]); out[i][0] =nn.getCorrectValueForOutput(out[i][0]); -// AnalysisLogger.getLogger().debug("To "+out[i][0]); +// logger.debug("To "+out[i][0]); } - AnalysisLogger.getLogger().debug("Features were correctly preprocessed - Training"); + logger.debug("Features were correctly preprocessed - Training"); // train the NN nn.train(in, out); learningscore=nn.en; - AnalysisLogger.getLogger().error("Final learning error: "+nn.en); - AnalysisLogger.getLogger().debug("Saving Network"); + logger.error("Final learning error: "+nn.en); + logger.debug("Saving Network"); save(fileName, nn); - AnalysisLogger.getLogger().debug("Done"); + logger.debug("Done"); } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().error("ERROR during training"); + logger.error("ERROR during training",e); } status = 100f; diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNN.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNN.java index e9082f3..034423f 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNN.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNN.java @@ -5,12 +5,9 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -27,9 +24,13 @@ import org.gcube.dataanalysis.ecoengine.interfaces.Model; import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ModelAquamapsNN implements Model { + private static Logger logger = LoggerFactory.getLogger(ModelAquamapsNN.class); + @Override public ALG_PROPS[] getProperties() { ALG_PROPS[] props = { ALG_PROPS.SPECIES_MODEL }; @@ -99,8 +100,7 @@ public class ModelAquamapsNN implements Model { @Override public void init(AlgorithmConfiguration Input, Model previousModel) { - AnalysisLogger.setLogger(Input.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); - + // init the database String defaultDatabaseFile = Input.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile; @@ -112,8 +112,7 @@ public class ModelAquamapsNN implements Model { try { connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, Input); } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().trace("ERROR initializing connection"); + logger.trace("ERROR initializing connection",e); } fileName = Input.getPersistencePath() + "neuralnetwork_" + Input.getParam("SpeciesName") + "_" + Input.getParam("UserName")+"_"+Input.getParam("NeuralNetworkName").replace(" ", ""); @@ -154,11 +153,11 @@ public class ModelAquamapsNN implements Model { @Override public void postprocess(AlgorithmConfiguration Input, Model previousModel) { - AnalysisLogger.getLogger().debug("Closing DB Connection"); + logger.debug("Closing DB Connection"); try{ connection.close(); }catch(Exception e){ - AnalysisLogger.getLogger().debug("Error in Closing DB Connection "+e.getLocalizedMessage()); + logger.debug("Error in Closing DB Connection "+e.getLocalizedMessage()); } } @@ -173,8 +172,8 @@ public class ModelAquamapsNN implements Model { // take all presence inputs List presences = DatabaseFactory.executeSQLQuery(String.format(takeElementsQuery, presenceTable), connection); // take all absence inputs -// AnalysisLogger.getLogger().trace("presence "+String.format(takeElementsQuery, presenceTable)); -// AnalysisLogger.getLogger().trace("absence "+String.format(takeElementsQuery, absenceTable)); +// logger.trace("presence "+String.format(takeElementsQuery, presenceTable)); +// logger.trace("absence "+String.format(takeElementsQuery, absenceTable)); List absences = DatabaseFactory.executeSQLQuery(String.format(takeElementsQuery, absenceTable), connection); int numbOfPresence = presences.size(); int numbOfAbsence = absences.size(); @@ -202,12 +201,12 @@ public class ModelAquamapsNN implements Model { // train the NN nn.train(in, out); learningscore=nn.en; - AnalysisLogger.getLogger().error("Final learning error: "+nn.en); + logger.error("Final learning error: "+nn.en); save(fileName, nn); } catch (Exception e) { e.printStackTrace(); - AnalysisLogger.getLogger().error("ERROR during training"); + logger.error("ERROR during training"); } status = 100f; } @@ -239,15 +238,14 @@ public class ModelAquamapsNN implements Model { ObjectOutputStream oos = new ObjectOutputStream(stream); oos.writeObject(nn); } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().error("ERROR in writing object on file: " + nomeFile); + logger.error("ERROR in writing object on file: " + nomeFile,e); } finally { try { stream.close(); } catch (IOException e) { } } - AnalysisLogger.getLogger().trace("OK in writing object on file: " + nomeFile); + logger.trace("OK in writing object on file: " + nomeFile); } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNNNS.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNNNS.java index bf20ae2..ede2009 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNNNS.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelAquamapsNNNS.java @@ -7,7 +7,6 @@ import java.io.ObjectOutputStream; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -24,9 +23,13 @@ import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutio import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions.Pattern; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ModelAquamapsNNNS implements Model { + private static Logger logger = LoggerFactory.getLogger(ModelAquamapsNN.class); + @Override public ALG_PROPS[] getProperties() { ALG_PROPS[] props = { ALG_PROPS.SPECIES_MODEL }; @@ -91,7 +94,6 @@ public class ModelAquamapsNNNS implements Model { @Override public void init(AlgorithmConfiguration Input, Model previousModel) { - AnalysisLogger.setLogger(Input.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); // init the database String defaultDatabaseFile = Input.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile; @@ -104,8 +106,7 @@ public class ModelAquamapsNNNS implements Model { try { connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, Input); } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().trace("ERROR initializing connection"); + logger.trace("ERROR initializing connection"); } fileName = Input.getPersistencePath() + "neuralnetwork_" + Input.getParam("SpeciesName") + "_" + Input.getParam("UserName"); @@ -169,14 +170,14 @@ public class ModelAquamapsNNNS implements Model { out[i] = nn.getPositiveCase(); Pattern pattern = new Pattern(in[i], out[i]); nn.IncrementalTrain(.2, pattern); - AnalysisLogger.getLogger().debug("-> "+i); + logger.debug("-> "+i); } for (int i = numbOfPresence; i < numberOfInputs; i++) { in[i] = NeuralNet.preprocessObjects((Object[]) absences.get(i-numbOfPresence)); out[i] = nn.getNegativeCase(); Pattern pattern = new Pattern(in[i], out[i]); nn.IncrementalTrain(.2, pattern); - AnalysisLogger.getLogger().debug("-> "+i); + logger.debug("-> "+i); } @@ -196,7 +197,7 @@ public class ModelAquamapsNNNS implements Model { } catch (Exception e) { e.printStackTrace(); - AnalysisLogger.getLogger().error("ERROR during training"); + logger.error("ERROR during training"); } status = 100f; } @@ -223,15 +224,14 @@ public class ModelAquamapsNNNS implements Model { ObjectOutputStream oos = new ObjectOutputStream(stream); oos.writeObject(nn); } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().error("ERROR in writing object on file: " + nomeFile); + logger.error("ERROR in writing object on file: " + nomeFile,e); } finally { try { stream.close(); } catch (IOException e) { } } - AnalysisLogger.getLogger().trace("OK in writing object on file: " + nomeFile); + logger.trace("OK in writing object on file: " + nomeFile); } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java index 6d5a853..dbfd5a0 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/ModelHSPEN.java @@ -8,7 +8,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.gcube.contentmanagement.graphtools.utils.HttpRequest; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; @@ -28,9 +27,13 @@ import org.gcube.dataanalysis.ecoengine.models.cores.aquamaps.AquamapsEnvelopeAl import org.gcube.dataanalysis.ecoengine.models.cores.aquamaps.EnvelopeSet; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ModelHSPEN implements Model { + private static Logger logger = LoggerFactory.getLogger(ModelHSPEN.class); + private float version; // DB SESSION protected SessionFactory connection; @@ -83,7 +86,6 @@ public class ModelHSPEN implements Model { outconfig = setup; defaultDatabaseFile = setup.getConfigPath() + defaultDatabaseFile; - AnalysisLogger.setLogger(setup.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); try { String defaultDatabaseFile = setup.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile; @@ -94,7 +96,7 @@ public class ModelHSPEN implements Model { connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, setup); } catch (Exception e) { - AnalysisLogger.getLogger().error("error intitializing",e); + logger.error("error intitializing",e); } outputTable = outconfig.getParam("OuputEnvelopeTable"); @@ -119,7 +121,7 @@ public class ModelHSPEN implements Model { // populates the selectedSpecies variable by getting species from db private List populateSpecies() { - AnalysisLogger.getLogger().trace("Distribution Generator ->getting all species list from DB"); + logger.trace("Distribution Generator ->getting all species list from DB"); List allspecies = DatabaseFactory.executeSQLQuery(dynamicSpeciesListQuery, connection); return allspecies; } @@ -174,20 +176,20 @@ public class ModelHSPEN implements Model { AlgorithmConfiguration config = (AlgorithmConfiguration) Input; // create and populate the novel table if (config.getParam("CreateTable").equalsIgnoreCase("true")){ - AnalysisLogger.getLogger().trace("Distribution Generator->recreating new table " + dynamicCreateTable); + logger.trace("Distribution Generator->recreating new table " + dynamicCreateTable); try{ DatabaseFactory.executeSQLUpdate(String.format(dynamicDropTable, config.getDatabaseUserName()), connection); }catch(Exception e){ - AnalysisLogger.getLogger().trace("Impossible to drop table - maybe not existing"); + logger.trace("Impossible to drop table - maybe not existing"); } try{ DatabaseFactory.executeSQLUpdate(String.format(dynamicCreateTable, config.getDatabaseUserName()), connection); }catch(Exception e){ - AnalysisLogger.getLogger().trace("Impossible to create table - maybe yet existing"); + logger.trace("Impossible to create table - maybe yet existing"); } } - AnalysisLogger.getLogger().trace("Distribution Generator->populating new table " + dynamicPopulateNewHspen); + logger.trace("Distribution Generator->populating new table " + dynamicPopulateNewHspen); DatabaseFactory.executeSQLUpdate(dynamicPopulateNewHspen, connection); } @@ -196,11 +198,11 @@ public class ModelHSPEN implements Model { long tstart = System.currentTimeMillis(); // INITIALIZATION try { - AnalysisLogger.getLogger().trace("ModelHSPENr->populating species"); + logger.trace("ModelHSPENr->populating species"); List allspecies = populateSpecies(); allSpeciesHspen = populateHspen(); - AnalysisLogger.getLogger().trace("ModelHSPENr->ENVELOPES GENERATION STARTED"); + logger.trace("ModelHSPENr->ENVELOPES GENERATION STARTED"); // initialize threads initializeThreads(numberOfthreads); @@ -225,7 +227,7 @@ public class ModelHSPEN implements Model { String speciesid = (String) species; if (speciesid.length() > 0) { // calculation on multiple threads - AnalysisLogger.getLogger().trace("ModelHSPENr->ANALIZING SPECIES: " + speciesid); + logger.trace("ModelHSPENr->ANALIZING SPECIES: " + speciesid); // wait for thread to be free wait4Thread(currentThread); // start species information calculation on the thread @@ -238,11 +240,11 @@ public class ModelHSPEN implements Model { // report probability float s = (float) ((int) (((float) globalcounter * 100f / (numberOfSpecies)) * 100f)) / 100f; status = (s == 100) ? 99 : s; - AnalysisLogger.getLogger().trace("STATUS->" + status + "%"); + logger.trace("STATUS->" + status + "%"); // increment global counter index globalcounter++; - AnalysisLogger.getLogger().warn("Number of Found Differences: " + countDifferences); + logger.warn("Number of Found Differences: " + countDifferences); } if (interruptProcessing) @@ -258,10 +260,10 @@ public class ModelHSPEN implements Model { } long computationT1 = System.currentTimeMillis(); - AnalysisLogger.getLogger().warn("All Envelopes Computation Finished in " + (computationT1 - computationT0) + " ms"); - AnalysisLogger.getLogger().warn("Number of Overall Found Differences: " + countDifferences); + logger.warn("All Envelopes Computation Finished in " + (computationT1 - computationT0) + " ms"); + logger.warn("Number of Overall Found Differences: " + countDifferences); } catch (Exception e) { - AnalysisLogger.getLogger().trace("Computation traminate prematurely: ", e); + logger.trace("Computation traminate prematurely: ", e); } finally { // shutdown threads executorService.shutdown(); @@ -270,7 +272,7 @@ public class ModelHSPEN implements Model { // set completeness status = 100.0f; long tstop = System.currentTimeMillis(); - AnalysisLogger.getLogger().warn("All Envelopes Computation Finished in " + (tstop - tstart) + " ms"); + logger.warn("All Envelopes Computation Finished in " + (tstop - tstart) + " ms"); } } @@ -291,7 +293,7 @@ public class ModelHSPEN implements Model { try { calcEnvelopes(species); } catch (Exception e) { - AnalysisLogger.getLogger().trace("" + e); + logger.trace("" + e); e.printStackTrace(); } threadActivity[index] = false; @@ -321,24 +323,24 @@ public class ModelHSPEN implements Model { String instruction = envSet.getEnvelopeString(); // take the result of the calculation long t1 = System.currentTimeMillis(); - AnalysisLogger.getLogger().trace("Computation for species " + species + " finished in " + (t1 - t0) + " ms"); + logger.trace("Computation for species " + species + " finished in " + (t1 - t0) + " ms"); if (instruction.length() > 0) { countDifferences++; // write results on the DB String query = String.format(dynamicAlterQuery, instruction, species); try { - AnalysisLogger.getLogger().trace("Envelope Generated - executing query: " + query); + logger.trace("Envelope Generated - executing query: " + query); DatabaseFactory.executeSQLUpdate(query, connection); } catch (Exception e) { - AnalysisLogger.getLogger().trace("could not execute update"); + logger.trace("could not execute update"); e.printStackTrace(); // System.exit(0); } } } catch (Exception ex) { - AnalysisLogger.getLogger().trace("Computation traminated prematurely: ", ex); + logger.trace("Computation traminated prematurely: ", ex); } numbOfProcessedSpecies++; // take ending time diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/AquamapsEnvelopeAlgorithm.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/AquamapsEnvelopeAlgorithm.java index d960fc0..ba77824 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/AquamapsEnvelopeAlgorithm.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/AquamapsEnvelopeAlgorithm.java @@ -3,12 +3,14 @@ package org.gcube.dataanalysis.ecoengine.models.cores.aquamaps; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AquamapsEnvelopeAlgorithm { + private static Logger logger = LoggerFactory.getLogger(AquamapsEnvelopeAlgorithm.class); private static final String selectValues = "SELECT DISTINCT %OCCURRENCEPOINTS%.CsquareCode, %OCCURRENCEPOINTS%.SpeciesID, %HCAF%.%1$s FROM %OCCURRENCEPOINTS% INNER JOIN %HCAF% ON %OCCURRENCEPOINTS%.CsquareCode = %HCAF%.CsquareCode WHERE %OCCURRENCEPOINTS%.SpeciesID = '%2$s' AND %HCAF%.%1$s <> -9999 AND %HCAF%.%1$s is not null AND %HCAF%.OceanArea > 0 AND %OCCURRENCEPOINTS%.goodcell = '1' ORDER BY %HCAF%.%1$s"; @@ -72,8 +74,8 @@ public class AquamapsEnvelopeAlgorithm { //the core of the procedure public static EnvelopeSet calcEnv(String species, Object[] singleSpeciesValues, List tempvalues,List salinityvalues,List primprodvalues,List icevalues,List landdistvalues){ if (tempvalues.size()<10){ - AnalysisLogger.getLogger().warn("WARNING: NOT ENOUGH OCCURRENCES FOR SPECIES: "+species); - AnalysisLogger.getLogger().warn("Leaving the hspen as is"); + logger.warn("WARNING: NOT ENOUGH OCCURRENCES FOR SPECIES: "+species); + logger.warn("Leaving the hspen as is"); return new EnvelopeSet(); } //take previousValues @@ -120,13 +122,13 @@ public class AquamapsEnvelopeAlgorithm { int countchunks = 0; if (!tempEnv.checkPrevious(prevTempMin,prevTempMax,prevTempPMin,prevTempPMax)) { - AnalysisLogger.getLogger().warn("DIFFERENCE ON SPECIES: "+species+" - "+prevTempMin+","+prevTempPMin+","+prevTempPMax+","+prevTempMax+" vs "+tempEnv.toString()); + logger.warn("DIFFERENCE ON SPECIES: "+species+" - "+prevTempMin+","+prevTempPMin+","+prevTempPMax+","+prevTempMax+" vs "+tempEnv.toString()); addingElements+=tempEnv.toString(); countchunks++; } if (!salinityEnv.checkPrevious(prevSalinityMin,prevSalinityMax,prevSalinityPMin,prevSalinityPMax)) { - AnalysisLogger.getLogger().warn("DIFFERENCE ON SPECIES: "+species+" - "+prevSalinityMin+","+prevSalinityPMin+","+prevSalinityPMax+","+prevSalinityMax+" vs "+salinityEnv.toString()); + logger.warn("DIFFERENCE ON SPECIES: "+species+" - "+prevSalinityMin+","+prevSalinityPMin+","+prevSalinityPMax+","+prevSalinityMax+" vs "+salinityEnv.toString()); if (countchunks>0) addingElements+=","; addingElements+=salinityEnv.toString(); @@ -134,7 +136,7 @@ public class AquamapsEnvelopeAlgorithm { } if (!primprodEnv.checkPrevious(prevPrimProdMin,prevPrimProdMax,prevPrimProdPMin,prevPrimProdPMax)) { - AnalysisLogger.getLogger().warn("DIFFERENCE ON SPECIES: "+species+" - "+prevPrimProdMin+","+prevPrimProdPMin+","+prevPrimProdPMax+","+prevPrimProdMax+" vs "+primprodEnv.toString()); + logger.warn("DIFFERENCE ON SPECIES: "+species+" - "+prevPrimProdMin+","+prevPrimProdPMin+","+prevPrimProdPMax+","+prevPrimProdMax+" vs "+primprodEnv.toString()); if (countchunks>0) addingElements+=","; addingElements+=primprodEnv.toString(); @@ -142,7 +144,7 @@ public class AquamapsEnvelopeAlgorithm { } if (!seaiceEnv.checkPrevious(prevIceMin,prevIceMax,prevIcePMin,prevIcePMax)) { - AnalysisLogger.getLogger().warn("DIFFERENCE ON SPECIES: "+species+" - "+prevIceMin+","+prevIcePMin+","+prevIcePMax+","+prevIceMax+" vs "+seaiceEnv.toString()); + logger.warn("DIFFERENCE ON SPECIES: "+species+" - "+prevIceMin+","+prevIcePMin+","+prevIcePMax+","+prevIceMax+" vs "+seaiceEnv.toString()); if (countchunks>0) addingElements+=","; addingElements+=seaiceEnv.toString(); @@ -150,7 +152,7 @@ public class AquamapsEnvelopeAlgorithm { } if (!landdistEnv.checkPrevious(prevLanddistMin,prevLanddistMax,prevLanddistPMin,prevLanddistPMax)) { - AnalysisLogger.getLogger().warn("DIFFERENCE ON SPECIES: "+species+" - "+prevLanddistMin+","+prevLanddistPMin+","+prevLanddistPMax+","+prevLanddistPMax+" vs "+landdistEnv.toString()); + logger.warn("DIFFERENCE ON SPECIES: "+species+" - "+prevLanddistMin+","+prevLanddistPMin+","+prevLanddistPMax+","+prevLanddistPMax+" vs "+landdistEnv.toString()); if (countchunks>0) addingElements+=","; addingElements+=landdistEnv.toString(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/MaxMinGenerator.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/MaxMinGenerator.java index fe3fbf3..1ab5cff 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/MaxMinGenerator.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/aquamaps/MaxMinGenerator.java @@ -4,12 +4,15 @@ import java.util.ArrayList; import java.util.List; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MaxMinGenerator { + private static Logger logger = LoggerFactory.getLogger(MaxMinGenerator.class); + String selectionQuery = "SELECT DISTINCT Max(hcaf_s.CenterLat) AS maxCLat, Min(hcaf_s.CenterLat) AS minCLat,speciesid FROM #occurrencecells# INNER JOIN hcaf_s ON #occurrencecells#.CsquareCode = hcaf_s.CsquareCode WHERE (((hcaf_s.oceanarea > 0))) AND #occurrencecells#.SpeciesID in (%1$s) AND #occurrencecells#.GoodCell <> 0 group by speciesid; "; String insertionQuery = "insert into maxminlat_%4$s values ('%1$s','%2$s','%3$s'); "; @@ -78,7 +81,7 @@ public class MaxMinGenerator { } try { - AnalysisLogger.getLogger().debug("inserting..."); + logger.debug("inserting..."); // DatabaseFactory.executeSQLUpdate(insert, vreConnection); DatabaseFactory.executeSQLUpdate(buffer.toString(), vreConnection); } catch (Exception e) { @@ -100,7 +103,7 @@ public class MaxMinGenerator { if (counter % 100 == 0) { try { - AnalysisLogger.getLogger().debug("inserting..."); + logger.debug("inserting..."); // DatabaseFactory.executeSQLUpdate(insert, vreConnection); DatabaseFactory.executeSQLUpdate(buffer.toString(), vreConnection); } catch (Exception e) { @@ -114,11 +117,11 @@ public class MaxMinGenerator { public void getAllSpecies(String hspentable) { // populates the selectedSpecies variable by getting species from db String query = String.format(SpeciesListQuery, hspentable); - AnalysisLogger.getLogger().warn("Distribution Generator ->getting all species list from DB"); - AnalysisLogger.getLogger().warn("Distribution Generator ->" + query); + logger.warn("Distribution Generator ->getting all species list from DB"); + logger.warn("Distribution Generator ->" + query); List allspecies = DatabaseFactory.executeSQLQuery(query, vreConnection); selectedSpecies = allspecies; - // AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + selectedSpecies.size()); + // logger.warn("Distribution Generator -> SIZE: " + selectedSpecies.size()); } protected void deleteDestinationTable(String table) { @@ -126,9 +129,9 @@ public class MaxMinGenerator { // clean the corresponding table on destination db try { DatabaseFactory.executeSQLUpdate(deleteQuery, vreConnection); - AnalysisLogger.getLogger().debug("destination table dropped"); + logger.debug("destination table dropped"); } catch (Exception e) { - AnalysisLogger.getLogger().debug("destination table NOT dropped"); + logger.debug("destination table NOT dropped"); } } @@ -136,12 +139,12 @@ public class MaxMinGenerator { protected void buildDestinationTable(String destinationTable) { String createQuery = String.format(creationQuery, destinationTable); - AnalysisLogger.getLogger().debug("Creating new table or destination schema: " + destinationTable); + logger.debug("Creating new table or destination schema: " + destinationTable); try { DatabaseFactory.executeSQLUpdate(String.format(createQuery, destinationTable), vreConnection); - AnalysisLogger.getLogger().debug("Table created"); + logger.debug("Table created"); } catch (Exception e) { - AnalysisLogger.getLogger().debug("Table NOT created"); + logger.debug("Table NOT created"); } } @@ -155,7 +158,7 @@ public class MaxMinGenerator { deleteDestinationTable("maxminlat_" + hspenTable); buildDestinationTable("maxminlat_" + hspenTable); int size = selectedSpecies.size(); - AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + size); + logger.warn("Distribution Generator -> SIZE: " + size); for (Object species : selectedSpecies) { String speciesid = (String) species; executeQuery(speciesid, hspenTable); @@ -164,10 +167,10 @@ public class MaxMinGenerator { status = MathFunctions.roundDecimal(((double) i *100/ (double) size), 2); // status = (double) i / (double) size; if (i%10==0) - AnalysisLogger.getLogger().debug("status " + status); + logger.debug("status " + status); } long t1 = System.currentTimeMillis(); - AnalysisLogger.getLogger().debug("elapsed Time: " + (t1 - t0)); + logger.debug("elapsed Time: " + (t1 - t0)); } catch (Exception e) { e.printStackTrace(); @@ -189,7 +192,7 @@ public class MaxMinGenerator { deleteDestinationTable(destinationTable); buildDestinationTable(destinationTable); int size = selectedSpecies.size(); - AnalysisLogger.getLogger().warn("Distribution Generator -> SIZE: " + size); + logger.warn("Distribution Generator -> SIZE: " + size); List specieslist = new ArrayList(); for (Object species : selectedSpecies) { String speciesid = (String) species; @@ -203,24 +206,24 @@ public class MaxMinGenerator { // status = (double) i / (double) size; if (i%100==0){ - AnalysisLogger.getLogger().debug("status " + status+" species processed "+i); + logger.debug("status " + status+" species processed "+i); insertQuery(specieslist, hspenTable); specieslist = null; specieslist = new ArrayList(); //long t1 = System.currentTimeMillis(); - //AnalysisLogger.getLogger().debug("elapsed Time: " + (t1 - t0)); + //logger.debug("elapsed Time: " + (t1 - t0)); } } if (specieslist.size()>0){ - AnalysisLogger.getLogger().debug("final status " + status+" species processed "+i); + logger.debug("final status " + status+" species processed "+i); insertQuery(specieslist, hspenTable); } long t1 = System.currentTimeMillis(); - AnalysisLogger.getLogger().debug("overall elapsed Time: " + (t1 - t0)); + logger.debug("overall elapsed Time: " + (t1 - t0)); } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/neuralnetworks/Neural_Network.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/neuralnetworks/Neural_Network.java index 54eef00..a25fafe 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/neuralnetworks/Neural_Network.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/neuralnetworks/Neural_Network.java @@ -5,9 +5,13 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.Serializable; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Neural_Network implements Serializable { + + private static Logger logger = LoggerFactory.getLogger(Neural_Network.class); + public Neuron[][] griglia; static final long serialVersionUID = 1; // originale = 1.2 @@ -340,7 +344,7 @@ public class Neural_Network implements Serializable { public void train(double[][] inputvet, double[][] correctoutputvet) { if (griglia[griglia.length - 1].length != correctoutputvet[0].length) - AnalysisLogger.getLogger().debug("Error : the vector of outputs has not a lenght equal to the output of the network"); + logger.debug("Error : the vector of outputs has not a lenght equal to the output of the network"); else { en = 2; int counter = 0; @@ -353,7 +357,7 @@ public class Neural_Network implements Serializable { en += energy(this.propagate(inputvet[i]), correctoutputvet[i]); } - AnalysisLogger.getLogger().debug("Learning Score: " + en); + logger.debug("Learning Score: " + en); counter++; status = (float)counter/(float)maxcycle; @@ -365,9 +369,9 @@ public class Neural_Network implements Serializable { } System.out.println("Final Error: " + en); if (counter >= maxcycle) - AnalysisLogger.getLogger().debug("training incomplete: didn't manage to reduce the error under the thr!"); + logger.debug("training incomplete: didn't manage to reduce the error under the thr!"); else - AnalysisLogger.getLogger().debug("training complete!"); + logger.debug("training complete!"); status = 100; } } @@ -411,7 +415,7 @@ public class Neural_Network implements Serializable { nn = (Neural_Network) ois.readObject(); } catch (Exception ex) { ex.printStackTrace(); - AnalysisLogger.getLogger().debug("Error in reading the object from file " + nomeFile + " ."); + logger.debug("Error in reading the object from file " + nomeFile + " ."); } finally { try { stream.close(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/pca/PrincipalComponentAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/pca/PrincipalComponentAnalysis.java index c4e7af7..9a969ef 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/pca/PrincipalComponentAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/models/cores/pca/PrincipalComponentAnalysis.java @@ -1,15 +1,14 @@ package org.gcube.dataanalysis.ecoengine.models.cores.pca; -import org.gcube.contentmanagement.graphtools.data.BigSamplesTable; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.utils.Operations; import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.example.ExampleSet; import com.rapidminer.operator.IOContainer; import com.rapidminer.operator.IOObject; -import com.rapidminer.operator.ModelApplier; import com.rapidminer.operator.features.transformation.PCA; import com.rapidminer.operator.features.transformation.PCAModel; import com.rapidminer.tools.OperatorService; @@ -17,67 +16,7 @@ import com.rapidminer.tools.OperatorService; public class PrincipalComponentAnalysis { - - public static void main(String[] args) throws Exception{ - - - AlgorithmConfiguration config = new AlgorithmConfiguration(); - config.setConfigPath("./cfg/"); - config.setPersistencePath("./cfg/"); - config.setNumberOfResources(1); - config.setAgent("QUALITY_ANALYSIS"); - - AnalysisLogger.setLogger(config.getConfigPath()+AlgorithmConfiguration.defaultLoggerFile); - config.initRapidMiner(); - - PrincipalComponentAnalysis pca = new PrincipalComponentAnalysis(); - - /*random example - int m= 20; - int n = 10; - - double values[][] = new double[m][n]; - - for (int i=0;i"+model.getEigenvalue(i)); - double[] eigen = model.getEigenvector(i); - for (int j=0;j features = DatabaseFactory.executeSQLQuery(query, connection); int numbOfFeatures = features.size(); @@ -236,12 +237,12 @@ public class FeedForwardNNFile extends ModelAquamapsNN { List maxmin = DatabaseFactory.executeSQLQuery("select max(" + trainingColumn + "), min(" + trainingColumn + ") from " + trainingDataSet, connection); maxfactor = Double.parseDouble("" + ((Object[]) maxmin.get(0))[0]); minfactor = Double.parseDouble("" + ((Object[]) maxmin.get(0))[1]); - AnalysisLogger.getLogger().debug("Calculated max: " + maxfactor + " min: " + minfactor); + logger.debug("Calculated max: " + maxfactor + " min: " + minfactor); // setup Neural Network int numberOfInputNodes = dbcolumnsList.length; int numberOfOutputNodes = 1; - AnalysisLogger.getLogger().debug("Training the ANN with " + numbOfFeatures + " training data and " + numberOfInputNodes + " inputs"); + logger.debug("Training the ANN with " + numbOfFeatures + " training data and " + numberOfInputNodes + " inputs"); if (layersNeurons != null) { int[] innerLayers = Neural_Network.setupInnerLayers(layersNeurons); nn = new Neural_Network(numberOfInputNodes, numberOfOutputNodes, innerLayers, Neural_Network.ACTIVATIONFUNCTION.SIGMOID); @@ -252,10 +253,10 @@ public class FeedForwardNNFile extends ModelAquamapsNN { nn.minfactor = minfactor; nn.setThreshold(learningThr); nn.setCycles(maxiter); - AnalysisLogger.getLogger().debug("network parameters: M: " + maxfactor + ", m: " + minfactor + ", lt: " + learningThr + ", it: " + maxiter); - AnalysisLogger.getLogger().debug("topology: " + nn.griglia.length + "X" + nn.griglia[0].length); + logger.debug("network parameters: M: " + maxfactor + ", m: " + minfactor + ", lt: " + learningThr + ", it: " + maxiter); + logger.debug("topology: " + nn.griglia.length + "X" + nn.griglia[0].length); - AnalysisLogger.getLogger().debug("Features preprocessing"); + logger.debug("Features preprocessing"); double[][] in = new double[numbOfFeatures][]; double[][] out = new double[numbOfFeatures][]; // build NN input @@ -265,20 +266,19 @@ public class FeedForwardNNFile extends ModelAquamapsNN { in[i] = Neural_Network.preprocessObjects(Arrays.copyOfRange((Object[]) features.get(i), 1, feats.length)); out[i] = Neural_Network.preprocessObjects(Arrays.copyOfRange((Object[]) features.get(i), 0, 1)); // apply reduction factor - // AnalysisLogger.getLogger().debug("Output Transformed from "+out[i][0]); + // logger.debug("Output Transformed from "+out[i][0]); out[i][0] = nn.getCorrectValueForOutput(out[i][0]); - // AnalysisLogger.getLogger().debug("To "+out[i][0]); + // logger.debug("To "+out[i][0]); } - AnalysisLogger.getLogger().debug("Features were correctly preprocessed - Training"); + logger.debug("Features were correctly preprocessed - Training"); // train the NN nn.train(in, out); - AnalysisLogger.getLogger().debug("Saving Network"); + logger.debug("Saving Network"); save(fileName, nn); - AnalysisLogger.getLogger().debug("Done"); + logger.debug("Done"); } catch (Exception e) { - e.printStackTrace(); - AnalysisLogger.getLogger().error("ERROR during training"); + logger.error("ERROR during training",e); } status = 100f; diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java index 7faa308..0d5014b 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSimpleSplitGenerator.java @@ -11,7 +11,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.gcube.contentmanagement.graphtools.utils.HttpRequest; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; @@ -21,9 +20,13 @@ import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.interfaces.Generator; import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionGeneric; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LocalSimpleSplitGenerator implements Generator { + private static Logger logger = LoggerFactory.getLogger(LocalSimpleSplitGenerator.class); + protected AlgorithmConfiguration config; protected ExecutorService executorService; protected int numberOfThreadsToUse; @@ -91,13 +94,12 @@ public class LocalSimpleSplitGenerator implements Generator { @Override public void init() { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); stopInterrupt = false; completeDistribution = new ConcurrentHashMap>(); try { initModel(); } catch (Exception e) { - AnalysisLogger.getLogger().error("error",e); + logger.error("error",e); } // probabilityBuffer = new Vector(); probabilityBuffer = new ConcurrentLinkedQueue(); @@ -156,19 +158,19 @@ public class LocalSimpleSplitGenerator implements Generator { // INITIALIZATION long tstart = System.currentTimeMillis(); try { - AnalysisLogger.getLogger().trace("generate->Take features reference"); + logger.trace("generate->Take features reference"); // take the area reference vectors environmentVectors = distributionModel.getGeographicalInfoObjects(); if ((environmentVectors ==null) ||(environmentVectors.size()==0)) throw new Exception("Empty Features Set"); // calculate the number of chunks needed spaceVectorsNumber = environmentVectors.size(); - AnalysisLogger.getLogger().trace("generate->Features to calc: "+spaceVectorsNumber); + logger.trace("generate->Features to calc: "+spaceVectorsNumber); - AnalysisLogger.getLogger().trace("generate->Take groups references"); + logger.trace("generate->Take groups references"); List speciesVectors = distributionModel.getMainInfoObjects(); int speciesVectorNumber = speciesVectors.size(); - AnalysisLogger.getLogger().trace("generate->Number of groups of features: "+speciesVectorNumber); + logger.trace("generate->Number of groups of features: "+speciesVectorNumber); // calculate number of chunks to take into account chunksize = spaceVectorsNumber / numberOfThreadsToUse; @@ -178,7 +180,7 @@ public class LocalSimpleSplitGenerator implements Generator { if ((spaceVectorsNumber % chunksize) != 0) numOfChunks += 1; - AnalysisLogger.getLogger().trace("generate->Calculation Started with " + numOfChunks + " chunks and " + speciesVectorNumber + " groups - chunk size will be "+chunksize); + logger.trace("generate->Calculation Started with " + numOfChunks + " chunks and " + speciesVectorNumber + " groups - chunk size will be "+chunksize); // initialize threads initializeThreads(); @@ -198,7 +200,7 @@ public class LocalSimpleSplitGenerator implements Generator { long computationT0 = System.currentTimeMillis(); // pre process for single species distributionModel.singleStepPreprocess(species, spaceVectorsNumber); - AnalysisLogger.getLogger().trace("-> species " + distributionModel.getMainInfoID(species) + " - n. " + (processedSpeciesCounter + 1)); + logger.trace("-> species " + distributionModel.getMainInfoID(species) + " - n. " + (processedSpeciesCounter + 1)); // CALCULATION CORE for (int k = 0; k < numOfChunks; k++) { // get the starting index @@ -217,7 +219,7 @@ public class LocalSimpleSplitGenerator implements Generator { status = ((float) overallcounter / ((float) (speciesVectorNumber * numOfChunks))) * 100f; if (status == 100) status = 99f; - // AnalysisLogger.getLogger().trace("STATUS->"+status+"%"); + // logger.trace("STATUS->"+status+"%"); // increment global counter index overallcounter++; } @@ -231,7 +233,7 @@ public class LocalSimpleSplitGenerator implements Generator { long computationT1 = System.currentTimeMillis(); // flushBuffer(); - AnalysisLogger.getLogger().trace("generate->Species Computation Finished in " + (computationT1 - computationT0) + " ms"); + logger.trace("generate->Species Computation Finished in " + (computationT1 - computationT0) + " ms"); // perform overall insert // insertCriteria(); // increment the count of processed species @@ -246,18 +248,18 @@ public class LocalSimpleSplitGenerator implements Generator { long computationT2 = System.currentTimeMillis(); // flushInterrupt = true; - AnalysisLogger.getLogger().trace("generate->All Species Computed in " + (computationT2 - tstart) + " ms"); + logger.trace("generate->All Species Computed in " + (computationT2 - tstart) + " ms"); } catch (Exception e) { - AnalysisLogger.getLogger().error("error",e); + logger.error("error",e); throw e; } finally { // REPORT OVERALL ELAPSED TIME - AnalysisLogger.getLogger().trace("generate-> Storing Probability Distribution"); + logger.trace("generate-> Storing Probability Distribution"); try{ distributionModel.storeDistribution(completeDistribution); }catch(Exception ee){ - AnalysisLogger.getLogger().trace("generate-> Error Storing Probability Distribution ",ee); + logger.trace("generate-> Error Storing Probability Distribution ",ee); } try{ distributionModel.postProcess(); @@ -268,7 +270,7 @@ public class LocalSimpleSplitGenerator implements Generator { }catch(Exception eeee){} long tend = System.currentTimeMillis(); long ttotal = tend - tstart; - AnalysisLogger.getLogger().warn("generate->Distribution Generator->Algorithm finished in: " + ((double) ttotal / (double) 60000) + " min\n"); + logger.warn("generate->Distribution Generator->Algorithm finished in: " + ((double) ttotal / (double) 60000) + " min\n"); status = 100f; } } @@ -296,10 +298,10 @@ public class LocalSimpleSplitGenerator implements Generator { } public Integer call() { - AnalysisLogger.getLogger().trace("threadCalculation->" + (threadIndex+1)); + logger.trace("threadCalculation->" + (threadIndex+1)); int max = Math.min(spaceindex + chunksize, spaceVectorsNumber); String speciesID = distributionModel.getMainInfoID(speciesVector); - AnalysisLogger.getLogger().trace("threadCalculation-> calculating elements from "+spaceindex+" to " + max +" for species "+speciesID); + logger.trace("threadCalculation-> calculating elements from "+spaceindex+" to " + max +" for species "+speciesID); Map geoDistrib = completeDistribution.get(speciesID); //if the map is null then generate a new map, otherwise update it if (geoDistrib==null){ @@ -317,7 +319,7 @@ public class LocalSimpleSplitGenerator implements Generator { processedRecordsCounter++; } - AnalysisLogger.getLogger().trace("FINISHED"); + logger.trace("FINISHED"); threadActivity[threadIndex] = false; return 0; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java index ff50961..69c5e37 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/LocalSplitGenerator.java @@ -13,7 +13,6 @@ import java.util.concurrent.Executors; import org.gcube.contentmanagement.graphtools.utils.HttpRequest; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; @@ -25,9 +24,13 @@ import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionTable; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LocalSplitGenerator implements Generator { + private static Logger logger = LoggerFactory.getLogger(LocalSplitGenerator.class); + private AlgorithmConfiguration config; private ExecutorService executorService; private int numberOfThreadsToUse; @@ -105,7 +108,6 @@ public class LocalSplitGenerator implements Generator { @Override public void init() { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); stopInterrupt = false; flushInterrupt = false; forceflush=false; @@ -113,7 +115,7 @@ public class LocalSplitGenerator implements Generator { try { initModel(); } catch (Exception e) { - AnalysisLogger.getLogger().error("error",e); + logger.error("error",e); } // probabilityBuffer = new Vector(); probabilityBuffer = new ConcurrentLinkedQueue(); @@ -127,7 +129,7 @@ public class LocalSplitGenerator implements Generator { probabilityInsertionStatement = probabilityInsertionStatement.replace("%ADDEDINFORMATION%", addedinfo); if (!distributionModel.isSynchronousProbabilityWrite()) { - AnalysisLogger.getLogger().trace("init()->insertion scheduler initialized"); + logger.trace("init()->insertion scheduler initialized"); // inizialize the scheduler for the insertions writerScheduler = new Timer(); writerScheduler.schedule(new DatabaseWriter(), 0, AlgorithmConfiguration.refreshResourcesTime); @@ -174,7 +176,7 @@ public class LocalSplitGenerator implements Generator { dbHibConnection = DatabaseFactory.initDBConnection(defaultDatabaseFile, config); } } catch (Exception e) { - AnalysisLogger.getLogger().warn("error initializing db session",e); + logger.warn("error initializing db session",e); } } @@ -182,17 +184,17 @@ public class LocalSplitGenerator implements Generator { private void createTable() throws Exception { if (config.getParam("CreateTable") != null && config.getParam("CreateTable").equalsIgnoreCase("true")) { try { - AnalysisLogger.getLogger().trace("recreating table: " + "drop table " + config.getParam("DistributionTable")); + logger.trace("recreating table: " + "drop table " + config.getParam("DistributionTable")); DatabaseFactory.executeSQLUpdate("drop table " + config.getParam("DistributionTable"), dbHibConnection); - AnalysisLogger.getLogger().trace("recreating table->OK"); + logger.trace("recreating table->OK"); } catch (Exception e) { - AnalysisLogger.getLogger().trace("recreating table->" + e.getLocalizedMessage()); + logger.trace("recreating table->" + e.getLocalizedMessage()); } // DatabaseFactory.executeUpdateNoTransaction(distributionModel.getDistributionTableStatement(), config.getParam("DatabaseDriver"), config.getParam("DatabaseUserName"), config.getParam("DatabasePassword"), config.getParam("DatabaseURL"), true); DatabaseFactory.executeUpdateNoTransaction(distributionModel.getDistributionTableStatement(), config.getDatabaseDriver(), config.getDatabaseUserName(), config.getDatabasePassword(), config.getDatabaseURL(), true); - AnalysisLogger.getLogger().trace("createTable()->OK!"); + logger.trace("createTable()->OK!"); } } @@ -217,7 +219,7 @@ public class LocalSplitGenerator implements Generator { } catch (Exception e) { } } - AnalysisLogger.getLogger().trace("CLOSING CONNECTIONS"); + logger.trace("CLOSING CONNECTIONS"); try{ dbHibConnection.close(); }catch(Exception eee){} @@ -240,16 +242,16 @@ public class LocalSplitGenerator implements Generator { // INITIALIZATION long tstart = System.currentTimeMillis(); try { - AnalysisLogger.getLogger().trace("generate->Using Local Computation algorithm " + distributionModel.getName()); + logger.trace("generate->Using Local Computation algorithm " + distributionModel.getName()); - AnalysisLogger.getLogger().trace("generate->Check for table creation"); + logger.trace("generate->Check for table creation"); createTable(); - AnalysisLogger.getLogger().trace("generate->Take area reference"); + logger.trace("generate->Take area reference"); // take the area reference vectors environmentVectors = DatabaseFactory.executeSQLQuery(distributionModel.getGeographicalInfoQuery(), dbHibConnection); - AnalysisLogger.getLogger().trace("generate->Take species reference"); + logger.trace("generate->Take species reference"); List speciesVectors = DatabaseFactory.executeSQLQuery(distributionModel.getMainInfoQuery(), dbHibConnection); - AnalysisLogger.getLogger().trace("generate->got all information"); + logger.trace("generate->got all information"); // calculate the number of chunks needed spaceVectorsNumber = environmentVectors.size(); @@ -263,7 +265,7 @@ public class LocalSplitGenerator implements Generator { if ((spaceVectorsNumber % chunksize) != 0) numOfChunks += 1; - AnalysisLogger.getLogger().trace("generate->Calculation Started with " + numOfChunks + " chunks and " + speciesVectorNumber + " species"); + logger.trace("generate->Calculation Started with " + numOfChunks + " chunks and " + speciesVectorNumber + " species"); // initialize threads initializeThreads(); @@ -283,7 +285,7 @@ public class LocalSplitGenerator implements Generator { long computationT0 = System.currentTimeMillis(); // pre process for single species distributionModel.singleStepPreprocess(species, spaceVectorsNumber); - AnalysisLogger.getLogger().trace("-------------------------------------------------> species " + distributionModel.getMainInfoID(species) + " - n. " + (processedSpeciesCounter + 1)); + logger.trace("-------------------------------------------------> species " + distributionModel.getMainInfoID(species) + " - n. " + (processedSpeciesCounter + 1)); // CALCULATION CORE for (int k = 0; k < numOfChunks; k++) { // get the starting index @@ -302,7 +304,7 @@ public class LocalSplitGenerator implements Generator { status = ((float) overallcounter / ((float) (speciesVectorNumber * numOfChunks))) * 100f; if (status == 100) status = 99f; - // AnalysisLogger.getLogger().trace("STATUS->"+status+"%"); + // logger.trace("STATUS->"+status+"%"); // increment global counter index overallcounter++; } @@ -322,7 +324,7 @@ public class LocalSplitGenerator implements Generator { long computationT1 = System.currentTimeMillis(); // flushBuffer(); - AnalysisLogger.getLogger().trace("generate->Species Computation Finished in " + (computationT1 - computationT0) + " ms"); + logger.trace("generate->Species Computation Finished in " + (computationT1 - computationT0) + " ms"); // perform overall insert // insertCriteria(); // increment the count of processed species @@ -338,10 +340,10 @@ public class LocalSplitGenerator implements Generator { long computationT2 = System.currentTimeMillis(); // flushInterrupt = true; - AnalysisLogger.getLogger().trace("generate->All Species Computed in " + (computationT2 - tstart) + " ms"); + logger.trace("generate->All Species Computed in " + (computationT2 - tstart) + " ms"); } catch (Exception e) { - AnalysisLogger.getLogger().error("error",e); + logger.error("error",e); throw e; } finally { try { @@ -353,7 +355,7 @@ public class LocalSplitGenerator implements Generator { } long tend = System.currentTimeMillis(); long ttotal = tend - tstart; - AnalysisLogger.getLogger().warn("generate->Distribution Generator->Algorithm finished in: " + ((double) ttotal / (double) 60000) + " min\n"); + logger.warn("generate->Distribution Generator->Algorithm finished in: " + ((double) ttotal / (double) 60000) + " min\n"); status = 100f; } } @@ -381,7 +383,7 @@ public class LocalSplitGenerator implements Generator { } public Integer call() { - // AnalysisLogger.getLogger().trace("threadCalculation->" + (threadIndex+1)); + // logger.trace("threadCalculation->" + (threadIndex+1)); int max = Math.min(spaceindex + chunksize, spaceVectorsNumber); String speciesID = distributionModel.getMainInfoID(speciesVector); @@ -417,26 +419,26 @@ public class LocalSplitGenerator implements Generator { public void run() { try { if (forceflush){ - AnalysisLogger.getLogger().trace("\t...flushing on db"); + logger.trace("\t...flushing on db"); // flush the objects flushBuffer(); - AnalysisLogger.getLogger().trace("\t...finished flushing on db"); + logger.trace("\t...finished flushing on db"); forceflush=false; } if (stopInterrupt) { - AnalysisLogger.getLogger().trace("\t...finally flushing on db"); + logger.trace("\t...finally flushing on db"); // flush the objects flushBuffer(); - AnalysisLogger.getLogger().trace("\t...finished finally flushing on db"); + logger.trace("\t...finished finally flushing on db"); flushInterrupt = true; this.cancel(); } else if ((probabilityBuffer != null) && (probabilityBuffer.size() > AlgorithmConfiguration.chunkSize)) { - // AnalysisLogger.getLogger().trace("\t...writing on db"); + // logger.trace("\t...writing on db"); writeOnDB(AlgorithmConfiguration.chunkSize); - // AnalysisLogger.getLogger().trace("\t...finished writing on db"); + // logger.trace("\t...finished writing on db"); } } catch (Throwable e) { - AnalysisLogger.getLogger().error("error",e); + logger.error("error",e); flushInterrupt = true; } @@ -468,13 +470,13 @@ public class LocalSplitGenerator implements Generator { String insertionString = String.format(probabilityInsertionStatement, config.getParam("DistributionTable"), sb.toString()); try { - // AnalysisLogger.getLogger().debug("->"+insertionString); + // logger.debug("->"+insertionString); DatabaseFactory.executeSQLUpdate(insertionString, dbHibConnection); } catch (Exception e) { e.printStackTrace(); } - AnalysisLogger.getLogger().trace("writeOnDB()->PROBABILITIES BUFFER REMAINING:" + probabilityBuffer.size()); + logger.trace("writeOnDB()->PROBABILITIES BUFFER REMAINING:" + probabilityBuffer.size()); sb = null; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java index d8fe44c..38bbdeb 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/RainyCloudGenerator.java @@ -1,10 +1,6 @@ package org.gcube.dataanalysis.ecoengine.processing; -import java.util.HashMap; -import java.util.List; - import org.gcube.contentmanagement.graphtools.utils.HttpRequest; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; @@ -13,12 +9,14 @@ import org.gcube.dataanalysis.ecoengine.connectors.RemoteHspecInputObject; import org.gcube.dataanalysis.ecoengine.connectors.RemoteHspecOutputObject; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; -import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; -import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; //deprecated public class RainyCloudGenerator { + private static Logger logger = LoggerFactory.getLogger(RainyCloudGenerator.class); + AlgorithmConfiguration config; private boolean interruptProcessing; RemoteGenerationManager remoteGenerationManager; @@ -46,7 +44,7 @@ public class RainyCloudGenerator { } public void init() { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + interruptProcessing = false; rhio = new RemoteHspecInputObject(); rhio.userName = config.getParam("ServiceUserName"); @@ -145,18 +143,18 @@ public class RainyCloudGenerator { e.printStackTrace(); } - AnalysisLogger.getLogger().trace("REMOTE PROCESSING STARTED"); + logger.trace("REMOTE PROCESSING STARTED"); boolean finish = false; while (!finish && !interruptProcessing) { float status = getStatus(); - // AnalysisLogger.getLogger().trace("Status "+status); + // logger.trace("Status "+status); if (status == 100) finish = true; Thread.sleep(500); } - AnalysisLogger.getLogger().trace("REMOTE PROCESSING ENDED"); + logger.trace("REMOTE PROCESSING ENDED"); } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/DynamicTransducerersFactory.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/DynamicTransducerersFactory.java index 9e42b45..2e72d7f 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/DynamicTransducerersFactory.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/DynamicTransducerersFactory.java @@ -5,15 +5,18 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.interfaces.DynamicTransducer; import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DynamicTransducerersFactory { + private static Logger logger = LoggerFactory.getLogger(DynamicTransducerersFactory.class); + public DynamicTransducerersFactory (){ } @@ -22,7 +25,7 @@ public class DynamicTransducerersFactory { public static Transducerer getTransducerer(AlgorithmConfiguration config) throws Exception { String agent = config.getAgent(); Map subTransducerers = getAllSubTransducerers(config); - AnalysisLogger.getLogger().debug("DynamicTransducerFactory: Getting the following Agent:"+agent+" From the list of N Transducers: "+subTransducerers.size()); + logger.debug("DynamicTransducerFactory: Getting the following Agent:"+agent+" From the list of N Transducers: "+subTransducerers.size()); Transducerer trans = subTransducerers.get(agent); return trans; } @@ -41,9 +44,9 @@ public class DynamicTransducerersFactory { List trans = ProcessorsFactory.getClasses(config.getConfigPath() + AlgorithmConfiguration.dynamicTransducerersFile); /* if (trans!=null && trans.size()>0) - AnalysisLogger.getLogger().debug("Dynamic Transducerers Factory: Found "+trans.size()+" external connectors"); + logger.debug("Dynamic Transducerers Factory: Found "+trans.size()+" external connectors"); else - AnalysisLogger.getLogger().debug("Dynamic Transducerers Factory: No external connectors found!"); + logger.debug("Dynamic Transducerers Factory: No external connectors found!"); */ return trans; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java index 8c003aa..475d5b7 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/GeneratorsFactory.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; @@ -14,6 +13,7 @@ import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm; public class GeneratorsFactory { + public GeneratorsFactory(){ } @@ -56,7 +56,6 @@ public class GeneratorsFactory { List generators = new ArrayList(); try { //initialize the logger - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); //take the algorithm String algorithm = config.getModel(); //take the algorithms list diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java index 7482914..d8a4182 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ModelersFactory.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; @@ -50,8 +49,6 @@ public class ModelersFactory { //modify this class in order to manage generators weight and match algorithm vs generators List modelers = new ArrayList(); try { - //initialize the logger - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); //take the algorithm String algorithm = config.getModel(); //take the algorithms list diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java index 869b853..825d7ac 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/ProcessorsFactory.java @@ -10,7 +10,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Properties; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.interfaces.Clusterer; @@ -22,9 +21,13 @@ import org.gcube.dataanalysis.ecoengine.interfaces.Model; import org.gcube.dataanalysis.ecoengine.interfaces.Modeler; import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ProcessorsFactory { + private static Logger logger = LoggerFactory.getLogger(ProcessorsFactory.class); + public static List getDefaultDatabaseConfiguration(String cfgPath) { String databasecfgfile = cfgPath + AlgorithmConfiguration.defaultConnectionFile; try { @@ -141,8 +144,6 @@ public class ProcessorsFactory { public static ComputationalAgent getProcessor(AlgorithmConfiguration config, String file,String explicitAlgorithm, ClassLoader cl) throws Exception { // modify this class in order to take the right generator algorithm try { - // initialize the logger - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); // take the algorithm String algorithm = explicitAlgorithm; if (explicitAlgorithm==null) @@ -171,7 +172,7 @@ public class ProcessorsFactory { } else if (algclass instanceof Clusterer) { Clusterer m = (Clusterer) algclass; - AnalysisLogger.getLogger().debug("algoritm is null !!!! "+(m==null)); + logger.debug("algoritm is null !!!! "+(m==null)); m.setConfiguration(config); return m; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/TransducerersFactory.java b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/TransducerersFactory.java index 7ab797f..ea688c2 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/TransducerersFactory.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/processing/factories/TransducerersFactory.java @@ -3,14 +3,17 @@ package org.gcube.dataanalysis.ecoengine.processing.factories; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TransducerersFactory { + private static Logger logger = LoggerFactory.getLogger(TransducerersFactory.class); + public TransducerersFactory() { } @@ -39,7 +42,7 @@ public class TransducerersFactory { else { config.setAgent(algorithmName); inputs = DynamicTransducerersFactory.getTransducerer(config).getInputParameters(); - AnalysisLogger.getLogger().debug("Dynamic INPUTS:"+inputs); + logger.debug("Dynamic INPUTS:"+inputs); return inputs; } } @@ -52,7 +55,7 @@ public class TransducerersFactory { else { config.setAgent(algorithmName); output = DynamicTransducerersFactory.getTransducerer(config).getOutput(); - AnalysisLogger.getLogger().debug("Dynamic Output:"+output); + logger.debug("Dynamic Output:"+output); return output; } } @@ -70,7 +73,7 @@ public class TransducerersFactory { else{ config.setAgent(algorithmName); input = DynamicTransducerersFactory.getTransducerer(config).getDescription(); - AnalysisLogger.getLogger().debug("Dynamic DESCRIPTION:"+input); + logger.debug("Dynamic DESCRIPTION:"+input); return input; } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java index 46c7172..d4ffc64 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java @@ -4,10 +4,13 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PeriodicityDetector { + private static Logger logger = LoggerFactory.getLogger(PeriodicityDetector.class); + /* * static int defaultSamplingRate = 1000;// Hz static float defaultSignalLengthTimeinSec = 5;// s static float defaultHiddenFrequency = 100f;// Hz static float defaultMinPossibleFreq = 0; // Hz static float defaultMaxPossibleFreq = 200; // Hz static float defaultSNratio = 2; static float defaultFreqError = 1f; */ @@ -71,9 +74,9 @@ public class PeriodicityDetector { public void demo() throws Exception { double[] signal = produceNoisySignal(defaultSignalLengthTimeinSec, defaultSamplingRate, defaultHiddenFrequency, defaultSNratio); - AnalysisLogger.getLogger().debug("Signal samples: " + signal.length); + logger.debug("Signal samples: " + signal.length); double F = detectFrequency(signal, defaultSamplingRate, defaultMinPossibleFreq, defaultMaxPossibleFreq, defaultFreqError, -1, true); - AnalysisLogger.getLogger().debug("Detected F:" + F + " indecision [" + lowermeanF + " , " + uppermeanF + "]"); + logger.debug("Detected F:" + F + " indecision [" + lowermeanF + " , " + uppermeanF + "]"); } public static void main(String[] args) throws Exception { @@ -115,7 +118,7 @@ public class PeriodicityDetector { if (pow <= 1) pow = Math.round(Math.log((float) signal.length / (float) ("" + signal.length).length()) / Math.log(2)); - AnalysisLogger.getLogger().debug("Suggested pow for window length=" + pow); + logger.debug("Suggested pow for window length=" + pow); } // adjust FFT Samples to be even @@ -130,8 +133,8 @@ public class PeriodicityDetector { wLength = (int) Math.pow(2, pow); - AnalysisLogger.getLogger().debug("Suggested windows length (samples)=" + wLength); - AnalysisLogger.getLogger().debug("Suggested windows length (s)=" + ((float) wLength / (float) samplingRate) + " s"); + logger.debug("Suggested windows length (samples)=" + wLength); + logger.debug("Suggested windows length (s)=" + ((float) wLength / (float) samplingRate) + " s"); int windowAnalysisSamples = (int) Math.pow(2, 14);// (int) windowAnalysisSamples = wLength; int windowShiftSamples = (int) Math.round((float) windowAnalysisSamples / 2f); @@ -139,9 +142,9 @@ public class PeriodicityDetector { float error = ((float) samplingRate / (float) windowAnalysisSamples); - AnalysisLogger.getLogger().debug("Error in the Measure will be: " + error + " Hz"); - AnalysisLogger.getLogger().debug("A priori Min Freq: " + minPossibleFreq + " s"); - AnalysisLogger.getLogger().debug("A priori Max Freq: " + maxPossibleFreq + " s"); + logger.debug("Error in the Measure will be: " + error + " Hz"); + logger.debug("A priori Min Freq: " + minPossibleFreq + " s"); + logger.debug("A priori Max Freq: " + maxPossibleFreq + " s"); if (maxPossibleFreq >= samplingRate) maxPossibleFreq = (float) (samplingRate / 2f) - (0.1f * samplingRate / 2f); @@ -173,7 +176,7 @@ public class PeriodicityDetector { double[] powers = signalMaximumAnalyzer.averagepower; currentspectrum = spectrum; // display the maximum freqs - AnalysisLogger.getLogger().debug("Number of frequency peaks " + maxfrequencies.length); + logger.debug("Number of frequency peaks " + maxfrequencies.length); // take the longest stable sequence of frequencies SignalConverter signalconverter = new SignalConverter(); maxfrequencies = signalconverter.takeLongestStableTract(maxfrequencies, 0.01); @@ -202,13 +205,13 @@ public class PeriodicityDetector { double meanF = MathFunctions.mean(maxfrequencies); // we consider a complete cycle double possibleperiod = 2d / meanF; - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Frequency " + meanF); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Periodicity " + possibleperiod); + logger.debug("TimeSeriesAnalysis->Frequency " + meanF); + logger.debug("TimeSeriesAnalysis->Periodicity " + possibleperiod); double maxperiod = Math.min(signal.length, currentWindowAnalysisSamples); if ((meanF <= minPossibleFreq) || (meanF >= maxPossibleFreq) || (possibleperiod == 0) || (possibleperiod > (maxperiod))) { - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Invalid periodicity " + (meanF <= minPossibleFreq) + " , " + (meanF >= maxPossibleFreq) + " , " + (possibleperiod == 0) + " , " + (possibleperiod > (maxperiod))); + logger.debug("TimeSeriesAnalysis->Invalid periodicity " + (meanF <= minPossibleFreq) + " , " + (meanF >= maxPossibleFreq) + " , " + (possibleperiod == 0) + " , " + (possibleperiod > (maxperiod))); meanF = 0; this.meanF = 0; @@ -225,7 +228,7 @@ public class PeriodicityDetector { this.endPeriodSampleIndex = 0; } else { - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->periodicity is valid " + possibleperiod); + logger.debug("TimeSeriesAnalysis->periodicity is valid " + possibleperiod); this.meanF = meanF; this.lowermeanF = Math.max(meanF - error, minPossibleFreq); this.uppermeanF = Math.min(meanF + error, maxFrequency); @@ -247,7 +250,7 @@ public class PeriodicityDetector { if (pow <= 1) pow = Math.round(Math.log((float) signal.length / (float) ("" + signal.length).length()) / Math.log(2)); - AnalysisLogger.getLogger().debug("Suggested pow for window length=" + pow); + logger.debug("Suggested pow for window length=" + pow); } // adjust FFT Samples to be even @@ -262,8 +265,8 @@ public class PeriodicityDetector { wLength = (int) Math.pow(2, pow); - AnalysisLogger.getLogger().debug("Suggested windows length (samples)=" + wLength); - AnalysisLogger.getLogger().debug("Suggested windows length (s)=" + ((float) wLength / (float) samplingRate) + " s"); + logger.debug("Suggested windows length (samples)=" + wLength); + logger.debug("Suggested windows length (s)=" + ((float) wLength / (float) samplingRate) + " s"); int windowAnalysisSamples = (int) Math.pow(2, 14);// (int) windowAnalysisSamples = wLength; int windowShiftSamples = (int) Math.round((float) windowAnalysisSamples / 2f); @@ -271,9 +274,9 @@ public class PeriodicityDetector { float error = ((float) samplingRate / (float) windowAnalysisSamples); - AnalysisLogger.getLogger().debug("Error in the Measure will be: " + error + " Hz"); - AnalysisLogger.getLogger().debug("A priori Min Freq: " + minPossibleFreq + " s"); - AnalysisLogger.getLogger().debug("A priori Max Freq: " + maxPossibleFreq + " s"); + logger.debug("Error in the Measure will be: " + error + " Hz"); + logger.debug("A priori Min Freq: " + minPossibleFreq + " s"); + logger.debug("A priori Max Freq: " + maxPossibleFreq + " s"); if (maxPossibleFreq >= samplingRate) maxPossibleFreq = (float) (samplingRate / 2f) - (0.1f * samplingRate / 2f); @@ -296,7 +299,7 @@ public class PeriodicityDetector { adjustParameters(signal, samplingRate, minPossibleFreq, maxPossibleFreq, wantedFreqError, FFTnsamples); //evaluate the minimum frequency resolution double frequencyRes = ((double)samplingRate/2d)/ (double)currentWindowAnalysisSamples; - AnalysisLogger.getLogger().debug("Frequency Resolution: "+frequencyRes); + logger.debug("Frequency Resolution: "+frequencyRes); // trace spectrum double[][] spectrum = SignalConverter.spectrogram("spectrogram", signal, samplingRate, currentWindowShiftSamples, currentWindowAnalysisSamples, false); if (display) @@ -351,10 +354,10 @@ public class PeriodicityDetector { && (period < maxperiod*0.55f) && (!getPowerSpectrumStregthInterpretation(power).equalsIgnoreCase("None"))) { - AnalysisLogger.getLogger().debug("DISCREPANCY WITH RESPECT TO THE PREVIOUS FREQ:"+(peakFreq-previousFreq)); - AnalysisLogger.getLogger().debug("RATIO WITH RESPECT TO THE PREVIOUS FREQ:"+((peakFreq-previousFreq)/error)); + logger.debug("DISCREPANCY WITH RESPECT TO THE PREVIOUS FREQ:"+(peakFreq-previousFreq)); + logger.debug("RATIO WITH RESPECT TO THE PREVIOUS FREQ:"+((peakFreq-previousFreq)/error)); if (counter == 0) { - AnalysisLogger.getLogger().debug("Section "+(i+1)); + logger.debug("Section "+(i+1)); peaks.put("*StartTime_In_Spectrogram"+prefix, "" + startTime); peaks.put("*EndTime_In_Spectrogram" + prefix, "" + endTime); } @@ -377,16 +380,16 @@ public class PeriodicityDetector { int minFidx = SignalConverter.frequencyIndex(minPossibleFreq, currentWindowAnalysisSamples, samplingRate); double spectrogramidx = SignalConverter.spectrumFreq2Idx(peakFreq.floatValue(), samplingRate, currentWindowAnalysisSamples)-minFidx; - AnalysisLogger.getLogger().debug("SpectorgramIdx_"+(counter+1)+":" + spectrogramidx); - AnalysisLogger.getLogger().debug("Strength_of_Periodicity_"+(counter+1)+":" + signalMaximumAnalyzer.currentSpikesPowerSpectra[i].get(freqCounter)); - AnalysisLogger.getLogger().debug("Strength_of_Periodicity_Interpretation"+(counter+1)+":" + getPowerSpectrumStregthInterpretation(signalMaximumAnalyzer.currentSpikesPowerSpectra[i].get(freqCounter))); - AnalysisLogger.getLogger().debug("Frequency_"+(counter+1)+":" + peakFreq); - AnalysisLogger.getLogger().debug("UpperFrequencyConfidence_"+(counter+1)+":" + uppermeanF); - AnalysisLogger.getLogger().debug("LowerFrequencyConfidence_"+(counter+1)+":" + lowermeanF); - AnalysisLogger.getLogger().debug("Period"+":" + period); - AnalysisLogger.getLogger().debug("UpperFrequencyPeriod_"+(counter+1)+":" + (1d / lowermeanF)); - AnalysisLogger.getLogger().debug("LowerFrequencyPeriod_"+(counter+1)+":"+ (1d / uppermeanF)); - AnalysisLogger.getLogger().debug(""); + logger.debug("SpectorgramIdx_"+(counter+1)+":" + spectrogramidx); + logger.debug("Strength_of_Periodicity_"+(counter+1)+":" + signalMaximumAnalyzer.currentSpikesPowerSpectra[i].get(freqCounter)); + logger.debug("Strength_of_Periodicity_Interpretation"+(counter+1)+":" + getPowerSpectrumStregthInterpretation(signalMaximumAnalyzer.currentSpikesPowerSpectra[i].get(freqCounter))); + logger.debug("Frequency_"+(counter+1)+":" + peakFreq); + logger.debug("UpperFrequencyConfidence_"+(counter+1)+":" + uppermeanF); + logger.debug("LowerFrequencyConfidence_"+(counter+1)+":" + lowermeanF); + logger.debug("Period"+":" + period); + logger.debug("UpperFrequencyPeriod_"+(counter+1)+":" + (1d / lowermeanF)); + logger.debug("LowerFrequencyPeriod_"+(counter+1)+":"+ (1d / uppermeanF)); + logger.debug(""); counter++; previousFreq=peakFreq; previousPeriod=period; diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java index 332082a..22e5bee 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java @@ -7,20 +7,22 @@ import java.util.Arrays; import javax.swing.JPanel; -import marytts.signalproc.display.SpectrogramCustom; -import marytts.signalproc.window.Window; - import org.gcube.contentmanagement.graphtools.data.BigSamplesTable; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis; import org.gcube.dataanalysis.ecoengine.transducers.TimeSeriesAnalysis; import org.gcube.dataanalysis.ecoengine.utils.Operations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.rapidminer.example.Attribute; import com.rapidminer.example.Example; import com.rapidminer.example.ExampleSet; import com.rapidminer.example.table.MemoryExampleTable; +import marytts.signalproc.display.SpectrogramCustom; +import marytts.signalproc.window.Window; + /** * includes tools for basic signal transformations: delta + double delta center frequency cepstral coefficients calculation spectrum frequency cut transformation to and from Rapid Miner Example Set filterbanks fequency to mel frequency to index in fft sinusoid signal generation inverse mel log10 mel filterbanks sample to time and time to sample signal timeline generation index to time in spectrogram spectrogram calculation and display time to index in spectrogram * @@ -29,6 +31,8 @@ import com.rapidminer.example.table.MemoryExampleTable; */ public class SignalConverter { + private static Logger logger = LoggerFactory.getLogger(DataAnalysis.class); + public static double[][] addDeltaDouble(double[][] features) throws Exception { int vectorL = features[0].length; double[][] delta = new double[features.length][features[0].length * 3]; @@ -120,17 +124,17 @@ public class SignalConverter { public static int[] fftBinIndices(double samplingRate, int frameSize, int numMelFilters, int numFequencies, float lowerFilterFreq, float upperFilterFreq) { int cbin[] = new int[numFequencies + 2]; - AnalysisLogger.getLogger().debug("New Filter banks: " + numFequencies); + logger.debug("New Filter banks: " + numFequencies); cbin[0] = (int) Math.round(lowerFilterFreq / samplingRate * frameSize); cbin[cbin.length - 1] = frequencyIndex(upperFilterFreq, frameSize, (float) samplingRate); - AnalysisLogger.getLogger().debug("F0: " + lowerFilterFreq); + logger.debug("F0: " + lowerFilterFreq); for (int i = 1; i <= numFequencies; i++) { double fc = centerFreq(i, samplingRate, lowerFilterFreq, numMelFilters); - AnalysisLogger.getLogger().debug("F" + (i) + ": " + fc); + logger.debug("F" + (i) + ": " + fc); cbin[i] = (int) Math.round(fc / samplingRate * frameSize); } - AnalysisLogger.getLogger().debug("F" + (cbin.length - 1) + ": " + upperFilterFreq); + logger.debug("F" + (cbin.length - 1) + ": " + upperFilterFreq); return cbin; } @@ -209,7 +213,7 @@ public class SignalConverter { int bestIndex = 1; for (int i = 1; i <= numMelFilters; i++) { double fc = centerFreq(i, samplingRate, lowerFilterFreq, numMelFilters); - AnalysisLogger.getLogger().debug("fc " + fc); + logger.debug("fc " + fc); if (fc > maxFilterFreq) { bestIndex = i; break; @@ -230,7 +234,7 @@ public class SignalConverter { for (int i = 0; i < signalLength; i++) { time[i] = (double) i / (double) samplingRate; } - AnalysisLogger.getLogger().debug("time " + time[signalLength - 1] * samplingRate + " vs " + signalLength); + logger.debug("time " + time[signalLength - 1] * samplingRate + " vs " + signalLength); return time; } @@ -243,7 +247,7 @@ public class SignalConverter { for (int k = 0; k < signal.length; k++) { samples.addSampleRow("" + k, signal[k]); } - AnalysisLogger.getLogger().debug("Example Set Created"); + logger.debug("Example Set Created"); return samples.generateExampleSet(); } @@ -306,12 +310,12 @@ public class SignalConverter { for (int k = 1; k < slice.length; k++) { double ele = slice[k]; if (!overfirstmin && (slice[k]>slice[k-1])){ - AnalysisLogger.getLogger().debug("First minimum in spectrum is at idx "+k); + logger.debug("First minimum in spectrum is at idx "+k); overfirstmin=true; } if (overfirstmin) { if (ele > (max + (Math.abs(max) * tolerance))) { -// AnalysisLogger.getLogger().debug(">max up:"+ele +">" +(max + (Math.abs(max) * tolerance))+" at idx "+k); +// logger.debug(">max up:"+ele +">" +(max + (Math.abs(max) * tolerance))+" at idx "+k); max = ele; bestidx = k; } @@ -328,7 +332,7 @@ public class SignalConverter { // System.out.println("min f idx: "+minFidx); maxs[j] = spectrumIdx2Frequency(minFidx + bestidx, samplingRate, windowSamples); double mean = org.gcube.contentmanagement.graphtools.utils.MathFunctions.mean(slice); - AnalysisLogger.getLogger().debug("max freq in spec: " + maxs[j]+" index "+minFidx + bestidx); + logger.debug("max freq in spec: " + maxs[j]+" index "+minFidx + bestidx); if (min == Double.MAX_VALUE) { min = max; } @@ -343,7 +347,7 @@ public class SignalConverter { averagepower[j] = Math.abs((max - mean) / max); } - AnalysisLogger.getLogger().debug("max power : " + max + " min power: " + min + " mean " + mean + " power " + averagepower[j]); + logger.debug("max power : " + max + " min power: " + min + " mean " + mean + " power " + averagepower[j]); j++; } @@ -382,7 +386,7 @@ public class SignalConverter { for (int i=0;i signals, int windowShiftSamples, int frameLength, int samplingRate) throws Exception { int signalLenght = signals.get(0).length; - AnalysisLogger.getLogger().debug("TRIALS LENGHT " + signalLenght); + logger.debug("TRIALS LENGHT " + signalLenght); List spectrograms = new ArrayList(); - AnalysisLogger.getLogger().debug("Getting Spectra"); + logger.debug("Getting Spectra"); int j = 0; // get all spectrograms for (double[] signal : signals) { double[][] spectro = SignalConverter.spectrogram("Spectrogram", signal, samplingRate, windowShiftSamples, frameLength, false); - AnalysisLogger.getLogger().debug("Signal Number " + (j + 1) + " spectrum lenght " + ((spectro.length * windowShiftSamples) / samplingRate)); + logger.debug("Signal Number " + (j + 1) + " spectrum lenght " + ((spectro.length * windowShiftSamples) / samplingRate)); spectrograms.add(spectro); j++; } - AnalysisLogger.getLogger().debug("Summing Spectra"); + logger.debug("Summing Spectra"); // sum all spectrograms double[][] sumSpectro = SignalProcessing.sumSpectra(spectrograms); spectrograms = null; @@ -120,7 +123,7 @@ public class SignalProcessing { public static Image renderSignalWithGenericTime(double[] signal, float t0, float timeshift, String name) { if (signal.length > 20000) { - AnalysisLogger.getLogger().debug("Too many points to display: " + signal.length); + logger.debug("Too many points to display: " + signal.length); return null; } org.jfree.data.xy.XYSeries xyseries = new org.jfree.data.xy.XYSeries(name); @@ -137,7 +140,7 @@ public class SignalProcessing { public static Image renderSignalWithGenericTime(double[] signal, double[] timeline, String name) { if (signal.length > 20000) { - AnalysisLogger.getLogger().debug("Too many points to display: " + signal.length); + logger.debug("Too many points to display: " + signal.length); return null; } @@ -156,7 +159,7 @@ public class SignalProcessing { public static Image renderSignalWithTime(double[] signal, Date[] dates, String name, String format) { org.jfree.data.time.TimeSeries series = new org.jfree.data.time.TimeSeries(name); if (signal.length > 20000) { - AnalysisLogger.getLogger().debug("Too many points to display: " + signal.length); + logger.debug("Too many points to display: " + signal.length); return null; } for (int i = 0; i < signal.length; i++) { @@ -164,7 +167,7 @@ public class SignalProcessing { FixedMillisecond ms = new FixedMillisecond(dates[i]); series.add(ms, signal[i]); } catch (Exception e) { - AnalysisLogger.getLogger().debug("Skipping value yet present: " + dates[i]); + logger.debug("Skipping value yet present: " + dates[i]); } } TimeSeriesCollection dataset = new TimeSeriesCollection(); @@ -183,7 +186,7 @@ public class SignalProcessing { org.jfree.data.time.TimeSeries series = new org.jfree.data.time.TimeSeries(name); if (signal.length > 20000) { - AnalysisLogger.getLogger().debug("Too many points to display: " + signal.length); + logger.debug("Too many points to display: " + signal.length); return null; } int offset = 0; @@ -195,7 +198,7 @@ public class SignalProcessing { FixedMillisecond ms = new FixedMillisecond(dates[i]); series.add(ms, signal[i-offset]); } catch (Exception e) { - AnalysisLogger.getLogger().debug("Skipping value yet present: " + dates[i]); + logger.debug("Skipping value yet present: " + dates[i]); } } dataset.addSeries(series); @@ -216,8 +219,8 @@ public class SignalProcessing { public static Image renderSignalSpectrogram2(double[][] spectrogram) { SpectrumPlot2 spectrumPlot = new SpectrumPlot2(spectrogram); - AnalysisLogger.getLogger().debug("Spectrum W:" + spectrumPlot.width); - AnalysisLogger.getLogger().debug("Spectrum H:" + spectrumPlot.height); + logger.debug("Spectrum W:" + spectrumPlot.width); + logger.debug("Spectrum H:" + spectrumPlot.height); // spectrumPlot.hzoomSet(2f); spectrumPlot.hzoomSet(640f / (float) spectrumPlot.width); spectrumPlot.vzoomSet(480f / (float) spectrumPlot.height); @@ -232,7 +235,7 @@ public class SignalProcessing { public static void displaySignalWithTime(double[] signal, Date[] dates, String name, String format) { org.jfree.data.time.TimeSeries series = new org.jfree.data.time.TimeSeries(name); if (signal.length > 20000) { - AnalysisLogger.getLogger().debug("Too many points to display: " + signal.length); + logger.debug("Too many points to display: " + signal.length); return; } for (int i = 0; i < signal.length; i++) { @@ -240,7 +243,7 @@ public class SignalProcessing { FixedMillisecond ms = new FixedMillisecond(dates[i]); series.add(ms, signal[i]); } catch (Exception e) { - AnalysisLogger.getLogger().debug("Skipping value yet present: " + dates[i]); + logger.debug("Skipping value yet present: " + dates[i]); } } TimeSeriesCollection dataset = new TimeSeriesCollection(); @@ -260,7 +263,7 @@ public class SignalProcessing { org.jfree.data.time.TimeSeries series = new org.jfree.data.time.TimeSeries(name); if (signal.length > 20000) { - AnalysisLogger.getLogger().debug("Too many points to display: " + signal.length); + logger.debug("Too many points to display: " + signal.length); return ; } int offset = 0; @@ -272,7 +275,7 @@ public class SignalProcessing { FixedMillisecond ms = new FixedMillisecond(dates[i]); series.add(ms, signal[i-offset]); } catch (Exception e) { - AnalysisLogger.getLogger().debug("Skipping value yet present: " + dates[i]); + logger.debug("Skipping value yet present: " + dates[i]); } } dataset.addSeries(series); @@ -294,13 +297,13 @@ public class SignalProcessing { } ExampleSet es = samples.generateExampleSet(); - AnalysisLogger.getLogger().debug("Example Set Created"); + logger.debug("Example Set Created"); AbsoluteSampling sampler = (AbsoluteSampling) OperatorService.createOperator("AbsoluteSampling"); sampler.setParameter("sample_size", "" + numElements); sampler.setParameter("local_random_seed", "-1"); es = sampler.apply(es); - AnalysisLogger.getLogger().debug("Finished"); + logger.debug("Finished"); SignalConverter.exampleSet2Signal(rebuiltSignal, es); @@ -315,17 +318,17 @@ public class SignalProcessing { List sumspectrograms = new ArrayList(); List spectrograms = new ArrayList(); - AnalysisLogger.getLogger().debug("Getting Spectra"); + logger.debug("Getting Spectra"); int j = 0; // get all spectrograms for (double[] signal : signals) { - AnalysisLogger.getLogger().debug("Signal Number " + (j + 1)); + logger.debug("Signal Number " + (j + 1)); double[][] spectro = SignalConverter.spectrogram("Spectrogram", signal, samplingRate, windowShiftSamples, frameLength, false); spectrograms.add(spectro); j++; } - AnalysisLogger.getLogger().debug("Summing Spectra"); + logger.debug("Summing Spectra"); // sum all spectrograms double[][] sumSpectro = SignalProcessing.sumSpectra(spectrograms); spectrograms = null; @@ -348,7 +351,7 @@ public class SignalProcessing { sampler.setParameter("attribute_name", "att0"); sampler.setParameter("replacement", "3"); es = sampler.apply(es); - AnalysisLogger.getLogger().debug("Finished"); + logger.debug("Finished"); double[] rebuiltSignal = new double[signal.length]; SignalConverter.exampleSet2Signal(rebuiltSignal, es, 0d); @@ -360,7 +363,7 @@ public class SignalProcessing { double t0 = timeseconds[0]; double t1 = timeseconds[timeseconds.length - 1]; int signalength = Math.abs((int) ((t1 - t0) * samplingRate) + 1); - AnalysisLogger.getLogger().debug("SignalProcessing->Old Time Series had: " + values.length + " samples. New Time Series will have: " + signalength + " samples"); + logger.debug("SignalProcessing->Old Time Series had: " + values.length + " samples. New Time Series will have: " + signalength + " samples"); if (values.length == signalength) return values; @@ -381,13 +384,13 @@ public class SignalProcessing { } ExampleSet es = samples.generateExampleSet(); - AnalysisLogger.getLogger().debug("Example Set Created"); + logger.debug("Example Set Created"); SeriesMissingValueReplenishment sampler = (SeriesMissingValueReplenishment) OperatorService.createOperator("SeriesMissingValueReplenishment"); sampler.setParameter("attribute_name", "att0"); sampler.setParameter("replacement", "3"); es = sampler.apply(es); - AnalysisLogger.getLogger().debug("Finished"); + logger.debug("Finished"); SignalConverter.exampleSet2Signal(rebuiltSignal, es); @@ -452,15 +455,15 @@ public class SignalProcessing { float centralTime = (maxTime / (2f * numOfCentralSeconds)); - AnalysisLogger.getLogger().debug("Max Time in the Spectrum " + maxTime + " Central time " + centralTime); + logger.debug("Max Time in the Spectrum " + maxTime + " Central time " + centralTime); int startIndex = (int) (centralTime / windowShiftTime); int endIndex = (int) ((centralTime + numOfCentralSeconds) / windowShiftTime); - AnalysisLogger.getLogger().debug("Recalculated lenght " + maxTime + " sec"); - AnalysisLogger.getLogger().debug("Lenght " + spectrum.length); + logger.debug("Recalculated lenght " + maxTime + " sec"); + logger.debug("Lenght " + spectrum.length); - AnalysisLogger.getLogger().debug("Start " + startIndex + " End " + endIndex + " max " + spectrum.length + " Cut lenght " + (endIndex - startIndex + 1) * windowShiftTime); + logger.debug("Start " + startIndex + " End " + endIndex + " max " + spectrum.length + " Cut lenght " + (endIndex - startIndex + 1) * windowShiftTime); double[][] cutSpectrum = new double[endIndex - startIndex + 1][spectrum[0].length]; diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/TimeSeries.java b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/TimeSeries.java index 49b410b..d748786 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/TimeSeries.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/TimeSeries.java @@ -7,13 +7,16 @@ import java.util.List; import java.util.Locale; import org.gcube.contentmanagement.graphtools.utils.DateGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.utils.Operations; import org.gcube.dataanalysis.ecoengine.utils.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TimeSeries { + private static Logger logger = LoggerFactory.getLogger(TimeSeries.class); + private double[] values; private Date[] times; private Date[] unsortedtimes; @@ -159,19 +162,19 @@ public class TimeSeries { if (counter == 0) { timepattern = DateGuesser.getPattern(timel); ts.setTimepattern(timepattern); - AnalysisLogger.getLogger().debug("Time pattern: " + timepattern); + logger.debug("Time pattern: " + timepattern); sdf = new SimpleDateFormat(timepattern, Locale.ENGLISH); } try{ time = (Date) sdf.parse(timel); }catch(Exception e){ - AnalysisLogger.getLogger().debug("Error in parsing...adjusting "+timel); + logger.debug("Error in parsing...adjusting "+timel); time = DateGuesser.convertDate(timel).getTime(); - AnalysisLogger.getLogger().debug("Error in parsing...adjusting "+timel+" in "+time); + logger.debug("Error in parsing...adjusting "+timel+" in "+time); } if (counter == 0) { - AnalysisLogger.getLogger().debug("Date detection: input " + timel + " output " + time); + logger.debug("Date detection: input " + timel + " output " + time); } ts.addElement(quantity, time, timel, counter); @@ -192,16 +195,16 @@ public class TimeSeries { samplingrate = 1d / (double) minimumtimegap; } - AnalysisLogger.getLogger().debug("TimeSeries->Samplig rate: " + samplingrate + " minimum gap in time: " + minimumtimegap); + logger.debug("TimeSeries->Samplig rate: " + samplingrate + " minimum gap in time: " + minimumtimegap); if (samplingrate == 0) return; double[] timeline = getMillisecondsTimeline(); - AnalysisLogger.getLogger().debug("TimeSeries->filling gaps"); + logger.debug("TimeSeries->filling gaps"); double[] newvalues = SignalProcessing.fillTimeSeries(values, timeline, samplingrate, config); if (newvalues.length != values.length) { - AnalysisLogger.getLogger().debug("TimeSeries->filling also time values"); + logger.debug("TimeSeries->filling also time values"); Date[] newtimeline = SignalProcessing.fillTimeLine(timeline, samplingrate, config); values = null; @@ -213,7 +216,7 @@ public class TimeSeries { timeLabels = new String[times.length]; } - AnalysisLogger.getLogger().debug("TimeSeries->Returning values"); + logger.debug("TimeSeries->Returning values"); timeLabels = new String[times.length]; SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.ROOT); for (int i = 0; i < times.length; i++) { diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SSAWorkflow.java b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SSAWorkflow.java index c370e3d..ea3851b 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SSAWorkflow.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SSAWorkflow.java @@ -3,11 +3,13 @@ package org.gcube.dataanalysis.ecoengine.signals.ssa; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; -import org.gcube.dataanalysis.ecoengine.signals.SignalProcessing; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SSAWorkflow { + private static Logger logger = LoggerFactory.getLogger(SSAWorkflow.class); + public static SSADataset applyCompleteWorkflow(List timeseries, int analysisWindowLength, float eigenValuesPercentageThreshold, int nPointsToForecast, boolean reportReconstructedSignal){ SSADataset data = new SSADataset(); @@ -34,10 +36,10 @@ public class SSAWorkflow { //build groups of indices List groupsModel = new ArrayList(); List groups = new ArrayList(); - AnalysisLogger.getLogger().debug("Listing All the Eigenvalues"); + logger.debug("Listing All the Eigenvalues"); for (int i = 0; i < data.getPercentList().size(); i++) { double currentperc = data.getPercentList().get(i); - AnalysisLogger.getLogger().debug("Eigenvalue: Number: "+i+" Percentage: "+currentperc); + logger.debug("Eigenvalue: Number: "+i+" Percentage: "+currentperc); if (currentperc>eigenValuesPercentageThreshold) groups.add(new SSAUnselectList(i, currentperc)); } @@ -61,7 +63,7 @@ public class SSAWorkflow { // SignalProcessing.displaySignalWithGenericTime(signal, 0, 1, "signal"); // SignalProcessing.displaySignalWithGenericTime(rsignal, 0, 1, "reconstructed signal"); - AnalysisLogger.getLogger().debug("SSA workflow DONE"); + logger.debug("SSA workflow DONE"); return data; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SingularSpectrumAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SingularSpectrumAnalysis.java index a5b3c45..71ed4dc 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SingularSpectrumAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/ssa/SingularSpectrumAnalysis.java @@ -1,17 +1,20 @@ package org.gcube.dataanalysis.ecoengine.signals.ssa; -import Jama.EigenvalueDecomposition; -import Jama.Matrix; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import javax.swing.DefaultListModel; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import Jama.EigenvalueDecomposition; +import Jama.Matrix; public class SingularSpectrumAnalysis { + private static Logger logger = LoggerFactory.getLogger(SingularSpectrumAnalysis.class); + /** * translation of the original time series into a sequence of multidimensional * vectors @@ -240,7 +243,7 @@ public class SingularSpectrumAnalysis { * @return the resulting matrix */ private static double[][] transpositionMatrix(double matrix[][]) { - AnalysisLogger.getLogger().debug("SSA->Building a matrix with dimensions: "+matrix[0].length+" X "+matrix.length); + logger.debug("SSA->Building a matrix with dimensions: "+matrix[0].length+" X "+matrix.length); double transpMatrix[][] = new double[matrix[0].length][matrix.length]; for (int i = 0; i < matrix.length; i++) { for (int j = 0; j < matrix[i].length; j++) { @@ -350,9 +353,9 @@ public class SingularSpectrumAnalysis { List eigenvectors = data.getEigenVectors().subList(0, bestEigenVectors); int L = data.getL(); int lastcoordinate = L-1; - AnalysisLogger.getLogger().debug("SSA: value for L: "+L); + logger.debug("SSA: value for L: "+L); int nEigenVectors = eigenvectors.size(); - AnalysisLogger.getLogger().debug("Number of Selected Eigenvectors For Reconstruction: "+nEigenVectors); + logger.debug("Number of Selected Eigenvectors For Reconstruction: "+nEigenVectors); double[] p = new double[nEigenVectors]; for (int i = 0;i0.1) - AnalysisLogger.getLogger().debug(" Probability " + probability); + logger.debug(" Probability " + probability); */ // System.exit(0); return probability; @@ -120,7 +123,7 @@ public class AquamapsNN extends AquamapsNative{ nn = (Neural_Network) ois.readObject(); } catch (Exception ex) { ex.printStackTrace(); - AnalysisLogger.getLogger().debug("Error in reading the object from file " + nomeFile + " ."); + logger.debug("Error in reading the object from file " + nomeFile + " ."); } finally { try { stream.close(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNNS.java b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNNS.java index 74957b5..4858a33 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNNS.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNNS.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; @@ -14,9 +13,13 @@ import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions.NeuralNet; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AquamapsNNNS extends AquamapsNative{ + private static Logger logger = LoggerFactory.getLogger(AquamapsNNNS.class); + private NeuralNet neuralnet; @Override @@ -83,7 +86,7 @@ public class AquamapsNNNS extends AquamapsNative{ if (probability>0.1) // if (probability<0) - AnalysisLogger.getLogger().debug(" Probability " + probability); + logger.debug(" Probability " + probability); // System.exit(0); return probability; @@ -104,7 +107,7 @@ public class AquamapsNNNS extends AquamapsNative{ nn = (NeuralNet) ois.readObject(); } catch (Exception ex) { ex.printStackTrace(); - AnalysisLogger.getLogger().debug("Error in reading the object from file " + nomeFile + " ."); + logger.debug("Error in reading the object from file " + nomeFile + " ."); } finally { try { stream.close(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNSuitable.java b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNSuitable.java index 3cf3fd4..ee8f265 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNSuitable.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsNNSuitable.java @@ -6,7 +6,6 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; @@ -16,9 +15,13 @@ import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AquamapsNNSuitable extends AquamapsSuitable{ + private static Logger logger = LoggerFactory.getLogger(AquamapsNNSuitable.class); + private Neural_Network neuralnet; @Override @@ -55,15 +58,15 @@ public class AquamapsNNSuitable extends AquamapsSuitable{ @Override public void init(AlgorithmConfiguration config, SessionFactory dbHibConnection) { - AnalysisLogger.getLogger().debug("Initializing ANNs"); + logger.debug("Initializing ANNs"); super.init(config,dbHibConnection); String persistencePath = config.getPersistencePath(); // String filename = persistencePath + "neuralnetwork_" + config.getParam("SpeciesName") + "_" + config.getParam("UserName")+"_"+config.getParam("NeuralNetworkName").replace(" ", ""); String nnname = config.getParam("NeuralNetworkName"); - AnalysisLogger.getLogger().debug("Init ANN in projection mode with filename: "+nnname); + logger.debug("Init ANN in projection mode with filename: "+nnname); String filename = new File(nnname).getAbsolutePath(); - AnalysisLogger.getLogger().debug("ANN: using file name: "+filename); + logger.debug("ANN: using file name: "+filename); if (filename!=null) neuralnet = loadNN(filename); } @@ -76,9 +79,9 @@ public class AquamapsNNSuitable extends AquamapsSuitable{ Object[] inputvector = new Object[wholevector.length - 6]; for (int i = 0; i < inputvector.length; i++) { inputvector[i] = wholevector[i + 1]; -// AnalysisLogger.getLogger().debug(i+": "+inputvector[i]); +// logger.debug(i+": "+inputvector[i]); } -// AnalysisLogger.getLogger().debug("species vs csquare:" + species + " , " + csquarecode); +// logger.debug("species vs csquare:" + species + " , " + csquarecode); float probability = 0; // if (csquarecode.equals("1000:102:2")) @@ -99,7 +102,7 @@ public class AquamapsNNSuitable extends AquamapsSuitable{ float probability = (float) output[0]; /* if (probability>0.1) - AnalysisLogger.getLogger().debug(" Probability " + probability); + logger.debug(" Probability " + probability); */ // System.exit(0); return probability; @@ -120,7 +123,7 @@ public class AquamapsNNSuitable extends AquamapsSuitable{ nn = (Neural_Network) ois.readObject(); } catch (Exception ex) { ex.printStackTrace(); - AnalysisLogger.getLogger().debug("Error in reading the object from file " + nomeFile + " ."); + logger.debug("Error in reading the object from file " + nomeFile + " ."); } finally { try { stream.close(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java index 833ecd9..8229143 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/AquamapsSuitable.java @@ -5,7 +5,6 @@ import java.util.HashMap; import java.util.List; import java.util.Queue; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -14,7 +13,6 @@ import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; -import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.DatabaseParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; @@ -22,9 +20,14 @@ import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributio import org.gcube.dataanalysis.ecoengine.models.cores.aquamaps.MaxMinGenerator; import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AquamapsSuitable implements SpatialProbabilityDistributionTable{ + private static Logger logger = LoggerFactory.getLogger(AquamapsSuitable.class); + + String selectAllSpeciesQuery = "select depthmin,meandepth,depthprefmin,pelagic,depthprefmax,depthmax,tempmin,layer,tempprefmin,tempprefmax,tempmax,salinitymin,salinityprefmin,salinityprefmax,salinitymax,primprodmin,primprodprefmin,primprodprefmax,primprodmax,iceconmin,iceconprefmin,iceconprefmax,iceconmax,landdistyn,landdistmin,landdistprefmin,landdistprefmax,landdistmax,nmostlat,smostlat,wmostlong,emostlong,faoareas,speciesid from %1$s;"; String csquareCodeQuery = "select csquarecode,depthmean,depthmax,depthmin, sstanmean,sbtanmean,salinitymean,salinitybmean, primprodmean,iceconann,landdist,oceanarea,centerlat,centerlong,faoaream,eezall,lme from %1$s d where oceanarea>0"; String createTableStatement = "CREATE TABLE %1$s ( speciesid character varying, csquarecode character varying, probability real, boundboxyn smallint, faoareayn smallint, faoaream integer, eezall character varying, lme integer) WITH (OIDS=FALSE ); CREATE INDEX CONCURRENTLY %1$s_idx ON %1$s USING btree (speciesid, csquarecode, faoaream, eezall, lme);"; @@ -74,18 +77,18 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{ //if it exists then set the table name if (select!=null){ hspenMinMaxLat = supposedminmaxlattable; - AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->the min max latitudes table yet exists "+hspenMinMaxLat); + logger.debug("Aquamaps Algorithm Init ->the min max latitudes table yet exists "+hspenMinMaxLat); } else{ //otherwise create it by calling the creator - AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->the min max latitudes table does not exist! - generating"); + logger.debug("Aquamaps Algorithm Init ->the min max latitudes table does not exist! - generating"); hspenMinMaxLat = generateMaxMinHspec(supposedminmaxlattable, hspenTable,config.getParam("OccurrencePointsTable"), dbHibConnection); - AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->min max latitudes table created in "+hspenMinMaxLat); + logger.debug("Aquamaps Algorithm Init ->min max latitudes table created in "+hspenMinMaxLat); } } - AnalysisLogger.getLogger().trace("Aquamaps Algorithm Init ->getting min max latitudes from "+hspenMinMaxLat); + logger.trace("Aquamaps Algorithm Init ->getting min max latitudes from "+hspenMinMaxLat); allSpeciesObservations = new HashMap>(); List SpeciesObservations = DatabaseFactory.executeSQLQuery(String.format(selectAllSpeciesObservationQuery, hspenMinMaxLat), dbHibConnection); @@ -98,7 +101,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{ allSpeciesObservations.put((String)speciesid, maxminInfo); } - AnalysisLogger.getLogger().trace("Aquamaps Algorithm Init ->init finished"); + logger.trace("Aquamaps Algorithm Init ->init finished"); } @Override @@ -185,7 +188,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{ //adjust FAO areas currentFAOAreas = core.procFAO_2050(currentFAOAreas); //get Bounding Box Information -// AnalysisLogger.getLogger().trace("TYPE:"+type); +// logger.trace("TYPE:"+type); currentSpeciesBoundingBoxInfo = core.getBoundingBoxInfo($paramData_NMostLat, $paramData_SMostLat, $paramData_WMostLong, $paramData_EMostLong, speciesObservations,type); //end of get BoundingBoxInformation } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/BayesianDistribution.java b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/BayesianDistribution.java index ad0e3da..6caa5f2 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/BayesianDistribution.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/BayesianDistribution.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; @@ -22,9 +21,13 @@ import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributio import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class BayesianDistribution implements SpatialProbabilityDistributionGeneric { + private static Logger logger = LoggerFactory.getLogger(BayesianDistribution.class); + protected static String FeaturesTableP = "FeaturesTable"; protected static String FeaturesTableColumnsP = "FeaturesColumnNames"; protected static String FinalTableLabel = "FinalTableLabel"; @@ -109,10 +112,10 @@ public abstract class BayesianDistribution implements SpatialProbabilityDistribu if (groupingFactor.trim().length()>0) insertBuffer = DatabaseUtils.insertFromBuffer(finalTableName, groupingFactor + "," + featuresTableColumns + "," + FinalTableValue, sb); -// AnalysisLogger.getLogger().debug("Insertion Query " + insertBuffer); - AnalysisLogger.getLogger().debug("Writing Distribution into the DB "); +// logger.debug("Insertion Query " + insertBuffer); + logger.debug("Writing Distribution into the DB "); DatabaseFactory.executeSQLUpdate(insertBuffer, dbConnection); - AnalysisLogger.getLogger().debug("Done!"); + logger.debug("Done!"); } @Override @@ -187,7 +190,7 @@ public abstract class BayesianDistribution implements SpatialProbabilityDistribu @Override public void init(AlgorithmConfiguration config) throws Exception { status = 0; - AnalysisLogger.getLogger().debug("Initializing Database Connection"); + logger.debug("Initializing Database Connection"); this.config = config; // initialization of the variables finalTableLabel = config.getParam(FinalTableLabel); @@ -206,12 +209,12 @@ public abstract class BayesianDistribution implements SpatialProbabilityDistribu // create a new table dbConnection = DatabaseUtils.initDBSession(config); try { - AnalysisLogger.getLogger().debug("Dropping table " + finalTableName); + logger.debug("Dropping table " + finalTableName); String dropStatement = DatabaseUtils.dropTableStatement(finalTableName); DatabaseFactory.executeSQLUpdate(dropStatement, dbConnection); - AnalysisLogger.getLogger().debug("Table " + finalTableName + " dropped"); + logger.debug("Table " + finalTableName + " dropped"); } catch (Exception e) { - AnalysisLogger.getLogger().debug("Could not drop table " + finalTableName); + logger.debug("Could not drop table " + finalTableName); } // create Table @@ -233,11 +236,11 @@ public abstract class BayesianDistribution implements SpatialProbabilityDistribu if (groupingFactor.length()>0){ createStatement = "create table "+finalTableName+" ( "+groupingFactor+" character varying "+columns+", "+FinalTableValue+" "+FinalTableValueType+")"; } - AnalysisLogger.getLogger().debug("Creating table: " + finalTableName + " by statement: " + createStatement); + logger.debug("Creating table: " + finalTableName + " by statement: " + createStatement); DatabaseFactory.executeSQLUpdate(createStatement, dbConnection); - AnalysisLogger.getLogger().debug("Table: " + finalTableName + " created"); + logger.debug("Table: " + finalTableName + " created"); /* - AnalysisLogger.getLogger().debug("Adding a new column to "+finalTableName); + logger.debug("Adding a new column to "+finalTableName); DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(finalTableLabel, FinalTableValue, FinalTableValueType), dbConnection); */ status = 10f; diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java index 836aa33..1fc63bf 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/DummyAlgorithm.java @@ -9,27 +9,24 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionGeneric; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ + private static Logger logger = LoggerFactory.getLogger(DummyAlgorithm.class); + List randomElements; String persistence; private String filename; static String persistedFilePrefix = "dummyfile"; - - public static void main (String[] args){ - String s = toString(330.6499f); - - System.out.println(s); - System.out.println(fromString(s)); - } + private static String toString(float number){ String s = ""+number; @@ -64,7 +61,7 @@ public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ @Override public void init(AlgorithmConfiguration config) { - AnalysisLogger.getLogger().trace("Dummy INIT"); + logger.trace("Dummy INIT"); randomElements = new ArrayList(); for (int i=0;i<170000;i++) { @@ -96,7 +93,7 @@ public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ @Override public List getGeographicalInfoObjects() { - AnalysisLogger.getLogger().trace("Dummy TAKING RANDOMS"); + logger.trace("Dummy TAKING RANDOMS"); List randomElements = new ArrayList(); for (int i=0;i<170000;i++) { @@ -107,7 +104,7 @@ public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ @Override public float calcProb(Object mainInfo, Object area) { -// AnalysisLogger.getLogger().debug("Calculation Probability"); +// logger.debug("Calculation Probability"); Float f1 = fromString((String) mainInfo); Float f2 = Float.valueOf((String) area); return (float) 100f*f1*f2; @@ -115,17 +112,17 @@ public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ @Override public void singleStepPreprocess(Object mainInfo, Object area) { - AnalysisLogger.getLogger().trace("Dummy SINGLE PREPROCESSING Step"); + logger.trace("Dummy SINGLE PREPROCESSING Step"); } @Override public void singleStepPostprocess(Object mainInfo, Object allAreasInformation) { - AnalysisLogger.getLogger().trace("Dummy SINGLE POSTPROCESSING Step"); + logger.trace("Dummy SINGLE POSTPROCESSING Step"); } @Override public void postProcess() { - AnalysisLogger.getLogger().trace("Dummy POSTPROCESS"); + logger.trace("Dummy POSTPROCESS"); } @Override @@ -138,14 +135,14 @@ public class DummyAlgorithm implements SpatialProbabilityDistributionGeneric{ ysize = distribution.get(s).size(); break; } - AnalysisLogger.getLogger().debug("Dummy overall dimension of the distribution: "+distribution.size()+" X "+ysize); + logger.debug("Dummy overall dimension of the distribution: "+distribution.size()+" X "+ysize); //Construct the LineNumberReader object filename = persistence+persistedFilePrefix+UUID.randomUUID(); - AnalysisLogger.getLogger().debug("Dummy Storing in "+filename); + logger.debug("Dummy Storing in "+filename); outputStream = new ObjectOutputStream(new FileOutputStream(persistence+persistedFilePrefix+"_"+UUID.randomUUID())); outputStream.writeObject(distribution); - AnalysisLogger.getLogger().debug("Dummy Stored"); + logger.debug("Dummy Stored"); } catch (Exception ex) { ex.printStackTrace(); } finally { diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/FeedForwardNeuralNetworkDistribution.java b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/FeedForwardNeuralNetworkDistribution.java index e0be2db..c275c9f 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/FeedForwardNeuralNetworkDistribution.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/FeedForwardNeuralNetworkDistribution.java @@ -1,10 +1,13 @@ package org.gcube.dataanalysis.ecoengine.spatialdistributions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FeedForwardNeuralNetworkDistribution extends BayesianDistribution{ + private static Logger logger = LoggerFactory.getLogger(FeedForwardNeuralNetworkDistribution.class); + private Neural_Network neuralnet; @Override @@ -32,7 +35,7 @@ public class FeedForwardNeuralNetworkDistribution extends BayesianDistribution{ // String filename = persistencePath + Neural_Network.generateNNName(""+mainInfo, userName, modelName); String filename = modelFile.getAbsolutePath(); neuralnet = Neural_Network.loadNN(filename); - AnalysisLogger.getLogger().debug("Using neural network with emission range: ("+neuralnet.minfactor+" ; "+neuralnet.maxfactor+"" ); + logger.debug("Using neural network with emission range: ("+neuralnet.minfactor+" ; "+neuralnet.maxfactor+"" ); } @Override diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java index fc63c1e..8ecf269 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/spatialdistributions/TestAlgorithm.java @@ -9,16 +9,19 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionGeneric; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestAlgorithm implements SpatialProbabilityDistributionGeneric{ + private static Logger logger = LoggerFactory.getLogger(TestAlgorithm.class); + @Override public ALG_PROPS[] getProperties() { ALG_PROPS[] p = {ALG_PROPS.PHENOMENON_VS_GEOINFO}; @@ -91,14 +94,14 @@ public class TestAlgorithm implements SpatialProbabilityDistributionGeneric{ ysize = distribution.get(s).size(); break; } - AnalysisLogger.getLogger().debug("overall dimension of the distribution: "+distribution.size()+" X "+ysize); + logger.debug("overall dimension of the distribution: "+distribution.size()+" X "+ysize); //Construct the LineNumberReader object filename = pers+"testProb"+UUID.randomUUID(); - AnalysisLogger.getLogger().debug(" Storing in "+filename); + logger.debug(" Storing in "+filename); outputStream = new ObjectOutputStream(new FileOutputStream(filename)); outputStream.writeObject(distribution); - AnalysisLogger.getLogger().debug("Stored"); + logger.debug("Stored"); } catch (Exception ex) { ex.printStackTrace(); } finally { diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/test/regression/Regressor.java b/src/main/java/org/gcube/dataanalysis/ecoengine/test/regression/Regressor.java index d7b97eb..37e0834 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/test/regression/Regressor.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/test/regression/Regressor.java @@ -1,12 +1,10 @@ package org.gcube.dataanalysis.ecoengine.test.regression; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; @Deprecated public class Regressor { - public static void process(ComputationalAgent agent) throws Exception { @@ -61,7 +59,6 @@ public class Regressor { config.setParam("DatabasePassword","cdec755ed6633ba"); config.setParam("DatabaseURL","jdbc:postgresql://postgresql-srv-dev.d4science.org/dmalgorithms"); config.setParam("DatabaseDriver","org.postgresql.Driver"); - AnalysisLogger.setLogger(config.getConfigPath()+AlgorithmConfiguration.defaultLoggerFile); return config; } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsDuplicatesDeleter.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsDuplicatesDeleter.java index baec9e6..173d825 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsDuplicatesDeleter.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsDuplicatesDeleter.java @@ -3,9 +3,7 @@ package org.gcube.dataanalysis.ecoengine.transducers; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; -import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; @@ -15,10 +13,15 @@ import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { + private static Logger logger = LoggerFactory.getLogger(DataAnalysis.class); + String tableName; List records = new ArrayList(); @@ -68,7 +71,6 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { @Override public void init() throws Exception { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); lonFld = config.getParam(longitudeColumn); latFld = config.getParam(latitudeColumn); recordedByFld = config.getParam(recordedByColumn); @@ -105,20 +107,20 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { public void takeFullRanges() { // take the elements from sx table - AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName); + logger.info("Taking elements from left table: " + leftTableName); leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(tableName, columns.toString(),"")+" limit 100000", dbconnection); } public void takeRange(int offsetLeft, int numLeft, int offsetRight, int numRight) { // take the elements from sx table - AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName); + logger.info("Taking elements from left table: " + leftTableName); leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(leftTableName, columns.toString(), "offset " + offsetLeft + " limit " + numLeft), dbconnection); } public void computeRange() throws Exception { try { // for each element in dx - AnalysisLogger.getLogger().trace("Processing"); + logger.trace("Processing"); status = 10; int similaritiesCounter = 0; int allrows = 0; @@ -141,7 +143,7 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { if (prob >= confidenceValue) { similaritiesCounter++; if (isBetterThan(testOcc, yetInserted)) { - AnalysisLogger.getLogger().trace("Found a similarity with P=" + prob + " between (" + "\"" + testOcc.scientificName + "\"" + "," + testOcc.x + "\"" + "," + "\"" + testOcc.y + "\"" + "," + "\"" + testOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(testOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + yetInserted.scientificName + "\"" + "," + "\"" + yetInserted.x + "\"" + "," + "\"" + yetInserted.y + "\"" + "," + "\"" + yetInserted.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(yetInserted.eventdate) + "\"" + ")"); + logger.trace("Found a similarity with P=" + prob + " between (" + "\"" + testOcc.scientificName + "\"" + "," + testOcc.x + "\"" + "," + "\"" + testOcc.y + "\"" + "," + "\"" + testOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(testOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + yetInserted.scientificName + "\"" + "," + "\"" + yetInserted.x + "\"" + "," + "\"" + yetInserted.y + "\"" + "," + "\"" + yetInserted.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(yetInserted.eventdate) + "\"" + ")"); objectstoinsert.remove(k); k--; insertedSize--; @@ -164,19 +166,19 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { rowcounter++; } - AnalysisLogger.getLogger().trace("Found " + similaritiesCounter + " similarities on " + allrows + " distinct elements"); + logger.trace("Found " + similaritiesCounter + " similarities on " + allrows + " distinct elements"); status = 90; // transform the complete list into a table persist(); // close DB connection } } catch (Exception e) { - AnalysisLogger.getLogger().error("error",e); + logger.error("error",e); throw e; } finally { shutdown(); status = 100; - AnalysisLogger.getLogger().trace("Occ Points Processing Finished and db closed"); + logger.trace("Occ Points Processing Finished and db closed"); } } @@ -185,25 +187,25 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { try { // init DB connection - AnalysisLogger.getLogger().trace("Initializing DB Connection"); + logger.trace("Initializing DB Connection"); dbconnection = DatabaseUtils.initDBSession(config); - AnalysisLogger.getLogger().trace("Taking Table Description"); - AnalysisLogger.getLogger().trace("Creating final table: " + finalTableName); + logger.trace("Taking Table Description"); + logger.trace("Creating final table: " + finalTableName); // create new merged table try { DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection); } catch (Exception e1) { } - AnalysisLogger.getLogger().trace("Preparing table: " + finalTableName); + logger.trace("Preparing table: " + finalTableName); prepareFinalTable(); - AnalysisLogger.getLogger().trace("Extracting columns from: " + finalTableName); + logger.trace("Extracting columns from: " + finalTableName); extractColumnNames(); - AnalysisLogger.getLogger().trace("Taken Table Description: " + columns); + logger.trace("Taken Table Description: " + columns); // take distinct elements from table - AnalysisLogger.getLogger().trace("Taking elements from table: " + tableName); + logger.trace("Taking elements from table: " + tableName); List rows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(tableName, columns.toString(), ""), dbconnection); // for each element in dx - AnalysisLogger.getLogger().trace("Processing"); + logger.trace("Processing"); status = 10; int similaritiesCounter = 0; int allrows = rows.size(); @@ -224,7 +226,7 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { if (prob >= confidenceValue) { similaritiesCounter++; if (isBetterThan(testOcc, yetInserted)) { - AnalysisLogger.getLogger().trace("Found a similarity with P=" + prob + " between (" + "\"" + testOcc.scientificName + "\"" + "," + testOcc.x + "\"" + "," + "\"" + testOcc.y + "\"" + "," + "\"" + testOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(testOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + yetInserted.scientificName + "\"" + "," + "\"" + yetInserted.x + "\"" + "," + "\"" + yetInserted.y + "\"" + "," + "\"" + yetInserted.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(yetInserted.eventdate) + "\"" + ")"); + logger.trace("Found a similarity with P=" + prob + " between (" + "\"" + testOcc.scientificName + "\"" + "," + testOcc.x + "\"" + "," + "\"" + testOcc.y + "\"" + "," + "\"" + testOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(testOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + yetInserted.scientificName + "\"" + "," + "\"" + yetInserted.x + "\"" + "," + "\"" + yetInserted.y + "\"" + "," + "\"" + yetInserted.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(yetInserted.eventdate) + "\"" + ")"); objectstoinsert.remove(k); k--; insertedSize--; @@ -247,19 +249,19 @@ public class OccurrencePointsDuplicatesDeleter extends OccurrencePointsMerger { rowcounter++; } - AnalysisLogger.getLogger().trace("Found " + similaritiesCounter + " similarities on " + allrows + " distinct elements"); + logger.trace("Found " + similaritiesCounter + " similarities on " + allrows + " distinct elements"); status = 90; // transform the complete list into a table persist(); // close DB connection } catch (Exception e) { - AnalysisLogger.getLogger().trace("An error occurred " + e.getLocalizedMessage()); + logger.trace("An error occurred " + e.getLocalizedMessage()); throw e; } finally { if (dbconnection != null) dbconnection.close(); status = 100; - AnalysisLogger.getLogger().trace("Occ Points Processing Finished and db closed"); + logger.trace("Occ Points Processing Finished and db closed"); } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsInSeaOnEarth.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsInSeaOnEarth.java index feb96d2..a5493b8 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsInSeaOnEarth.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsInSeaOnEarth.java @@ -3,9 +3,7 @@ package org.gcube.dataanalysis.ecoengine.transducers; import java.util.ArrayList; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; -import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; @@ -16,9 +14,13 @@ import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class OccurrencePointsInSeaOnEarth extends OccurrencePointsMerger{ + private static Logger logger = LoggerFactory.getLogger(OccurrencePointsInSeaOnEarth.class); + //NOTE: 0.125 is the diagonal of a csquare, which is the maximum extent to which a point can lie in a csquare private static String inthesea="select * into %1$s from (select distinct a.* from %2$s as a join hcaf_d as b on ((b.centerlat-a.%3$s)*(b.centerlat-a.%3$s)+(b.centerlong-a.%4$s)*(b.centerlong-a.%4$s)<= 0.125) and b.oceanarea>0) as t limit 10000"; private static String onearth="select * into %1$s from (select distinct a.* from %2$s as a join hcaf_d as b on ((b.centerlat-a.%3$s)*(b.centerlat-a.%3$s)+(b.centerlong-a.%4$s)*(b.centerlong-a.%4$s)<= 0.125) and b.landdist<=0.3) as t limit 10000"; @@ -57,7 +59,6 @@ public class OccurrencePointsInSeaOnEarth extends OccurrencePointsMerger{ @Override public void init() throws Exception { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); lonFld = config.getParam(longitudeColumn); latFld = config.getParam(latitudeColumn); tableName = config.getParam(tableNameF); @@ -82,10 +83,10 @@ public class OccurrencePointsInSeaOnEarth extends OccurrencePointsMerger{ try { // init DB connection - AnalysisLogger.getLogger().trace("Initializing DB Connection"); + logger.trace("Initializing DB Connection"); dbconnection = DatabaseUtils.initDBSession(config); - AnalysisLogger.getLogger().trace("Taking Table Description"); - AnalysisLogger.getLogger().trace("Creating merged table: " + finalTableName); + logger.trace("Taking Table Description"); + logger.trace("Creating merged table: " + finalTableName); // create new merged table try{ DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection); @@ -98,10 +99,10 @@ public class OccurrencePointsInSeaOnEarth extends OccurrencePointsMerger{ else generationquery = String.format(onearth,finalTableName,tableName,latFld,lonFld); - AnalysisLogger.getLogger().trace("Applying filter " + filter.name()); - AnalysisLogger.getLogger().trace("Applying query " + generationquery); + logger.trace("Applying filter " + filter.name()); + logger.trace("Applying query " + generationquery); DatabaseFactory.executeSQLUpdate(generationquery, dbconnection); - AnalysisLogger.getLogger().trace("Final Table created!"); + logger.trace("Final Table created!"); } catch (Exception e) { throw e; @@ -111,7 +112,7 @@ public class OccurrencePointsInSeaOnEarth extends OccurrencePointsMerger{ dbconnection.close(); }catch(Exception e2){} status = 100; - AnalysisLogger.getLogger().trace("Occ Points Processing Finished and db closed"); + logger.trace("Occ Points Processing Finished and db closed"); } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsMerger.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsMerger.java index 90378a8..f638967 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsMerger.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/OccurrencePointsMerger.java @@ -7,7 +7,6 @@ import java.util.Date; import java.util.List; import org.gcube.contentmanagement.graphtools.utils.DateGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.gcube.contentmanagement.lexicalmatcher.utils.DistanceCalculator; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; @@ -26,9 +25,13 @@ import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class OccurrencePointsMerger implements Transducerer { + private static Logger logger = LoggerFactory.getLogger(OccurrencePointsMerger.class); + static protected String finalTableNameL = "final_Table_Name"; static protected String longitudeColumn = "longitudeColumn"; static protected String latitudeColumn = "latitudeColumn"; @@ -132,7 +135,7 @@ public class OccurrencePointsMerger implements Transducerer { */ record.eventdate = DateGuesser.convertDate(value$); if (displaydateconvert) { - AnalysisLogger.getLogger().info("From " + value$ + "->" + convert2conventionalFormat(record.eventdate) + " pattern " + DateGuesser.getPattern(value$)); + logger.info("From " + value$ + "->" + convert2conventionalFormat(record.eventdate) + " pattern " + DateGuesser.getPattern(value$)); displaydateconvert = false; } @@ -385,7 +388,7 @@ public class OccurrencePointsMerger implements Transducerer { int todel = objectstodelete.size(); int counter = 0; StringBuffer buffer = new StringBuffer(); - AnalysisLogger.getLogger().info("Deleting " + todel + " objects"); + logger.info("Deleting " + todel + " objects"); if (todel > 0) { for (OccurrenceRecord record : objectstodelete) { buffer.append("("); @@ -434,9 +437,9 @@ public class OccurrencePointsMerger implements Transducerer { if ((counter>0)&&(counter%500==0)){ String updateQ = DatabaseUtils.deleteFromBuffer(finalTableName, buffer); -// AnalysisLogger.getLogger().debug("Update:\n"+updateQ); +// logger.debug("Update:\n"+updateQ); DatabaseFactory.executeSQLUpdate(updateQ, dbconnection); - AnalysisLogger.getLogger().info("Partial Objects deleted"); + logger.info("Partial Objects deleted"); buffer = new StringBuffer(); } else @@ -448,15 +451,15 @@ public class OccurrencePointsMerger implements Transducerer { } String updateQ = DatabaseUtils.deleteFromBuffer(finalTableName, buffer); -// AnalysisLogger.getLogger().debug("Update:\n"+updateQ); +// logger.debug("Update:\n"+updateQ); DatabaseFactory.executeSQLUpdate(updateQ, dbconnection); - AnalysisLogger.getLogger().info("All Objects deleted"); + logger.info("All Objects deleted"); } buffer = new StringBuffer(); ArrayList insertedStrings = new ArrayList(); int toins = objectstoinsert.size(); - AnalysisLogger.getLogger().info("Inserting " + toins + " objects"); + logger.info("Inserting " + toins + " objects"); counter = 0; if (toins > 0) { for (OccurrenceRecord record : objectstoinsert) { @@ -470,7 +473,7 @@ public class OccurrencePointsMerger implements Transducerer { if ((counter>0)&&(counter%500==0)){ insertBuffer(buffer); - AnalysisLogger.getLogger().info("Partial Objects inserted"); + logger.info("Partial Objects inserted"); buffer = new StringBuffer(); } else @@ -481,9 +484,9 @@ public class OccurrencePointsMerger implements Transducerer { } insertBuffer(buffer); - AnalysisLogger.getLogger().info("Objects inserted"); + logger.info("Objects inserted"); - AnalysisLogger.getLogger().info("Inserted " + counter + " objects"); + logger.info("Inserted " + counter + " objects"); } objectstoinsert = null; @@ -503,7 +506,7 @@ public class OccurrencePointsMerger implements Transducerer { String updateQ = "SET datestyle = \"ISO, MDY\"; "+DatabaseUtils.insertFromString(finalTableName, columns.toString(), subBuffer); // System.out.println("Update:\n"+updateQ); - AnalysisLogger.getLogger().debug("Update:\n"+updateQ); + logger.debug("Update:\n"+updateQ); DatabaseFactory.executeSQLUpdate(updateQ, dbconnection); } @@ -527,14 +530,14 @@ public class OccurrencePointsMerger implements Transducerer { public void initDB(boolean buildTable) throws Exception { // init DB connection - AnalysisLogger.getLogger().info("Initializing DB Connection"); + logger.info("Initializing DB Connection"); dbconnection = DatabaseUtils.initDBSession(config); - AnalysisLogger.getLogger().info("Taking Table Description"); + logger.info("Taking Table Description"); extractColumnNames(); if (buildTable) { - AnalysisLogger.getLogger().info("Taken Table Description: " + columns); - AnalysisLogger.getLogger().info("Creating final table: " + finalTableName); + logger.info("Taken Table Description: " + columns); + logger.info("Creating final table: " + finalTableName); // create new merged table try { @@ -574,36 +577,36 @@ public class OccurrencePointsMerger implements Transducerer { public void takeFullRanges() { // take the elements from sx table - AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName); + logger.info("Taking elements from left table: " + leftTableName); leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), " limit 10000"), dbconnection); // take the elements from dx table - AnalysisLogger.getLogger().info("Taking elements from right table: " + rightTableName); + logger.info("Taking elements from right table: " + rightTableName); rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), " limit 10000"), dbconnection); } public void takeRange(int offsetLeft, int numLeft, int offsetRight, int numRight) { // take the elements from sx table - AnalysisLogger.getLogger().info("Taking elements from left table: " + leftTableName); + logger.info("Taking elements from left table: " + leftTableName); leftRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(leftTableName, columns.toString(), "order by "+columns.toString()+" offset " + offsetLeft + " limit " + numLeft), dbconnection); // take the elements from dx table - AnalysisLogger.getLogger().info("Taking elements from right table: " + rightTableName); + logger.info("Taking elements from right table: " + rightTableName); rightRows = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsElementsStatement(rightTableName, columns.toString(), "order by "+columns.toString()+" offset " + offsetRight + " limit " + numRight), dbconnection); } public void computeRange() throws Exception { try { - AnalysisLogger.getLogger().info("Processing " + leftTableName + " vs " + rightTableName); - // AnalysisLogger.getLogger().info("ELEMENTS " + getNumLeftObjects() + " vs " + getNumRightObjects()); + logger.info("Processing " + leftTableName + " vs " + rightTableName); + // logger.info("ELEMENTS " + getNumLeftObjects() + " vs " + getNumRightObjects()); status = 10; int rightCounter = 0; int similaritiesCounter = 0; int allrightrows = rightRows.size(); if ((allrightrows > 0) && (getNumLeftObjects() > 0)) { for (Object rRow : rightRows) { - // AnalysisLogger.getLogger().info("RR CONV"); + // logger.info("RR CONV"); // transform into an occurrence object OccurrenceRecord rightOcc = row2OccurrenceRecord((Object[]) rRow); - // AnalysisLogger.getLogger().info("RR CONV - OK"); + // logger.info("RR CONV - OK"); // for each element in sx int k = 0; boolean found = false; @@ -611,15 +614,15 @@ public class OccurrencePointsMerger implements Transducerer { OccurrenceRecord bestleftOcc = null; for (Object lRow : leftRows) { OccurrenceRecord leftOcc = null; - // AnalysisLogger.getLogger().info("LL CONV"); + // logger.info("LL CONV"); leftOcc = row2OccurrenceRecord((Object[]) lRow); p = extProb(leftOcc, rightOcc); - // AnalysisLogger.getLogger().info("P"); + // logger.info("P"); if (p >= confidenceValue) { bestleftOcc = leftOcc; found = true; similaritiesCounter++; - AnalysisLogger.getLogger().info("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")"); + logger.info("Found a similarity with P=" + p + " between (" + "\"" + leftOcc.scientificName + "\"" + ",\"" + leftOcc.x + "\"" + "," + "\"" + leftOcc.y + "\"" + "," + "\"" + leftOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(leftOcc.eventdate) + "\"" + ") VS " + "(" + "\"" + rightOcc.scientificName + "\"" + "," + "\"" + rightOcc.x + "\"" + "," + "\"" + rightOcc.y + "\"" + "," + "\"" + rightOcc.recordedby + "\"" + "," + "\"" + convert2conventionalFormat(rightOcc.eventdate) + "\"" + ")"); // break; if (!firstbest) manageHighProbability(p, bestleftOcc, rightOcc); @@ -646,23 +649,23 @@ public class OccurrencePointsMerger implements Transducerer { status = Math.min(90, 10f + (80 * ((float) rightCounter) / ((float) allrightrows))); if (rightCounter % 500 == 0) { - AnalysisLogger.getLogger().info("Persisting ... " + rightCounter + " over " + allrightrows); + logger.info("Persisting ... " + rightCounter + " over " + allrightrows); persist(); } } } - AnalysisLogger.getLogger().info("Found " + similaritiesCounter + " similarities on " + rightCounter + " elements"); + logger.info("Found " + similaritiesCounter + " similarities on " + rightCounter + " elements"); status = 90; // transform the complete list into a table persist(); // close DB connection } catch (Exception e) { - AnalysisLogger.getLogger().error("error in computation",e); + logger.error("error in computation",e); throw e; } finally { shutdown(); status = 100; - AnalysisLogger.getLogger().info("Occ Points Processing Finished and db closed"); + logger.info("Occ Points Processing Finished and db closed"); } } @@ -679,7 +682,7 @@ public class OccurrencePointsMerger implements Transducerer { public void postProcess() throws Exception{ /* - AnalysisLogger.getLogger().info("Post processing ... Deleting duplicates"); + logger.info("Post processing ... Deleting duplicates"); OccurrencePointsDuplicatesDeleter opdd = new OccurrencePointsDuplicatesDeleter(); opdd.setConfiguration(config); @@ -687,7 +690,7 @@ public class OccurrencePointsMerger implements Transducerer { opdd.initDB(false); opdd.takeFullRanges(); opdd.computeRange(); - AnalysisLogger.getLogger().info("Post processing ... Finished"); + logger.info("Post processing ... Finished"); */ } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/QueryExecutor.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/QueryExecutor.java index c3de97d..c446f9f 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/QueryExecutor.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/QueryExecutor.java @@ -2,21 +2,25 @@ package org.gcube.dataanalysis.ecoengine.transducers; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis; import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; //implements a creator of tables which simply executes a query public class QueryExecutor implements Transducerer { + private static Logger logger = LoggerFactory.getLogger(DataAnalysis.class); + protected String query = ""; protected String finalTableName = ""; protected String finalTableLabel = ""; @@ -85,21 +89,21 @@ public class QueryExecutor implements Transducerer { SessionFactory dbconnection = null; try{ - AnalysisLogger.getLogger().trace("Initializing DB Connection"); + logger.trace("Initializing DB Connection"); dbconnection = DatabaseUtils.initDBSession(config); - AnalysisLogger.getLogger().trace("Deleting Previous Table "+DatabaseUtils.dropTableStatement(finalTableName)); + logger.trace("Deleting Previous Table "+DatabaseUtils.dropTableStatement(finalTableName)); try{ DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(finalTableName), dbconnection); }catch(Exception ee){ } status = 10; - AnalysisLogger.getLogger().trace("Deleted"); - AnalysisLogger.getLogger().trace("Executing query: "+query); + logger.trace("Deleted"); + logger.trace("Executing query: "+query); DatabaseFactory.executeSQLUpdate(query, dbconnection); - AnalysisLogger.getLogger().trace("Executed!"); + logger.trace("Executed!"); } catch (Exception e) { - AnalysisLogger.getLogger().trace("ERROR:",e); + logger.trace("ERROR:",e); throw e; } finally { if (dbconnection != null) @@ -107,7 +111,7 @@ public class QueryExecutor implements Transducerer { dbconnection.close(); }catch(Exception e2){} status = 100; - AnalysisLogger.getLogger().trace("Processing Finished and db closed"); + logger.trace("Processing Finished and db closed"); } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java index 34fc87e..1d4c3fd 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java @@ -13,7 +13,6 @@ import java.util.List; import org.gcube.contentmanagement.graphtools.utils.DateGuesser; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; @@ -21,6 +20,7 @@ import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis; import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm; import org.gcube.dataanalysis.ecoengine.signals.PeriodicityDetector; import org.gcube.dataanalysis.ecoengine.signals.SignalProcessing; @@ -33,6 +33,8 @@ import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.Operations; import org.gcube.dataanalysis.ecoengine.utils.Tuple; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { private static String timeSeriesTable = "TimeSeriesTable"; @@ -58,6 +60,8 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { LOW, NORMAL, HIGH } + private static Logger logger = LoggerFactory.getLogger(DataAnalysis.class); + @Override public void init() throws Exception { @@ -112,18 +116,18 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { } } - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Table Name: " + tablename); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Time Column: " + timecolumn); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Values Column: " + valuescolum); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Aggregation: " + aggregationFunc); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->FFT Window Samples: " + fftWindowSamplesDouble); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->SSA Window Samples: " + windowLength); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->SSA Eigenvalues threshold: " + eigenvaluespercthr); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->SSA Points to Reconstruct: " + pointsToReconstruct); + logger.debug("TimeSeriesAnalysis->Table Name: " + tablename); + logger.debug("TimeSeriesAnalysis->Time Column: " + timecolumn); + logger.debug("TimeSeriesAnalysis->Values Column: " + valuescolum); + logger.debug("TimeSeriesAnalysis->Aggregation: " + aggregationFunc); + logger.debug("TimeSeriesAnalysis->FFT Window Samples: " + fftWindowSamplesDouble); + logger.debug("TimeSeriesAnalysis->SSA Window Samples: " + windowLength); + logger.debug("TimeSeriesAnalysis->SSA Eigenvalues threshold: " + eigenvaluespercthr); + logger.debug("TimeSeriesAnalysis->SSA Points to Reconstruct: " + pointsToReconstruct); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Extracting Points..."); + logger.debug("TimeSeriesAnalysis->Extracting Points..."); String query = "select * from (select " + aggregationFunc + "( CAST ( " + valuescolum + " as real))," + timecolumn + " from " + tablename + " group by " + timecolumn + ") as a"; - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Query to execute: " + query); + logger.debug("TimeSeriesAnalysis->Query to execute: " + query); List results = DatabaseFactory.executeSQLQuery(query, dbconnection); status = 10; @@ -132,9 +136,9 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { else if (results.size() > maxpoints) throw new Exception("Too long Time Series: a maximum of distinct " + maxpoints + " in time is allowed"); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Points Extracted!"); + logger.debug("TimeSeriesAnalysis->Points Extracted!"); // build signal - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Building signal"); + logger.debug("TimeSeriesAnalysis->Building signal"); List> signal = new ArrayList>(); int sizesignal = 0; for (Object row : results) { @@ -146,28 +150,28 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { } status = 20; - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Signal built with success. Size: " + sizesignal); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Building Time Series"); + logger.debug("TimeSeriesAnalysis->Signal built with success. Size: " + sizesignal); + logger.debug("TimeSeriesAnalysis->Building Time Series"); TimeSeries ts = TimeSeries.buildFromSignal(signal, config); String timepattern = ts.getTimepattern(); String chartpattern = "MM-dd-yy"; if (timepattern.equals("s") || (DateGuesser.isJavaDateOrigin(ts.getTime()[0]) && DateGuesser.isJavaDateOrigin(ts.getTime()[ts.getTime().length - 1]))) { - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Changing chart pattern to Seconds!"); + logger.debug("TimeSeriesAnalysis->Changing chart pattern to Seconds!"); chartpattern = "HH:mm:ss:SS"; } else - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Chart pattern remains " + chartpattern); + logger.debug("TimeSeriesAnalysis->Chart pattern remains " + chartpattern); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniformly sampling the signal"); + logger.debug("TimeSeriesAnalysis->Uniformly sampling the signal"); if (display) SignalProcessing.displaySignalWithTime(ts.getValues(), ts.getTime(), "Time Series", chartpattern); signalImg = SignalProcessing.renderSignalWithTime(ts.getValues(), ts.getTime(), "Original Time Series", chartpattern); int originalSignalLength = ts.getValues().length; ts.convertToUniformSignal(0); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniform sampling finished"); + logger.debug("TimeSeriesAnalysis->Uniform sampling finished"); status = 30; // spectrum and signal processing - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Detecting periodicities"); + logger.debug("TimeSeriesAnalysis->Detecting periodicities"); PeriodicityDetector pd = new PeriodicityDetector(); LinkedHashMap frequencies = pd.detectAllFrequencies(ts.getValues(), 1, 0.01f, 0.5f, -1, fftWindowSamplesDouble, sensitivity, display); outputParameters.put("Original Time Series Length", "" + originalSignalLength); @@ -187,11 +191,11 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { * outputParameters.put("Detected Frequency (samples^-1)", ""+MathFunctions.roundDecimal(F,2)); outputParameters.put("Indecision on Frequency", "["+MathFunctions.roundDecimal(pd.lowermeanF,2)+" , "+MathFunctions.roundDecimal(pd.uppermeanF,2) + "]"); outputParameters.put("Average detected Period (samples)", ""+MathFunctions.roundDecimal(pd.meanPeriod,2)); outputParameters.put("Indecision on Average Period", "["+MathFunctions.roundDecimal(pd.lowermeanPeriod,2)+" , "+MathFunctions.roundDecimal(pd.uppermeanPeriod,2) + "]"); outputParameters.put("Samples range in which periodicity was detected", "from "+pd.startPeriodSampleIndex+" to "+pd.endPeriodSampleIndex); outputParameters.put("Period Strength with interpretation", ""+MathFunctions.roundDecimal(pd.periodicityStrength,2)+" ("+pd.getPeriodicityStregthInterpretation()+")"); */ - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Periodicity Detected!"); + logger.debug("TimeSeriesAnalysis->Periodicity Detected!"); status = 60; System.gc(); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Executing SSA analysis"); + logger.debug("TimeSeriesAnalysis->Executing SSA analysis"); List values = new ArrayList(); for (double v : ts.getValues()) { values.add(v); @@ -202,14 +206,14 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { if (windowLength < ts.getValues().length) ssa = SSAWorkflow.applyCompleteWorkflow(values, windowLength, eigenvaluespercthr, pointsToReconstruct, false); else { - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->SSA analysis impossible to complete"); + logger.debug("TimeSeriesAnalysis->SSA analysis impossible to complete"); outputParameters.put("SSA Note:", "The window length is higher than the signal length. Please reduce the value to less than the signal length."); return; } - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->SSA analysis completed"); + logger.debug("TimeSeriesAnalysis->SSA analysis completed"); status = 70; - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Rendering Images"); + logger.debug("TimeSeriesAnalysis->Rendering Images"); uniformSignalImg = SignalProcessing.renderSignalWithTime(ts.getValues(), ts.getTime(), "Uniformly Sampled Time Series", chartpattern); if (uniformSignalImg == null) outputParameters.put("Note:", "The charts for uniformly sampled and forecasted signals contain too many points and will not be displayed. The values will be only reported in the output file."); @@ -235,9 +239,9 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { eigenValues[i] = ssa.getPercentList().get(i); } eigenValuesImg = SignalProcessing.renderSignalWithGenericTime(eigenValues, 0f, 1, "SSA Eigenvalues"); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Images Rendered"); + logger.debug("TimeSeriesAnalysis->Images Rendered"); System.gc(); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Producing Files"); + logger.debug("TimeSeriesAnalysis->Producing Files"); outputfilename = new File(config.getPersistencePath(), valuescolum + "_SignalProcessing.csv"); BufferedWriter bw = new BufferedWriter(new FileWriter(outputfilename)); bw.write("Uniformly Sampled Time Series,Time Line,Forecasted Time Series,SSA Eigenvalues\n"); @@ -263,15 +267,15 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { bw.write("\n"); } bw.close(); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Files Produced"); + logger.debug("TimeSeriesAnalysis->Files Produced"); if (display) { SignalProcessing.displaySignalWithTime(ts.getValues(), ts.getTime(), "Uniformly Sampled Time Series", chartpattern); SignalProcessing.displaySignalWithGenericTime(ts.getValues(), 0, 1, "Uniformly Sampled Time Series in Samples"); SignalProcessing.displaySignalWithTime(ssa.getForecastSignal(), newtimes, "Forecasted Time Series", chartpattern); SignalProcessing.displaySignalWithGenericTime(eigenValues, 0f, 1, "SSA Eigenvalues"); } - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->" + outputParameters); - AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Computation has finished"); + logger.debug("TimeSeriesAnalysis->" + outputParameters); + logger.debug("TimeSeriesAnalysis->Computation has finished"); } catch (Throwable e) { e.printStackTrace(); throw new Exception(e.getLocalizedMessage()); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/AbstractChartsProducer.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/AbstractChartsProducer.java index 4a0fc28..3a59c09 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/AbstractChartsProducer.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/AbstractChartsProducer.java @@ -8,7 +8,6 @@ import java.util.LinkedHashMap; import java.util.List; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; @@ -18,9 +17,13 @@ import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; import org.gcube.dataanalysis.ecoengine.utils.IOHelper; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class AbstractChartsProducer extends StandardLocalExternalAlgorithm { + private static Logger logger = LoggerFactory.getLogger(AbstractChartsProducer.class); + protected static String inputTableParameter = "InputTable"; protected static String attributesParameter = "Attributes"; protected static String quantitiesParameter = "Quantities"; @@ -57,7 +60,7 @@ public abstract class AbstractChartsProducer extends StandardLocalExternalAlgori @Override public void init() throws Exception { - AnalysisLogger.getLogger().debug("ChartsProducer Initialized"); + logger.debug("ChartsProducer Initialized"); } @Override @@ -107,20 +110,19 @@ public abstract class AbstractChartsProducer extends StandardLocalExternalAlgori @Override protected void process() throws Exception { status = 10; - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); try { - AnalysisLogger.getLogger().debug("ChartsProducer started"); + logger.debug("ChartsProducer started"); String driver = config.getParam("DatabaseDriver"); String username = config.getParam("DatabaseUserName"); String pwd = config.getParam("DatabasePassword"); String url = config.getParam("DatabaseURL"); String table = IOHelper.getInputParameter(config, inputTableParameter); - AnalysisLogger.getLogger().debug("ChartsProducer: Driver: " + driver + " User " + username + " URL " + url + " Table: " + table); + logger.debug("ChartsProducer: Driver: " + driver + " User " + username + " URL " + url + " Table: " + table); connection = DatabaseUtils.initDBSession(config); - AnalysisLogger.getLogger().debug("ChartsProducer: Connection initialized"); + logger.debug("ChartsProducer: Connection initialized"); LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); conf.setDatabaseUserName(username); @@ -140,7 +142,7 @@ public abstract class AbstractChartsProducer extends StandardLocalExternalAlgori for (String quantity : quantities) { //produce chart with dimensions,quantity, time String query = InfoRetrievalQuery(table,dimensions,quantity,time); - AnalysisLogger.getLogger().debug("ChartsProducer: Query for retrieving information "+query); + logger.debug("ChartsProducer: Query for retrieving information "+query); List values = DatabaseFactory.executeSQLQuery(query, connection); if (values==null) throw new Exception("There are issued in managing selected attributes and quantities"); @@ -166,7 +168,7 @@ public abstract class AbstractChartsProducer extends StandardLocalExternalAlgori if (noCharts) throw new Exception("Error - no chart was produced because of incompatibility with the selected input parameters"); - AnalysisLogger.getLogger().debug("ChartsProducer: finished"); + logger.debug("ChartsProducer: finished"); } catch (Throwable e) { e.printStackTrace(); @@ -182,7 +184,7 @@ public abstract class AbstractChartsProducer extends StandardLocalExternalAlgori @Override public void shutdown() { - AnalysisLogger.getLogger().debug("ChartsProducer shutdown"); + logger.debug("ChartsProducer shutdown"); DatabaseUtils.closeDBConnection(connection); } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/QuantitiesAttributesChartsTransducerer.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/QuantitiesAttributesChartsTransducerer.java index 16ccb1b..4afa0ad 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/QuantitiesAttributesChartsTransducerer.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/QuantitiesAttributesChartsTransducerer.java @@ -11,7 +11,6 @@ import org.gcube.contentmanagement.graphtools.plotting.graphs.HistogramGraph; import org.gcube.contentmanagement.graphtools.plotting.graphs.RadarGraph; import org.gcube.contentmanagement.graphtools.plotting.graphs.ScatterGraphGeneric; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -22,9 +21,12 @@ import org.jfree.data.category.DefaultCategoryDataset; import org.jfree.data.function.NormalDistributionFunction2D; import org.jfree.data.general.DatasetUtilities; import org.jfree.data.xy.XYSeriesCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class QuantitiesAttributesChartsTransducerer extends AbstractChartsProducer { + private static Logger logger = LoggerFactory.getLogger(QuantitiesAttributesChartsTransducerer.class); @Override protected void setInputParameters() { @@ -78,7 +80,7 @@ public class QuantitiesAttributesChartsTransducerer extends AbstractChartsProduc // List meanvar = DatabaseFactory.executeSQLQuery("select avg("+"( CAST ( " + quantity + " as real))"+"), variance("+"( CAST ( " + quantity + " as real))"+") from "+IOHelper.getInputParameter(config, inputTableParameter), connection); String selectMeanVar = "select avg(( CAST ( "+quantity+" as real))), variance(( CAST ( "+quantity+" as real))) from (select "+quantity+" from "+IOHelper.getInputParameter(config, inputTableParameter)+" where "+quantity+" IS NOT NULL and CAST("+quantity+" as character varying) <> '' ) as a"; - AnalysisLogger.getLogger().debug("QuantitiesAttributesCharts: select for mean and variance: "+selectMeanVar); + logger.debug("QuantitiesAttributesCharts: select for mean and variance: "+selectMeanVar); List meanvar = DatabaseFactory.executeSQLQuery(selectMeanVar, connection); @@ -86,13 +88,13 @@ public class QuantitiesAttributesChartsTransducerer extends AbstractChartsProduc try{ meanvarsrow = (Object[]) meanvar.get(0);}catch(Exception e){ - AnalysisLogger.getLogger().debug("QuantitiesAttributesCharts: cannot detect mean and variance for "+quantity); + logger.debug("QuantitiesAttributesCharts: cannot detect mean and variance for "+quantity); } double mean = MathFunctions.roundDecimal(Double.parseDouble(""+meanvarsrow[0]),2); double variance = MathFunctions.roundDecimal(Math.sqrt(Double.parseDouble(""+meanvarsrow[1])),2); - AnalysisLogger.getLogger().debug("QuantitiesAttributesCharts: " + mean + " and variance:" + variance); + logger.debug("QuantitiesAttributesCharts: " + mean + " and variance:" + variance); NormalDistributionFunction2D normaldistributionfunction2d = new NormalDistributionFunction2D(mean, variance); org.jfree.data.xy.XYSeries gaussianxyseries = DatasetUtilities.sampleFunction2DToSeries(normaldistributionfunction2d, (mean - (2 * variance)), (mean + (2 * variance)), 121, "Distribution of "+quantity); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/TimeSeriesChartsTransducerer.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/TimeSeriesChartsTransducerer.java index 783571c..4aa3d73 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/TimeSeriesChartsTransducerer.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/charts/TimeSeriesChartsTransducerer.java @@ -13,7 +13,6 @@ import org.gcube.contentmanagement.graphtools.data.conversions.ImageTools; import org.gcube.contentmanagement.graphtools.plotting.graphs.TimeSeriesGraph; import org.gcube.contentmanagement.graphtools.plotting.graphs.TransectLineGraph; import org.gcube.contentmanagement.graphtools.utils.DateGuesser; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; @@ -26,9 +25,13 @@ import org.jfree.data.time.FixedMillisecond; import org.jfree.data.time.TimeSeries; import org.jfree.data.time.TimeSeriesCollection; import org.jfree.data.time.TimeSeriesDataItem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TimeSeriesChartsTransducerer extends QuantitiesAttributesChartsTransducerer { + private static Logger logger = LoggerFactory.getLogger(TimeSeriesChartsTransducerer.class); + @Override protected void setInputParameters() { List templates = new ArrayList(); @@ -60,13 +63,13 @@ public class TimeSeriesChartsTransducerer extends QuantitiesAttributesChartsTran TimeAnalyzer analyzer = new TimeAnalyzer(); sampleTime = analyzer.string2Date(timel); timepattern = analyzer.getPattern(); - AnalysisLogger.getLogger().debug("TimeSeriesChart: Time pattern: " + timepattern); + logger.debug("TimeSeriesChart: Time pattern: " + timepattern); formatter = new SimpleDateFormat(timepattern); } return formatter.parse(timel); } catch (ParseException e) { - AnalysisLogger.getLogger().debug("Error parsing date " + timel + " using pattern " + timepattern); + logger.debug("Error parsing date " + timel + " using pattern " + timepattern); return null; } } @@ -108,12 +111,12 @@ public class TimeSeriesChartsTransducerer extends QuantitiesAttributesChartsTran else indicesToTake = Operations.uniformIntegerSampling(0, nrows - 1, nrows); - AnalysisLogger.getLogger().debug("TimeSeriesChartsTransducerer: uniform sampling - taking " + indicesToTake.length + " over " + nrows); + logger.debug("TimeSeriesChartsTransducerer: uniform sampling - taking " + indicesToTake.length + " over " + nrows); for (int i = 0; i < indicesToTake.length; i++) { Object row = rows.get(indicesToTake[i]); Object[] array = (Object[]) row; - // AnalysisLogger.getLogger().debug("TimeSeriesChartsTransducerer: "+Arrays.toString(array)); + // logger.debug("TimeSeriesChartsTransducerer: "+Arrays.toString(array)); Double q = null; Date timeD = null; String timel = "" + array[array.length - 1]; @@ -121,7 +124,7 @@ public class TimeSeriesChartsTransducerer extends QuantitiesAttributesChartsTran q = Double.parseDouble("" + array[array.length - 2]); timeD = getTime(timel); } catch (Exception e) { - AnalysisLogger.getLogger().debug("TimeSeriesChartsTransducerer: warning skipping value " + q + "," + timel); + logger.debug("TimeSeriesChartsTransducerer: warning skipping value " + q + "," + timel); } if (q != null && timeD != null) { @@ -130,7 +133,7 @@ public class TimeSeriesChartsTransducerer extends QuantitiesAttributesChartsTran if (item != null) { double prevquant = (Double) item.getValue(); q = prevquant + q; - AnalysisLogger.getLogger().debug("TimeSeriesChartsTransducerer: a previous quantity was found for time " + timel + " : " + prevquant + " setting to " + (prevquant + q)); + logger.debug("TimeSeriesChartsTransducerer: a previous quantity was found for time " + timel + " : " + prevquant + " setting to " + (prevquant + q)); item.setValue(q); } else series.add(ms, q); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/HspecDiscrepanciesCalculator.java b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/HspecDiscrepanciesCalculator.java index 2f358d4..3d6e1d0 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/HspecDiscrepanciesCalculator.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/HspecDiscrepanciesCalculator.java @@ -3,9 +3,10 @@ package org.gcube.dataanalysis.ecoengine.utils; import java.math.BigInteger; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -14,6 +15,8 @@ import org.hibernate.SessionFactory; */ public class HspecDiscrepanciesCalculator { + private static Logger logger = LoggerFactory.getLogger(HspecDiscrepanciesCalculator.class); + private BigInteger numOfElements; private int errorCounter; //connection setup @@ -42,11 +45,10 @@ public class HspecDiscrepanciesCalculator { //init connections public HspecDiscrepanciesCalculator(AlgorithmConfiguration config) throws Exception { - AnalysisLogger.setLogger(config.getConfigPath() + LogFile); referencedbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); destinationdbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); - AnalysisLogger.getLogger().debug("OriginalDB initialized"); + logger.debug("OriginalDB initialized"); } //counts the elements in a table @@ -54,7 +56,7 @@ public class HspecDiscrepanciesCalculator { { BigInteger count = BigInteger.ZERO; String countingQuery = "select count(*) from "+tablename; - AnalysisLogger.getLogger().debug("Getting DB elements by this query: "+countingQuery); + logger.debug("Getting DB elements by this query: "+countingQuery); List result = DatabaseFactory.executeSQLQuery(countingQuery, session); count = (BigInteger) result.get(0); return count; @@ -64,7 +66,7 @@ public class HspecDiscrepanciesCalculator { //takes a chunk of elements from the database, belonging to the set of 170 selected species public List takeChunkOfDestElements(String tablename,String selectedColumns,String criteria, SessionFactory session,String whereclause) { String query = String.format(selectDestElementsQuery,selectedColumns,tablename,whereclause); -// AnalysisLogger.getLogger().debug("takeChunkOfDestinationElements-> executing query on DB: " + query); +// logger.debug("takeChunkOfDestinationElements-> executing query on DB: " + query); List results = DatabaseFactory.executeSQLQuery(query, session); return results; } @@ -72,7 +74,7 @@ public class HspecDiscrepanciesCalculator { //takes a chunk of elements from the database, belonging to the set of 170 selected species public List takeChunkOfElements(String tablename,String selectedColumns,String criteria, int limit, int offset, SessionFactory session) { String query = String.format(selectElementsQuery,selectedColumns,tablename,criteria)+ " limit " + limit + " offset " + offset; - AnalysisLogger.getLogger().debug("takeChunkOfElements-> executing query on DB: " + query); + logger.debug("takeChunkOfElements-> executing query on DB: " + query); List results = DatabaseFactory.executeSQLQuery(query, session); return results; } @@ -120,7 +122,7 @@ public class HspecDiscrepanciesCalculator { // take the number of elements numOfElements = countElements(analyzedTable, destinationdbConnection); - AnalysisLogger.getLogger().debug("Remote DB contains " + numOfElements + " elements."); + logger.debug("Remote DB contains " + numOfElements + " elements."); int maxNumber = numOfElements.intValue(); int numOfChunks = maxNumber / chunkSize; if ((maxNumber % chunkSize) > 0) { @@ -157,9 +159,9 @@ public class HspecDiscrepanciesCalculator { columns = destrow.length; } else{ - AnalysisLogger.getLogger().debug("ERROR - COULD NOT FIND "+refrow+" ON DESTINATION TABLE"); + logger.debug("ERROR - COULD NOT FIND "+refrow+" ON DESTINATION TABLE"); for (int k=0;kThreshold){ errorCounter++; equal = false; - AnalysisLogger.getLogger().debug("ERROR - DISCREPANCY AT NUMBERS COMPARISON: "+refelem+" vs "+destelem); + logger.debug("ERROR - DISCREPANCY AT NUMBERS COMPARISON: "+refelem+" vs "+destelem); } } else if (!refelem.equals(destelem)){ errorCounter++; equal = false; - AnalysisLogger.getLogger().debug("ERROR - DISCREPANCY AT STRING COMPARISON: "+refelem+" vs "+destelem); + logger.debug("ERROR - DISCREPANCY AT STRING COMPARISON: "+refelem+" vs "+destelem); } if (!equal) break; @@ -195,11 +197,11 @@ public class HspecDiscrepanciesCalculator { if (!equal) break; else - AnalysisLogger.getLogger().debug("CHUNK NUMBER "+i+" of "+numOfChunks+" OK!"); + logger.debug("CHUNK NUMBER "+i+" of "+numOfChunks+" OK!"); } long t1 = System.currentTimeMillis(); - AnalysisLogger.getLogger().debug("ELAPSED TIME: " + (t1-t0) + " ms"); + logger.debug("ELAPSED TIME: " + (t1-t0) + " ms"); //close connections referencedbConnection.close(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java index d4e9253..a96dded 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TablesDiscrepanciesCalculator.java @@ -3,9 +3,10 @@ package org.gcube.dataanalysis.ecoengine.utils; import java.math.BigInteger; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -14,6 +15,8 @@ import org.hibernate.SessionFactory; */ public class TablesDiscrepanciesCalculator { + private static Logger logger = LoggerFactory.getLogger(TablesDiscrepanciesCalculator.class); + private BigInteger numOfElements; private int errorCounter; //connection setup @@ -41,11 +44,10 @@ public class TablesDiscrepanciesCalculator { //init connections public TablesDiscrepanciesCalculator(AlgorithmConfiguration config) throws Exception { - AnalysisLogger.setLogger(config.getConfigPath() + LogFile); referencedbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); - AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + logger.debug("ReferenceDB initialized"); destinationdbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config); - AnalysisLogger.getLogger().debug("OriginalDB initialized"); + logger.debug("OriginalDB initialized"); } //counts the elements in a table @@ -53,7 +55,7 @@ public class TablesDiscrepanciesCalculator { { BigInteger count = BigInteger.ZERO; String countingQuery = "select count(*) from "+tablename; - AnalysisLogger.getLogger().debug("Getting DB elements by this query: "+countingQuery); + logger.debug("Getting DB elements by this query: "+countingQuery); List result = DatabaseFactory.executeSQLQuery(countingQuery, session); count = (BigInteger) result.get(0); return count; @@ -63,7 +65,7 @@ public class TablesDiscrepanciesCalculator { //takes a chunk of elements from the database, belonging to the set of 170 selected species public List takeChunkOfElements(String tablename,String selectedColumns,String criteria, int limit, int offset, SessionFactory session) { String query = String.format(selectElementsQuery,selectedColumns,tablename,criteria)+ " limit " + limit + " offset " + offset; - AnalysisLogger.getLogger().debug("takeChunkOfElements-> executing query on DB: " + query); + logger.debug("takeChunkOfElements-> executing query on DB: " + query); List results = DatabaseFactory.executeSQLQuery(query, session); return results; } @@ -110,7 +112,7 @@ public class TablesDiscrepanciesCalculator { // take the number of elements numOfElements = countElements(analyzedTable, destinationdbConnection); - AnalysisLogger.getLogger().debug("Remote DB contains " + numOfElements + " elements."); + logger.debug("Remote DB contains " + numOfElements + " elements."); int maxNumber = numOfElements.intValue(); int numOfChunks = maxNumber / chunkSize; if ((maxNumber % chunkSize) > 0) { @@ -141,13 +143,13 @@ public class TablesDiscrepanciesCalculator { if (Math.abs(d-isNumber(destelem))>Threshold){ errorCounter++; equal = false; - AnalysisLogger.getLogger().debug("ERROR - DISCREPANCY AT NUMBERS COMPARISON: "+refelem+" vs "+destelem); + logger.debug("ERROR - DISCREPANCY AT NUMBERS COMPARISON: "+refelem+" vs "+destelem); } } else if (!refelem.equals(destelem)){ errorCounter++; equal = false; - AnalysisLogger.getLogger().debug("ERROR - DISCREPANCY AT STRING COMPARISON: "+refelem+" vs "+destelem); + logger.debug("ERROR - DISCREPANCY AT STRING COMPARISON: "+refelem+" vs "+destelem); } if (!equal) break; @@ -160,11 +162,11 @@ public class TablesDiscrepanciesCalculator { if (!equal) break; else - AnalysisLogger.getLogger().debug("CHUNK NUMBER "+i+" OK!"); + logger.debug("CHUNK NUMBER "+i+" OK!"); } long t1 = System.currentTimeMillis(); - AnalysisLogger.getLogger().debug("ELAPSED TIME: " + (t1-t0) + " ms"); + logger.debug("ELAPSED TIME: " + (t1-t0) + " ms"); //close connections referencedbConnection.close(); diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TrainingSetsGenerator.java b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TrainingSetsGenerator.java index ad94cf9..c848919 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TrainingSetsGenerator.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/TrainingSetsGenerator.java @@ -1,14 +1,16 @@ package org.gcube.dataanalysis.ecoengine.utils; -import java.util.HashMap; import java.util.List; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.hibernate.SessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TrainingSetsGenerator { + private static Logger logger = LoggerFactory.getLogger(TrainingSetsGenerator.class); + static String getProbabilititesQuery = "select count(*) as distribprob from %1$s as a join %2$s as b on a.%3$s=b.%4$s and b.%5$s %6$s %7$s"; static String getNumberOfElementsQuery = "select count(*) from %1$s"; @@ -46,7 +48,7 @@ public class TrainingSetsGenerator { deleteTable(testTable, trainingTable); String testSetQuery = String.format(getRandomElements, table, testTable, numberOfElements); - AnalysisLogger.getLogger().trace("TestSet Creation Query: "+testSetQuery); + logger.trace("TestSet Creation Query: "+testSetQuery); try { DatabaseFactory.executeSQLUpdate(testSetQuery, connection); } catch (Exception e) { @@ -54,13 +56,13 @@ public class TrainingSetsGenerator { } String trainingSetQuery = String.format(getOtherElements, table, testTable, key, trainingTable); - AnalysisLogger.getLogger().trace("TrainingSet Creation Query: "+trainingSetQuery); + logger.trace("TrainingSet Creation Query: "+trainingSetQuery); try { DatabaseFactory.executeSQLUpdate(trainingSetQuery, connection); } catch (Exception e) { e.printStackTrace(); } - AnalysisLogger.getLogger().trace("DONE!"); + logger.trace("DONE!"); } public void generate(AlgorithmConfiguration config) throws Exception { @@ -73,7 +75,6 @@ public class TrainingSetsGenerator { SessionFactory connection; public void init(AlgorithmConfiguration config) throws Exception { - AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); // init db connection connection = AlgorithmConfiguration.getConnectionFromConfig(config); } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/ZipTools.java b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/ZipTools.java index 9513168..70358d8 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/utils/ZipTools.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/utils/ZipTools.java @@ -1,21 +1,21 @@ package org.gcube.dataanalysis.ecoengine.utils; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.util.Date; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; -import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ZipTools { - public static void unZip(String zipFile, String outputFolder) throws Exception { + private static Logger logger = LoggerFactory.getLogger(ZipTools.class); + + public static void unZip(String zipFile, String outputFolder) throws Exception { byte[] buffer = new byte[1024]; @@ -37,21 +37,21 @@ public class ZipTools { String fileName = ze.getName(); File newFile = new File(outputFolder + File.separator + fileName); - AnalysisLogger.getLogger().debug("Unzipping : " + newFile.getAbsoluteFile()); + logger.debug("Unzipping : " + newFile.getAbsoluteFile()); if (ze.isDirectory()) new File(outputFolder,ze.getName()).mkdir(); - + else{ - new File(newFile.getParent()).mkdirs(); - FileOutputStream fos = new FileOutputStream(newFile); + new File(newFile.getParent()).mkdirs(); + FileOutputStream fos = new FileOutputStream(newFile); - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } - fos.close(); + fos.close(); } ze = zis.getNextEntry(); } @@ -59,54 +59,54 @@ public class ZipTools { zis.closeEntry(); zis.close(); - AnalysisLogger.getLogger().debug("Unzip done"); + logger.debug("Unzip done"); } catch (IOException ex) { ex.printStackTrace(); } } - - static public void zipFolder(String srcFolder, String destZipFile) throws Exception { - ZipOutputStream zip = null; - FileOutputStream fileWriter = null; - fileWriter = new FileOutputStream(destZipFile); - zip = new ZipOutputStream(fileWriter); + static public void zipFolder(String srcFolder, String destZipFile) throws Exception { + ZipOutputStream zip = null; + FileOutputStream fileWriter = null; - addFolderToZip("", srcFolder, zip); - zip.flush(); - zip.close(); - } + fileWriter = new FileOutputStream(destZipFile); + zip = new ZipOutputStream(fileWriter); - static private void addFileToZip(String path, String srcFile, ZipOutputStream zip) - throws Exception { + addFolderToZip("", srcFolder, zip); + zip.flush(); + zip.close(); + } - File folder = new File(srcFile); - if (folder.isDirectory()) { - addFolderToZip(path, srcFile, zip); - } else { - byte[] buf = new byte[1024]; - int len; - FileInputStream in = new FileInputStream(srcFile); - zip.putNextEntry(new ZipEntry(path + "/" + folder.getName())); - while ((len = in.read(buf)) > 0) { - zip.write(buf, 0, len); - } - } - } + static private void addFileToZip(String path, String srcFile, ZipOutputStream zip) + throws Exception { - static private void addFolderToZip(String path, String srcFolder, ZipOutputStream zip) - throws Exception { - File folder = new File(srcFolder); + File folder = new File(srcFile); + if (folder.isDirectory()) { + addFolderToZip(path, srcFile, zip); + } else { + byte[] buf = new byte[1024]; + int len; + FileInputStream in = new FileInputStream(srcFile); + zip.putNextEntry(new ZipEntry(path + "/" + folder.getName())); + while ((len = in.read(buf)) > 0) { + zip.write(buf, 0, len); + } + } + } + + static private void addFolderToZip(String path, String srcFolder, ZipOutputStream zip) + throws Exception { + File folder = new File(srcFolder); + + for (String fileName : folder.list()) { + if (path.equals("")) { + addFileToZip(folder.getName(), srcFolder + "/" + fileName, zip); + } else { + addFileToZip(path + "/" + folder.getName(), srcFolder + "/" + fileName, zip); + } + } + } - for (String fileName : folder.list()) { - if (path.equals("")) { - addFileToZip(folder.getName(), srcFolder + "/" + fileName, zip); - } else { - addFileToZip(path + "/" + folder.getName(), srcFolder + "/" + fileName, zip); - } - } - } - }