diff --git a/etc/build.properties b/etc/build.properties index 43f3a7b..c817a81 100644 --- a/etc/build.properties +++ b/etc/build.properties @@ -1,4 +1,4 @@ name = EcologicalEngine -package = org.gcube +package = org.gcube.dataanalysis.ecologicalengine # Where the jar will be stored lib.dir = Dependencies/org.gcube.dataanalysis.ecologicalengine.jar diff --git a/src/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java b/src/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java index 38aef8e..195ae28 100644 --- a/src/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java +++ b/src/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java @@ -22,10 +22,28 @@ public class BioClimateAnalysis { private static String countHighProbabilityCells = "select count(*) from %1$s where probability>%2$s"; private static String countSeaCells = "select count(*) from %1$s as a join %2$s as b on a.oceanarea>0 and a.csquarecode=b.csquarecode and ((a.iceconannb.iceconann+%3$s) or " + "(a.salinitymeanb.salinitymean+%3$s) or (a.sstanmeanb.sstanmean+%3$s))"; private static String iceLeakage = "select count(*) from %1$s as a join %2$s as b on a.csquarecode=b.csquarecode and (a.iceconann0"; + private static String takeRangeOfDepths= "select distinct depthmin, max(depthmax) from %1$s group by depthmin order by depthmin"; + private static String countNumberOfSpecies= "count(*) from %1$s where depthmin<%2$s and depthmin>=%3$s and depthmax<%4$s and depthmax>=%5$s"; + + + + private static enum FIELD { + iceconann, sstanmean, salinitymean + }; + + private static String takeAvgSelection = "select avg(%1$s),%2$s from %3$s %4$s group by %2$s order by %2$s"; + private static String [] selectionCriteria = {"faoaream", "lme"}; + private static String [] timeseriesNames = {"Ice Conc.", "Sea Surface Temperature","Salinity"}; + private static String [] quantitiesNames= {FIELD.iceconann.name(), FIELD.sstanmean.name(),FIELD.salinitymean.name()}; + private static String [] selectionFilters= {"where faoaream>0", "where lme>0"}; + private static String [] criteriaNames = {"FaoArea", "LME"}; + + + // private static String takeSubHspec = "select %1$s from #CLAUSE# order by %1$s"; private static String meanVal = "select avg(%2$s) from %1$s where oceanarea>0"; // private static String meanVal = "select %2$s from %1$s where csquarecode = '1311:478:4'"; - + private String configPath; private String temporaryDirectory; @@ -34,36 +52,33 @@ public class BioClimateAnalysis { private String[] csquareTable; private String[] finalDistributionTable; - //hspec + // hspec private int[] highProbabilityCells; private double[] discrepancies; - //hcaf + // hcaf private double[] avgIce; private double[] avgSST; private double[] avgSalinity; - + private Evaluator eval; private float status; private boolean liveRender; boolean doHcafAn; boolean doHspecAn; - + private LexicalEngineConfiguration config; - - private static enum FIELD { - iceconann, sstanmean, salinitymean - }; - + static int width = 680; + static int height = 420; public static void main(String[] args) throws Exception { String configPath = "./cfg/"; String databaseUrl = "jdbc:postgresql://localhost/testdb"; String databaseUser = "gcube"; String databasePassword = "d4science2"; - BioClimateAnalysis bioClimate = new BioClimateAnalysis(configPath, configPath, databaseUrl, databaseUser, databasePassword,true); - bioClimate.produceGraphs2D(); + BioClimateAnalysis bioClimate = new BioClimateAnalysis(configPath, configPath, databaseUrl, databaseUser, databasePassword, true); + bioClimate.produceGraphs2D(); } private static String[] SERIES = { "High Probability Cells Trend (>0.8)", "Number of Changing Cells", "Reducing Ice Concentration Trend", "Average Discrepancy Between Distributions", "Average Trends", "Ice Concentration", "Sea Surface Temperature", "Salinity" }; @@ -89,55 +104,52 @@ public class BioClimateAnalysis { return producedImages; } - private void produceGraphs() throws Exception { + private void produceGraphs(String [] csquareTableNames,String[] hspecTableNames) throws Exception { producedImages = new ArrayList(); int numberOfTrends = highProbabilityCells.length; - + // create the datasets... DefaultCategoryDataset probabilityTrend = new DefaultCategoryDataset(); DefaultCategoryDataset discrepanciesTrend = new DefaultCategoryDataset(); - + DefaultCategoryDataset avgIceD = new DefaultCategoryDataset(); DefaultCategoryDataset avgSSTD = new DefaultCategoryDataset(); DefaultCategoryDataset avgSalinityD = new DefaultCategoryDataset(); for (int i = 0; i < numberOfTrends; i++) { - if (doHcafAn){ - avgIceD.addValue(avgIce[i], "Ice Conc.", csquareTable[i]); - avgSSTD.addValue(avgSST[i], "SST", csquareTable[i]); - avgSalinityD.addValue(avgSalinity[i], "Salinity", csquareTable[i]); + if (doHcafAn) { + avgIceD.addValue(avgIce[i], "Ice Conc.", csquareTableNames[i]); + avgSSTD.addValue(avgSST[i], "SST", csquareTableNames[i]); + avgSalinityD.addValue(avgSalinity[i], "Salinity", csquareTableNames[i]); } - if (doHspecAn){ - probabilityTrend.addValue(highProbabilityCells[i], "Number Of Cells", finalDistributionTable[i]); + if (doHspecAn) { + probabilityTrend.addValue(highProbabilityCells[i], "Number Of Cells", hspecTableNames[i]); if (i > 0) { discrepanciesTrend.addValue(discrepancies[i], "Mean Discrepancy Respect to Prev. Distrib.", finalDistributionTable[i]); } } } - int width = 680; - int height = 420; - if (doHspecAn){ - - + + if (doHspecAn) { + double min = Operations.getMin(discrepancies); discrepancies[0] = min; - - - if (liveRender){ - BioClimateGraph lineg1 = new BioClimateGraph(SERIES[0], Operations.getMax(highProbabilityCells), Operations.getMin(highProbabilityCells)); + + if (liveRender) { + BioClimateGraph lineg1 = new BioClimateGraph(SERIES[0], Operations.getMax(highProbabilityCells), Operations.getMin(highProbabilityCells)); BioClimateGraph lineg4 = new BioClimateGraph(SERIES[3], Operations.getMax(discrepancies), min); lineg4.render(discrepanciesTrend); lineg1.render(probabilityTrend); } - + producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, probabilityTrend, SERIES[0], Operations.getMax(highProbabilityCells), Operations.getMin(highProbabilityCells))); producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, discrepanciesTrend, SERIES[3], Operations.getMax(discrepancies), min)); } - if (doHcafAn){ - - if (liveRender){ + if (doHcafAn) { + + if (liveRender) { BioClimateGraph lineg6 = new BioClimateGraph(SERIES[5], Operations.getMax(avgIce), Operations.getMin(avgIce)); BioClimateGraph lineg7 = new BioClimateGraph(SERIES[6], Operations.getMax(avgSST), Operations.getMin(avgSST)); BioClimateGraph lineg8 = new BioClimateGraph(SERIES[7], Operations.getMax(avgSalinity), Operations.getMin(avgSalinity)); @@ -145,82 +157,175 @@ public class BioClimateAnalysis { lineg7.render(avgSSTD); lineg8.render(avgSalinityD); } - + producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, avgIceD, SERIES[5], Operations.getMax(avgIce), Operations.getMin(avgIce))); - producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, avgSSTD, SERIES[6], Operations.getMax(avgSST), Operations.getMin(avgSST))); + producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, avgSSTD, SERIES[6], Operations.getMax(avgSST), Operations.getMin(avgSST))); producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, avgSalinityD, SERIES[7], Operations.getMax(avgSalinity), Operations.getMin(avgSalinity))); - + } - + AnalysisLogger.getLogger().trace("Produced All Images"); - + } + public void hcafEvolutionAnalysis(String[] hcafTable,String[] hcafTableNames) throws Exception { + globalEvolutionAnalysis(hcafTable, null, hcafTableNames, null, null, null); + } + + public void hspecEvolutionAnalysis(String[] hspecTables, String[] hspecTableNames, String probabilityColumn, String csquareColumn) throws Exception { + globalEvolutionAnalysis(null, hspecTables, null, hspecTableNames, probabilityColumn, csquareColumn); + } + + private List producedCharts; - - public void hcafEvolutionAnalysis(String[] hcafTable) throws Exception{ - evolutionAnalysis(hcafTable, null, null, null); - } - - public void hspecEvolutionAnalysis(String[] hspecTables, String probabilityColumn, String csquareColumn) throws Exception{ - evolutionAnalysis(null, hspecTables, probabilityColumn, csquareColumn); + public void produceCharts(HashMap> GeoMap,String[] hcafTablesNames){ + //produce a char for each feature + producedCharts = new ArrayList(); + for (String featurename:GeoMap.keySet()){ + DefaultCategoryDataset chart = new DefaultCategoryDataset(); + HashMap timeseries = GeoMap.get(featurename); + double absmax = -Double.MAX_VALUE; + double absmin = Double.MAX_VALUE; + for (String timeserie:timeseries.keySet()){ + double[] points = timeseries.get(timeserie); + for (int i=0;iabsmax) + absmax = points[i]; + if (points [i] 0) { + + //a map for each feature. each sub map contains a trend for faoaream, lme etc. + HashMap> GeoMap= new HashMap>(); + + float statusstep = 80f / (float) numbOfTables; + //for each table + for (int i = 0; i < numbOfTables; i++) { + //for each criterion to apply: fao area, lme etc. + for (int j=0;j quantityCriterion = DatabaseFactory.executeSQLQuery(query, referencedbConnection); + //for each row + for (Object element: quantityCriterion){ + Object [] row = (Object[]) element; + //take avg value + double value = (row[0]==null)?0:Double.parseDouble(""+row[0]); + //take code for criterion + String code = ""+row[1]; + + String chartName = timeseriesNames[k]+" for "+criteriaNames[j]+"_"+code; + //put the code and the value in the timeseries associated to the feature name + HashMap submap = GeoMap.get(chartName); + if (submap==null){ + submap = new HashMap(); + GeoMap.put(chartName, submap); + } + + String timeseries = criteriaNames[j]+"_"+code; + double [] elements = submap.get(timeseries); + if (elements==null){ + elements = new double[numbOfTables]; + submap.put(timeseries, elements); + } + elements[i] = value; + } + } + } + status = status + statusstep; + } + + status = 80f; + produceCharts(GeoMap,hcafTableNames); + } + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + status = 100f; + referencedbConnection.close(); + } + } + + public void globalEvolutionAnalysis(String[] hcafTable, String[] hspecTables,String[] hcafTablesNames, String [] hspecTableNames, String probabilityColumn, String csquareColumn) throws Exception { + try { + referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); + AnalysisLogger.getLogger().debug("ReferenceDB initialized"); + doHcafAn = true; + doHspecAn = true; + + if (hcafTable == null) + doHcafAn = false; + if (hspecTables == null) + doHspecAn = false; + + status = 0f; + this.csquareTable = hcafTable; + this.finalDistributionTable = hspecTables; + + int numbOfPoints = (hcafTable != null) ? hcafTable.length : hspecTables.length; + + highProbabilityCells = new int[numbOfPoints]; + discrepancies = new double[numbOfPoints]; + avgIce = new double[numbOfPoints]; + avgSST = new double[numbOfPoints]; + avgSalinity = new double[numbOfPoints]; + + float statusstep = 80f / (float) numbOfPoints; - for (int i = 0; i < numbOfPoints; i++) { if (doHspecAn) highProbabilityCells[i] = calcHighProbabilityCells(hspecTables[i], 0.8); - if (doHcafAn){ + if (doHcafAn) { avgIce[i] = avgValue(hcafTable[i], FIELD.iceconann.name()); avgSST[i] = avgValue(hcafTable[i], FIELD.sstanmean.name()); avgSalinity[i] = avgValue(hcafTable[i], FIELD.salinitymean.name()); - AnalysisLogger.getLogger().trace("(" + hcafTable[i] +"): " + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); + AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "): " + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); } - - + if (doHspecAn) { if (i == 0) { discrepancies[i] = 1.0; } else { -// OLD CALCULATION discrepancies[i] = MathFunctions.roundDecimal(calcDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5); + // OLD CALCULATION discrepancies[i] = MathFunctions.roundDecimal(calcDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5); discrepancies[i] = MathFunctions.roundDecimal(calcOverDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5); } - AnalysisLogger.getLogger().trace("(" + hspecTables[i] +"): DISCREPANCY " + discrepancies[i] +" HIGH PROB CELLS " + highProbabilityCells[i]); + AnalysisLogger.getLogger().trace("(" + hspecTables[i] + "): DISCREPANCY " + discrepancies[i] + " HIGH PROB CELLS " + highProbabilityCells[i]); } - -// AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "," + hspecTables[i] + "): HIGH PROB CELLS " + highProbabilityCells[i] + " DISCREPANCY " + discrepancies[i] + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); + + // AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "," + hspecTables[i] + "): HIGH PROB CELLS " + highProbabilityCells[i] + " DISCREPANCY " + discrepancies[i] + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); status = status + statusstep; } status = 80f; - produceGraphs(); + produceGraphs(hcafTablesNames,hspecTableNames); } catch (Exception e) { e.printStackTrace(); @@ -231,9 +336,8 @@ public class BioClimateAnalysis { } } - // init connections - public BioClimateAnalysis(String configPath, String persistencePath, String databaseURL, String databaseUserName, String databasePassword,boolean liveRender) throws Exception { + public BioClimateAnalysis(String configPath, String persistencePath, String databaseURL, String databaseUserName, String databasePassword, boolean liveRender) throws Exception { this.configPath = configPath; this.temporaryDirectory = persistencePath; if (!configPath.endsWith("/")) @@ -246,7 +350,7 @@ public class BioClimateAnalysis { config.setDatabaseURL(databaseURL); config.setDatabaseUserName(databaseUserName); config.setDatabasePassword(databasePassword); - + this.liveRender = liveRender; } @@ -254,10 +358,12 @@ public class BioClimateAnalysis { AnalysisLogger.getLogger().trace("Calculating High Prob Cells"); List countage = DatabaseFactory.executeSQLQuery(String.format(countHighProbabilityCells, hspec, probabilty), referencedbConnection); int count = Integer.parseInt("" + countage.get(0)); - AnalysisLogger.getLogger().trace("Calc High Prob Cells: "+count); + AnalysisLogger.getLogger().trace("Calc High Prob Cells: " + count); return count; } + + public double avgValue(String hcaf1, String field) throws Exception { List countage = DatabaseFactory.executeSQLQuery(String.format(meanVal, hcaf1, field), referencedbConnection); @@ -287,26 +393,26 @@ public class BioClimateAnalysis { } public double calcOverDiscrepancy(String configPath, String persistencePath, String firstTable, String secondTable, String probabilityColumnName, String csquareColumn, float comparisonThreshold) throws Exception { - - List nelementsQ1 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(firstTable),referencedbConnection); - int nelements = Integer.parseInt(""+nelementsQ1.get(0)); - AnalysisLogger.getLogger().trace("Number Of elements1: "+nelementsQ1); - List nelementsQ2 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(secondTable),referencedbConnection); - int nelements2 = Integer.parseInt(""+nelementsQ2.get(0)); - AnalysisLogger.getLogger().trace("Number Of elements2: "+nelementsQ1); - - List sumFirst = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(firstTable,probabilityColumnName),referencedbConnection); - double sum1 = Double.parseDouble(""+sumFirst.get(0)); - AnalysisLogger.getLogger().trace("Sum1: "+sum1); - - List sumSecond = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(secondTable,probabilityColumnName),referencedbConnection); - double sum2 = Double.parseDouble(""+sumSecond.get(0)); - AnalysisLogger.getLogger().trace("Sum2: "+sum1); - - double d = (double)(sum2-sum1)/(double)(nelements+nelements2); + + List nelementsQ1 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(firstTable), referencedbConnection); + int nelements = Integer.parseInt("" + nelementsQ1.get(0)); + AnalysisLogger.getLogger().trace("Number Of elements1: " + nelementsQ1); + List nelementsQ2 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(secondTable), referencedbConnection); + int nelements2 = Integer.parseInt("" + nelementsQ2.get(0)); + AnalysisLogger.getLogger().trace("Number Of elements2: " + nelementsQ1); + + List sumFirst = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(firstTable, probabilityColumnName), referencedbConnection); + double sum1 = Double.parseDouble("" + sumFirst.get(0)); + AnalysisLogger.getLogger().trace("Sum1: " + sum1); + + List sumSecond = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(secondTable, probabilityColumnName), referencedbConnection); + double sum2 = Double.parseDouble("" + sumSecond.get(0)); + AnalysisLogger.getLogger().trace("Sum2: " + sum1); + + double d = (double) (sum2 - sum1) / (double) (nelements + nelements2); return d; } - + public double calcDiscrepancy(String configPath, String persistencePath, String firstTable, String secondTable, String probabilityColumnName, String csquareColumn, float comparisonThreshold) throws Exception { AlgorithmConfiguration config = new AlgorithmConfiguration(); @@ -327,14 +433,13 @@ public class BioClimateAnalysis { config.setParam("ComparisonThreshold", "" + comparisonThreshold); - config.setParam("MaxSamples", ""+30000); - + config.setParam("MaxSamples", "" + 30000); + eval = EvaluatorsFactory.getEvaluators(config).get(0); HashMap out = eval.process(config); Double d = Double.parseDouble(out.get("MEAN")); return d; } - - + }