gCubeSystem
/
ecological-engine
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@51720 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2012-03-02 08:41:16 +00:00
parent 03a37a04c7
commit 872c1528fa
2 changed files with 211 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/build.properties
View File

@ -1,4 +1,4 @@
name = EcologicalEngine name = EcologicalEngine
package = org.gcube package = org.gcube.dataanalysis.ecologicalengine
# Where the jar will be stored # Where the jar will be stored
lib.dir = Dependencies/org.gcube.dataanalysis.ecologicalengine.jar lib.dir = Dependencies/org.gcube.dataanalysis.ecologicalengine.jar

217
src/org/gcube/dataanalysis/ecoengine/evaluation/bioclimate/BioClimateAnalysis.java
View File

@ -22,6 +22,24 @@ public class BioClimateAnalysis {
private static String countHighProbabilityCells = "select count(*) from %1$s where probability>%2$s"; private static String countHighProbabilityCells = "select count(*) from %1$s where probability>%2$s";
private static String countSeaCells = "select count(*) from %1$s as a join %2$s as b on a.oceanarea>0 and a.csquarecode=b.csquarecode and ((a.iceconann<b.iceconann-%3$s or a.iceconann>b.iceconann+%3$s) or " + "(a.salinitymean<b.salinitymean-%3$s or a.salinitymean>b.salinitymean+%3$s) or (a.sstanmean<b.sstanmean-%3$s or a.sstanmean>b.sstanmean+%3$s))"; private static String countSeaCells = "select count(*) from %1$s as a join %2$s as b on a.oceanarea>0 and a.csquarecode=b.csquarecode and ((a.iceconann<b.iceconann-%3$s or a.iceconann>b.iceconann+%3$s) or " + "(a.salinitymean<b.salinitymean-%3$s or a.salinitymean>b.salinitymean+%3$s) or (a.sstanmean<b.sstanmean-%3$s or a.sstanmean>b.sstanmean+%3$s))";
private static String iceLeakage = "select count(*) from %1$s as a join %2$s as b on a.csquarecode=b.csquarecode and (a.iceconann<b.iceconann) and a.oceanarea>0"; private static String iceLeakage = "select count(*) from %1$s as a join %2$s as b on a.csquarecode=b.csquarecode and (a.iceconann<b.iceconann) and a.oceanarea>0";
private static String takeRangeOfDepths= "select distinct depthmin, max(depthmax) from %1$s group by depthmin order by depthmin";
private static String countNumberOfSpecies= "count(*) from %1$s where depthmin<%2$s and depthmin>=%3$s and depthmax<%4$s and depthmax>=%5$s";
private static enum FIELD {
iceconann, sstanmean, salinitymean
};
private static String takeAvgSelection = "select avg(%1$s),%2$s from %3$s %4$s group by %2$s order by %2$s";
private static String [] selectionCriteria = {"faoaream", "lme"};
private static String [] timeseriesNames = {"Ice Conc.", "Sea Surface Temperature","Salinity"};
private static String [] quantitiesNames= {FIELD.iceconann.name(), FIELD.sstanmean.name(),FIELD.salinitymean.name()};
private static String [] selectionFilters= {"where faoaream>0", "where lme>0"};
private static String [] criteriaNames = {"FaoArea", "LME"};
// private static String takeSubHspec = "select %1$s from #CLAUSE# order by %1$s";
private static String meanVal = "select avg(%2$s) from %1$s where oceanarea>0"; private static String meanVal = "select avg(%2$s) from %1$s where oceanarea>0";
// private static String meanVal = "select %2$s from %1$s where csquarecode = '1311:478:4'"; // private static String meanVal = "select %2$s from %1$s where csquarecode = '1311:478:4'";
@ -34,10 +52,10 @@ public class BioClimateAnalysis {
private String[] csquareTable; private String[] csquareTable;
private String[] finalDistributionTable; private String[] finalDistributionTable;
//hspec // hspec
private int[] highProbabilityCells; private int[] highProbabilityCells;
private double[] discrepancies; private double[] discrepancies;
//hcaf // hcaf
private double[] avgIce; private double[] avgIce;
private double[] avgSST; private double[] avgSST;
private double[] avgSalinity; private double[] avgSalinity;
@ -51,18 +69,15 @@ public class BioClimateAnalysis {
private LexicalEngineConfiguration config; private LexicalEngineConfiguration config;
private static enum FIELD { static int width = 680;
iceconann, sstanmean, salinitymean static int height = 420;
};
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
String configPath = "./cfg/"; String configPath = "./cfg/";
String databaseUrl = "jdbc:postgresql://localhost/testdb"; String databaseUrl = "jdbc:postgresql://localhost/testdb";
String databaseUser = "gcube"; String databaseUser = "gcube";
String databasePassword = "d4science2"; String databasePassword = "d4science2";
BioClimateAnalysis bioClimate = new BioClimateAnalysis(configPath, configPath, databaseUrl, databaseUser, databasePassword,true); BioClimateAnalysis bioClimate = new BioClimateAnalysis(configPath, configPath, databaseUrl, databaseUser, databasePassword, true);
bioClimate.produceGraphs2D(); bioClimate.produceGraphs2D();
} }
@ -89,7 +104,7 @@ public class BioClimateAnalysis {
return producedImages; return producedImages;
} }
private void produceGraphs() throws Exception { private void produceGraphs(String [] csquareTableNames,String[] hspecTableNames) throws Exception {
producedImages = new ArrayList<Image>(); producedImages = new ArrayList<Image>();
int numberOfTrends = highProbabilityCells.length; int numberOfTrends = highProbabilityCells.length;
@ -102,29 +117,26 @@ public class BioClimateAnalysis {
DefaultCategoryDataset avgSalinityD = new DefaultCategoryDataset(); DefaultCategoryDataset avgSalinityD = new DefaultCategoryDataset();
for (int i = 0; i < numberOfTrends; i++) { for (int i = 0; i < numberOfTrends; i++) {
if (doHcafAn){ if (doHcafAn) {
avgIceD.addValue(avgIce[i], "Ice Conc.", csquareTable[i]); avgIceD.addValue(avgIce[i], "Ice Conc.", csquareTableNames[i]);
avgSSTD.addValue(avgSST[i], "SST", csquareTable[i]); avgSSTD.addValue(avgSST[i], "SST", csquareTableNames[i]);
avgSalinityD.addValue(avgSalinity[i], "Salinity", csquareTable[i]); avgSalinityD.addValue(avgSalinity[i], "Salinity", csquareTableNames[i]);
} }
if (doHspecAn){ if (doHspecAn) {
probabilityTrend.addValue(highProbabilityCells[i], "Number Of Cells", finalDistributionTable[i]); probabilityTrend.addValue(highProbabilityCells[i], "Number Of Cells", hspecTableNames[i]);
if (i > 0) { if (i > 0) {
discrepanciesTrend.addValue(discrepancies[i], "Mean Discrepancy Respect to Prev. Distrib.", finalDistributionTable[i]); discrepanciesTrend.addValue(discrepancies[i], "Mean Discrepancy Respect to Prev. Distrib.", finalDistributionTable[i]);
} }
} }
} }
int width = 680;
int height = 420;
if (doHspecAn){
if (doHspecAn) {
double min = Operations.getMin(discrepancies); double min = Operations.getMin(discrepancies);
discrepancies[0] = min; discrepancies[0] = min;
if (liveRender) {
if (liveRender){
BioClimateGraph lineg1 = new BioClimateGraph(SERIES[0], Operations.getMax(highProbabilityCells), Operations.getMin(highProbabilityCells)); BioClimateGraph lineg1 = new BioClimateGraph(SERIES[0], Operations.getMax(highProbabilityCells), Operations.getMin(highProbabilityCells));
BioClimateGraph lineg4 = new BioClimateGraph(SERIES[3], Operations.getMax(discrepancies), min); BioClimateGraph lineg4 = new BioClimateGraph(SERIES[3], Operations.getMax(discrepancies), min);
lineg4.render(discrepanciesTrend); lineg4.render(discrepanciesTrend);
@ -135,9 +147,9 @@ public class BioClimateAnalysis {
producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, discrepanciesTrend, SERIES[3], Operations.getMax(discrepancies), min)); producedImages.add(BioClimateGraph.renderStaticImgObject(width, height, discrepanciesTrend, SERIES[3], Operations.getMax(discrepancies), min));
} }
if (doHcafAn){ if (doHcafAn) {
if (liveRender){ if (liveRender) {
BioClimateGraph lineg6 = new BioClimateGraph(SERIES[5], Operations.getMax(avgIce), Operations.getMin(avgIce)); BioClimateGraph lineg6 = new BioClimateGraph(SERIES[5], Operations.getMax(avgIce), Operations.getMin(avgIce));
BioClimateGraph lineg7 = new BioClimateGraph(SERIES[6], Operations.getMax(avgSST), Operations.getMin(avgSST)); BioClimateGraph lineg7 = new BioClimateGraph(SERIES[6], Operations.getMax(avgSST), Operations.getMin(avgSST));
BioClimateGraph lineg8 = new BioClimateGraph(SERIES[7], Operations.getMax(avgSalinity), Operations.getMin(avgSalinity)); BioClimateGraph lineg8 = new BioClimateGraph(SERIES[7], Operations.getMax(avgSalinity), Operations.getMin(avgSalinity));
@ -156,17 +168,112 @@ public class BioClimateAnalysis {
} }
public void hcafEvolutionAnalysis(String[] hcafTable,String[] hcafTableNames) throws Exception {
globalEvolutionAnalysis(hcafTable, null, hcafTableNames, null, null, null);
public void hcafEvolutionAnalysis(String[] hcafTable) throws Exception{
evolutionAnalysis(hcafTable, null, null, null);
} }
public void hspecEvolutionAnalysis(String[] hspecTables, String probabilityColumn, String csquareColumn) throws Exception{ public void hspecEvolutionAnalysis(String[] hspecTables, String[] hspecTableNames, String probabilityColumn, String csquareColumn) throws Exception {
evolutionAnalysis(null, hspecTables, probabilityColumn, csquareColumn); globalEvolutionAnalysis(null, hspecTables, null, hspecTableNames, probabilityColumn, csquareColumn);
} }
public void evolutionAnalysis(String[] hcafTable, String[] hspecTables, String probabilityColumn, String csquareColumn) throws Exception { private List<Image> producedCharts;
public void produceCharts(HashMap<String,HashMap<String,double []>> GeoMap,String[] hcafTablesNames){
//produce a char for each feature
producedCharts = new ArrayList<Image>();
for (String featurename:GeoMap.keySet()){
DefaultCategoryDataset chart = new DefaultCategoryDataset();
HashMap<String,double []> timeseries = GeoMap.get(featurename);
double absmax = -Double.MAX_VALUE;
double absmin = Double.MAX_VALUE;
for (String timeserie:timeseries.keySet()){
double[] points = timeseries.get(timeserie);
for (int i=0;i<points.length;i++){
if (points [i]>absmax)
absmax = points[i];
if (points [i]<absmin)
absmin = points[i];
chart.addValue(points[i], timeserie,hcafTablesNames[i]);
}
}
if (liveRender) {
BioClimateGraph lineg1 = new BioClimateGraph(featurename, absmax, absmin);
lineg1.render(chart);
}
producedCharts.add(BioClimateGraph.renderStaticImgObject(width, height, chart, featurename, absmax, absmin));
}
}
public void geographicEvolutionAnalysis(String[] hcafTable,String []hcafTableNames) throws Exception {
try {
referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config);
AnalysisLogger.getLogger().debug("ReferenceDB initialized");
doHcafAn = true;
doHspecAn = true;
status = 0f;
this.csquareTable = hcafTable;
int numbOfTables = (hcafTable != null) ? hcafTable.length : 0;
if (numbOfTables > 0) {
//a map for each feature. each sub map contains a trend for faoaream, lme etc.
HashMap<String,HashMap<String,double []>> GeoMap= new HashMap<String, HashMap<String,double[]>>();
float statusstep = 80f / (float) numbOfTables;
//for each table
for (int i = 0; i < numbOfTables; i++) {
//for each criterion to apply: fao area, lme etc.
for (int j=0;j<criteriaNames.length;j++){
//for each quantity to display: ice concentration
for (int k=0;k<quantitiesNames.length;k++){
String query = String.format(takeAvgSelection, quantitiesNames[k],selectionCriteria[j],hcafTable[i],selectionFilters[j]);
AnalysisLogger.getLogger().debug("Query to be executed : "+query);
//take couples (avg,code)
List<Object> quantityCriterion = DatabaseFactory.executeSQLQuery(query, referencedbConnection);
//for each row
for (Object element: quantityCriterion){
Object [] row = (Object[]) element;
//take avg value
double value = (row[0]==null)?0:Double.parseDouble(""+row[0]);
//take code for criterion
String code = ""+row[1];
String chartName = timeseriesNames[k]+" for "+criteriaNames[j]+"_"+code;
//put the code and the value in the timeseries associated to the feature name
HashMap<String,double []> submap = GeoMap.get(chartName);
if (submap==null){
submap = new HashMap<String, double[]>();
GeoMap.put(chartName, submap);
}
String timeseries = criteriaNames[j]+"_"+code;
double [] elements = submap.get(timeseries);
if (elements==null){
elements = new double[numbOfTables];
submap.put(timeseries, elements);
}
elements[i] = value;
}
}
}
status = status + statusstep;
}
status = 80f;
produceCharts(GeoMap,hcafTableNames);
}
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
status = 100f;
referencedbConnection.close();
}
}
public void globalEvolutionAnalysis(String[] hcafTable, String[] hspecTables,String[] hcafTablesNames, String [] hspecTableNames, String probabilityColumn, String csquareColumn) throws Exception {
try { try {
referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config); referencedbConnection = DatabaseFactory.initDBConnection(configPath + AlgorithmConfiguration.defaultConnectionFile, config);
AnalysisLogger.getLogger().debug("ReferenceDB initialized"); AnalysisLogger.getLogger().debug("ReferenceDB initialized");
@ -182,7 +289,7 @@ public class BioClimateAnalysis {
this.csquareTable = hcafTable; this.csquareTable = hcafTable;
this.finalDistributionTable = hspecTables; this.finalDistributionTable = hspecTables;
int numbOfPoints = (hcafTable!=null)? hcafTable.length:hspecTables.length; int numbOfPoints = (hcafTable != null) ? hcafTable.length : hspecTables.length;
highProbabilityCells = new int[numbOfPoints]; highProbabilityCells = new int[numbOfPoints];
discrepancies = new double[numbOfPoints]; discrepancies = new double[numbOfPoints];
@ -192,35 +299,33 @@ public class BioClimateAnalysis {
float statusstep = 80f / (float) numbOfPoints; float statusstep = 80f / (float) numbOfPoints;
for (int i = 0; i < numbOfPoints; i++) { for (int i = 0; i < numbOfPoints; i++) {
if (doHspecAn) if (doHspecAn)
highProbabilityCells[i] = calcHighProbabilityCells(hspecTables[i], 0.8); highProbabilityCells[i] = calcHighProbabilityCells(hspecTables[i], 0.8);
if (doHcafAn){ if (doHcafAn) {
avgIce[i] = avgValue(hcafTable[i], FIELD.iceconann.name()); avgIce[i] = avgValue(hcafTable[i], FIELD.iceconann.name());
avgSST[i] = avgValue(hcafTable[i], FIELD.sstanmean.name()); avgSST[i] = avgValue(hcafTable[i], FIELD.sstanmean.name());
avgSalinity[i] = avgValue(hcafTable[i], FIELD.salinitymean.name()); avgSalinity[i] = avgValue(hcafTable[i], FIELD.salinitymean.name());
AnalysisLogger.getLogger().trace("(" + hcafTable[i] +"): " + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "): " + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]);
} }
if (doHspecAn) { if (doHspecAn) {
if (i == 0) { if (i == 0) {
discrepancies[i] = 1.0; discrepancies[i] = 1.0;
} else { } else {
// OLD CALCULATION discrepancies[i] = MathFunctions.roundDecimal(calcDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5); // OLD CALCULATION discrepancies[i] = MathFunctions.roundDecimal(calcDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5);
discrepancies[i] = MathFunctions.roundDecimal(calcOverDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5); discrepancies[i] = MathFunctions.roundDecimal(calcOverDiscrepancy(configPath, temporaryDirectory, hspecTables[i], hspecTables[i - 1], probabilityColumn, csquareColumn, 0.1f), 5);
} }
AnalysisLogger.getLogger().trace("(" + hspecTables[i] +"): DISCREPANCY " + discrepancies[i] +" HIGH PROB CELLS " + highProbabilityCells[i]); AnalysisLogger.getLogger().trace("(" + hspecTables[i] + "): DISCREPANCY " + discrepancies[i] + " HIGH PROB CELLS " + highProbabilityCells[i]);
} }
// AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "," + hspecTables[i] + "): HIGH PROB CELLS " + highProbabilityCells[i] + " DISCREPANCY " + discrepancies[i] + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]); // AnalysisLogger.getLogger().trace("(" + hcafTable[i] + "," + hspecTables[i] + "): HIGH PROB CELLS " + highProbabilityCells[i] + " DISCREPANCY " + discrepancies[i] + " ICE " + avgIce[i] + " SST " + avgSST[i] + " SAL " + avgSalinity[i]);
status = status + statusstep; status = status + statusstep;
} }
status = 80f; status = 80f;
produceGraphs(); produceGraphs(hcafTablesNames,hspecTableNames);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
@ -231,9 +336,8 @@ public class BioClimateAnalysis {
} }
} }
// init connections // init connections
public BioClimateAnalysis(String configPath, String persistencePath, String databaseURL, String databaseUserName, String databasePassword,boolean liveRender) throws Exception { public BioClimateAnalysis(String configPath, String persistencePath, String databaseURL, String databaseUserName, String databasePassword, boolean liveRender) throws Exception {
this.configPath = configPath; this.configPath = configPath;
this.temporaryDirectory = persistencePath; this.temporaryDirectory = persistencePath;
if (!configPath.endsWith("/")) if (!configPath.endsWith("/"))
@ -254,10 +358,12 @@ public class BioClimateAnalysis {
AnalysisLogger.getLogger().trace("Calculating High Prob Cells"); AnalysisLogger.getLogger().trace("Calculating High Prob Cells");
List<Object> countage = DatabaseFactory.executeSQLQuery(String.format(countHighProbabilityCells, hspec, probabilty), referencedbConnection); List<Object> countage = DatabaseFactory.executeSQLQuery(String.format(countHighProbabilityCells, hspec, probabilty), referencedbConnection);
int count = Integer.parseInt("" + countage.get(0)); int count = Integer.parseInt("" + countage.get(0));
AnalysisLogger.getLogger().trace("Calc High Prob Cells: "+count); AnalysisLogger.getLogger().trace("Calc High Prob Cells: " + count);
return count; return count;
} }
public double avgValue(String hcaf1, String field) throws Exception { public double avgValue(String hcaf1, String field) throws Exception {
List<Object> countage = DatabaseFactory.executeSQLQuery(String.format(meanVal, hcaf1, field), referencedbConnection); List<Object> countage = DatabaseFactory.executeSQLQuery(String.format(meanVal, hcaf1, field), referencedbConnection);
@ -288,22 +394,22 @@ public class BioClimateAnalysis {
public double calcOverDiscrepancy(String configPath, String persistencePath, String firstTable, String secondTable, String probabilityColumnName, String csquareColumn, float comparisonThreshold) throws Exception { public double calcOverDiscrepancy(String configPath, String persistencePath, String firstTable, String secondTable, String probabilityColumnName, String csquareColumn, float comparisonThreshold) throws Exception {
List<Object> nelementsQ1 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(firstTable),referencedbConnection); List<Object> nelementsQ1 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(firstTable), referencedbConnection);
int nelements = Integer.parseInt(""+nelementsQ1.get(0)); int nelements = Integer.parseInt("" + nelementsQ1.get(0));
AnalysisLogger.getLogger().trace("Number Of elements1: "+nelementsQ1); AnalysisLogger.getLogger().trace("Number Of elements1: " + nelementsQ1);
List<Object> nelementsQ2 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(secondTable),referencedbConnection); List<Object> nelementsQ2 = DatabaseFactory.executeSQLQuery(DatabaseUtils.countElementsStatement(secondTable), referencedbConnection);
int nelements2 = Integer.parseInt(""+nelementsQ2.get(0)); int nelements2 = Integer.parseInt("" + nelementsQ2.get(0));
AnalysisLogger.getLogger().trace("Number Of elements2: "+nelementsQ1); AnalysisLogger.getLogger().trace("Number Of elements2: " + nelementsQ1);
List<Object> sumFirst = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(firstTable,probabilityColumnName),referencedbConnection); List<Object> sumFirst = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(firstTable, probabilityColumnName), referencedbConnection);
double sum1 = Double.parseDouble(""+sumFirst.get(0)); double sum1 = Double.parseDouble("" + sumFirst.get(0));
AnalysisLogger.getLogger().trace("Sum1: "+sum1); AnalysisLogger.getLogger().trace("Sum1: " + sum1);
List<Object> sumSecond = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(secondTable,probabilityColumnName),referencedbConnection); List<Object> sumSecond = DatabaseFactory.executeSQLQuery(DatabaseUtils.sumElementsStatement(secondTable, probabilityColumnName), referencedbConnection);
double sum2 = Double.parseDouble(""+sumSecond.get(0)); double sum2 = Double.parseDouble("" + sumSecond.get(0));
AnalysisLogger.getLogger().trace("Sum2: "+sum1); AnalysisLogger.getLogger().trace("Sum2: " + sum1);
double d = (double)(sum2-sum1)/(double)(nelements+nelements2); double d = (double) (sum2 - sum1) / (double) (nelements + nelements2);
return d; return d;
} }
@ -327,7 +433,7 @@ public class BioClimateAnalysis {
config.setParam("ComparisonThreshold", "" + comparisonThreshold); config.setParam("ComparisonThreshold", "" + comparisonThreshold);
config.setParam("MaxSamples", ""+30000); config.setParam("MaxSamples", "" + 30000);
eval = EvaluatorsFactory.getEvaluators(config).get(0); eval = EvaluatorsFactory.getEvaluators(config).get(0);
HashMap<String, String> out = eval.process(config); HashMap<String, String> out = eval.process(config);
@ -336,5 +442,4 @@ public class BioClimateAnalysis {
return d; return d;
} }
} }