deploy of charts and little correction on Gaussian charts

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@112129 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2015-02-16 17:02:26 +00:00
parent af8419ac0f
commit 937d559ae4
5 changed files with 483 additions and 3 deletions

View File

@ -94,7 +94,7 @@ public class GaussianDistributionGraph extends GenericStandaloneGraph {
protected JFreeChart createChart(Dataset dataset) {
String label = "mean:"+mean+" variance:"+variance;
String label = "mean:"+mean+" std dev:"+variance;
if (label.length()>30)
label = label.substring(0,30)+"...";
@ -114,9 +114,9 @@ public class GaussianDistributionGraph extends GenericStandaloneGraph {
return jfreechart;
}
public static JFreeChart createStaticChart(Dataset dataset, double mean, double variance) {
public static JFreeChart createStaticChart(Dataset dataset, double mean, double stddev) {
String label = "mean:"+mean+" variance:"+variance;
String label = "mean:"+mean+" std dev:"+stddev;
if (label.length()>30)
label = label.substring(0,30)+"...";

View File

@ -0,0 +1,59 @@
package org.gcube.dataanalysis.ecoengine.test;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.transducers.charts.QuantitiesAttributesChartsTransducerer;
import org.gcube.dataanalysis.ecoengine.transducers.charts.TimeSeriesChartsTransducerer;
public class ChartsTests {
public static void main(String[] args) throws Exception{
AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile);
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
config.setParam("InputTable", "hspen_mini");
// config.setParam("Attributes", "speciesid#lifestage#faoareas");
config.setParam("Quantities", "depthmax#speccode");
config.setGcubeScope("/gcube/devsec/devVRE");
QuantitiesAttributesChartsTransducerer cscreator = new QuantitiesAttributesChartsTransducerer();
cscreator.setConfiguration(config);
cscreator.compute();
System.out.println("DONE! "+cscreator.getOutput());
}
public static void main1(String[] args) throws Exception{
AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile);
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
config.setParam("InputTable", "timeseries_id4dd368bf_63fb_4d19_8e31_20ced63a477d");
config.setParam("Attributes", "country#area");
// config.setParam("Attributes", "");
config.setParam("Quantities", "quantity");
config.setParam("Time", "time");
config.setGcubeScope("/gcube/devsec/devVRE");
TimeSeriesChartsTransducerer cscreator = new TimeSeriesChartsTransducerer();
cscreator.setConfiguration(config);
cscreator.compute();
System.out.println("DONE! "+cscreator.getOutput());
}
}

View File

@ -0,0 +1,156 @@
package org.gcube.dataanalysis.ecoengine.transducers.charts;
import java.awt.Image;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import org.gcube.contentmanagement.graphtools.data.conversions.ImageTools;
import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm;
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
import org.hibernate.SessionFactory;
import org.jfree.chart.JFreeChart;
public abstract class AbstractChartsProducer extends StandardLocalExternalAlgorithm {
static String inputTableParameter = "InputTable";
static String attributesParameter = "Attributes";
static String quantitiesParameter = "Quantities";
static String timeParameter = "Time";
public HashMap<String, Image> producedImages = new HashMap<String, Image>();
public boolean displaycharts = true;
static int maxSeries = 10;
SessionFactory connection = null;
@Override
protected abstract void setInputParameters();
@Override
public StatisticalType getOutput() {
PrimitiveType images = new PrimitiveType(HashMap.class.getName(), producedImages, PrimitiveTypes.IMAGES, "images", "Charts");
LinkedHashMap<String, StatisticalType> outputmap = new LinkedHashMap<String, StatisticalType>();
outputmap.put("Images", images);
PrimitiveType output = new PrimitiveType(HashMap.class.getName(), outputmap, PrimitiveTypes.MAP, "ResultsMap", "Results Map");
return output;
}
@Override
public void init() throws Exception {
AnalysisLogger.getLogger().debug("ChartsProducer Initialized");
}
@Override
public abstract String getDescription();
public String[] getDimensions() {
String dimensionS = IOHelper.getInputParameter(config, attributesParameter);
String[] dimensions = null;
try{dimensions = dimensionS.split(AlgorithmConfiguration.getListSeparator());}catch(Exception e){}
if (dimensions!=null && dimensions.length==1 && dimensions[0].trim().length()==0)
dimensions = null;
return dimensions;
}
public String[] getQuantities() {
String quantitieS = IOHelper.getInputParameter(config, quantitiesParameter);
String[] quantities = quantitieS.split(AlgorithmConfiguration.getListSeparator());
return quantities;
}
public String getTimeDimension(){
String timeS = IOHelper.getInputParameter(config, timeParameter);
return timeS;
}
public String InfoRetrievalQuery(String table, String[] dimensions, String quantity,String time) {
if (time!=null){
if (dimensions!=null && dimensions.length>0)
return "select * from (select distinct " + Arrays.toString(dimensions).replace("[", "").replace("]", "") + " , " + quantity +","+time+" from "+table+" order by " + quantity + " limit " + maxSeries+") as a order by "+time;
else
return "select * from (select distinct " +quantity +","+time+" from "+table+" order by " + quantity + " limit " + maxSeries+") as a order by "+time;
}
else{
if (dimensions!=null && dimensions.length>0)
return "select distinct " + Arrays.toString(dimensions).replace("[", "").replace("]", "") + " , " + quantity +" from "+table+" order by " + quantity + " limit " + maxSeries;
else
return "select distinct row_number() over(), " + quantity +" from "+table+" order by " + quantity + " limit " + maxSeries;
}
}
@Override
protected void process() throws Exception {
status = 0;
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
long t0 = System.currentTimeMillis();
try {
AnalysisLogger.getLogger().debug("ChartsProducer started");
String driver = config.getParam("DatabaseDriver");
String username = config.getParam("DatabaseUserName");
String pwd = config.getParam("DatabasePassword");
String url = config.getParam("DatabaseURL");
String table = IOHelper.getInputParameter(config, inputTableParameter);
AnalysisLogger.getLogger().debug("ChartsProducer: Driver: " + driver + " User " + username + " URL " + url + " Table: " + table);
connection = DatabaseUtils.initDBSession(config);
AnalysisLogger.getLogger().debug("ChartsProducer: Connection initialized");
LexicalEngineConfiguration conf = new LexicalEngineConfiguration();
conf.setDatabaseUserName(username);
conf.setDatabasePassword(pwd);
conf.setDatabaseDriver(driver);
conf.setDatabaseURL(url);
conf.setDatabaseDialect("org.hibernate.dialect.PostgreSQLDialect");
String[] dimensions = getDimensions();
String[] quantities = getQuantities();
String time = getTimeDimension();
//one quantity for each chart
for (String quantity : quantities) {
//produce chart with dimensions,quantity, time
String query = InfoRetrievalQuery(table,dimensions,quantity,time);
AnalysisLogger.getLogger().debug("ChartsProducer: Query for retrieving information "+query);
List<Object> values = DatabaseFactory.executeSQLQuery(query, connection);
JFreeChart[] charts= createCharts(dimensions,quantity,time,values,displaycharts);
for (JFreeChart chart:charts){
if (chart!=null){
Image image= ImageTools.toImage(chart.createBufferedImage(680, 420));
producedImages.put("Charts focused on "+quantity, image);
}
}
}
AnalysisLogger.getLogger().debug("ChartsProducer: finished");
} catch (Throwable e) {
e.printStackTrace();
throw new Exception(e.getMessage());
} finally {
shutdown();
status = 100;
}
}
public abstract JFreeChart[] createCharts(String[] dimensions,String quantity,String time,List<Object> rows, boolean displaychart);
@Override
public void shutdown() {
AnalysisLogger.getLogger().debug("ChartsProducer shutdown");
DatabaseUtils.closeDBConnection(connection);
}
}

View File

@ -0,0 +1,126 @@
package org.gcube.dataanalysis.ecoengine.transducers.charts;
import java.util.ArrayList;
import java.util.List;
import org.gcube.contentmanagement.graphtools.plotting.graphs.GaussianDistributionGraph;
import org.gcube.contentmanagement.graphtools.plotting.graphs.HistogramGraph;
import org.gcube.contentmanagement.graphtools.plotting.graphs.RadarGraph;
import org.gcube.contentmanagement.graphtools.plotting.graphs.ScatterGraphGeneric;
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
import org.jfree.chart.JFreeChart;
import org.jfree.data.category.DefaultCategoryDataset;
import org.jfree.data.function.NormalDistributionFunction2D;
import org.jfree.data.general.DatasetUtilities;
import org.jfree.data.xy.XYSeriesCollection;
public class QuantitiesAttributesChartsTransducerer extends AbstractChartsProducer {
@Override
protected void setInputParameters() {
List<TableTemplates> templates = new ArrayList<TableTemplates>();
templates.add(TableTemplates.GENERIC);
InputTable tinput = new InputTable(templates, inputTableParameter, "The input table");
inputs.add(tinput);
ColumnTypesList p1 = new ColumnTypesList(inputTableParameter, attributesParameter, "The dimensions to consider in the charts", true);
ColumnTypesList p2 = new ColumnTypesList(inputTableParameter, quantitiesParameter, "The numeric quantities to visualize ", false);
inputs.add(p1);
inputs.add(p2);
DatabaseType.addDefaultDBPars(inputs);
}
@Override
public String getDescription() {
return "An algorithm producing generic charts of attributes vs. quantities. Charts are displayed per quantity column. Histograms, Scattering and Radar charts are produced for the top ten quantities. A gaussian distribution reports overall statistics for the quantities.";
}
@Override
public JFreeChart[] createCharts(String[] dimensions, String quantity, String time, List<Object> rows, boolean displaychart) {
if (dimensions==null)
dimensions=new String[0];
DefaultCategoryDataset datasetHisto = new DefaultCategoryDataset();
for (Object row : rows) {
Object[] array = (Object[]) row;
Double q = null;
try {
q = Double.parseDouble("" + array[array.length - 1]);
} catch (Exception e) {
}
if (q != null) {
int dimIdx = 0;
for (String dimension : dimensions) {
String dimensionValue = "";
if (array[dimIdx]!=null)
dimensionValue = ""+array[dimIdx];
if (dimensionValue.length()>0)
datasetHisto.addValue(q, dimensionValue, dimension);
dimIdx++;
}
}
}
List<Object> meanvar = DatabaseFactory.executeSQLQuery("select avg("+quantity+"), variance("+quantity+") from "+IOHelper.getInputParameter(config, inputTableParameter), connection);
Object[] meanvarsrow = (Object[]) meanvar.get(0);
double mean = MathFunctions.roundDecimal(Double.parseDouble(""+meanvarsrow[0]),2);
double variance = MathFunctions.roundDecimal(Math.sqrt(Double.parseDouble(""+meanvarsrow[1])),2);
AnalysisLogger.getLogger().debug("QuantitiesAttributesCharts: " + mean + " and variance:" + variance);
NormalDistributionFunction2D normaldistributionfunction2d = new NormalDistributionFunction2D(mean, variance);
org.jfree.data.xy.XYSeries gaussianxyseries = DatasetUtilities.sampleFunction2DToSeries(normaldistributionfunction2d, (mean - (2 * variance)), (mean + (2 * variance)), 121, "Distribution of "+quantity);
XYSeriesCollection gaussianxyseriescollection = new XYSeriesCollection();
gaussianxyseriescollection .addSeries(gaussianxyseries);
JFreeChart charthisto = null;
JFreeChart chartscattering = null;
JFreeChart chartradar = null;
if (dimensions.length>0){
charthisto = HistogramGraph.createStaticChart(datasetHisto);
chartscattering = ScatterGraphGeneric.createStaticChart(datasetHisto);
chartradar = RadarGraph.createStaticChart(datasetHisto);
}
JFreeChart chartgaussian = GaussianDistributionGraph.createStaticChart(gaussianxyseriescollection, mean, variance);
if (displaychart){
HistogramGraph tsg = new HistogramGraph("QuantitiesAttributesHistogram");
tsg.render(datasetHisto);
ScatterGraphGeneric scatter = new ScatterGraphGeneric("QuantitiesAttributesScattering");
scatter.render(datasetHisto);
RadarGraph radar = new RadarGraph("QuantitiesAttributesRadar");
radar.render(datasetHisto);
GaussianDistributionGraph gaussian = new GaussianDistributionGraph("Gaussian");
gaussian.mean=mean;
gaussian.variance=variance;
gaussian.render(gaussianxyseriescollection);
}
JFreeChart [] charts = new JFreeChart[4];
charts [0] = charthisto;
charts [1] = chartscattering;
charts [2] = chartradar;
charts [3] = chartgaussian;
return charts;
}
}

View File

@ -0,0 +1,139 @@
package org.gcube.dataanalysis.ecoengine.transducers.charts;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import org.gcube.contentmanagement.graphtools.plotting.graphs.LineGraph;
import org.gcube.contentmanagement.graphtools.plotting.graphs.TimeSeriesGraph;
import org.gcube.contentmanagement.graphtools.plotting.graphs.TransectLineGraph;
import org.gcube.contentmanagement.graphtools.utils.DateGuesser;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
import org.jfree.chart.JFreeChart;
import org.jfree.data.category.DefaultCategoryDataset;
import org.jfree.data.time.FixedMillisecond;
import org.jfree.data.time.TimeSeries;
import org.jfree.data.time.TimeSeriesCollection;
import com.sun.org.apache.bcel.internal.generic.LNEG;
public class TimeSeriesChartsTransducerer extends QuantitiesAttributesChartsTransducerer {
@Override
protected void setInputParameters() {
super.setInputParameters();
ColumnType p = new ColumnType(inputTableParameter, timeParameter, "The column containing time information", "year", false);
inputs.add(p);
}
@Override
public String getDescription() {
return "An algorithm producing time series charts of attributes vs. quantities. Charts are displayed per quantity column and only the top ten quantities are reported.";
}
String timepattern = "";
SimpleDateFormat formatter = null;
Date sampleTime = null;
public Date getTime(String timel) {
timel = timel.replace("time:", "");
try {
if (formatter == null) {
timepattern = DateGuesser.getPattern(timel);
AnalysisLogger.getLogger().debug("Time pattern: " + timepattern);
formatter = new SimpleDateFormat(timepattern);
sampleTime = formatter.parse(timel);
}
return formatter.parse(timel);
} catch (ParseException e) {
e.printStackTrace();
return null;
}
}
public String getChartPattern(Date time) {
if (timepattern.equals("s") || DateGuesser.isJavaDateOrigin(time)) {
return "HH:mm:ss:SS";
} else {
if (timepattern.length() == 4)
return "yyyy";
else
return "MM-dd-yy";
}
}
@Override
public JFreeChart[] createCharts(String[] dimensions, String quantity, String time, List<Object> rows, boolean displaychart) {
if (dimensions==null)
dimensions = new String[0];
TimeSeriesCollection dataset = new TimeSeriesCollection();
TimeSeries series = null;
if (dimensions.length>0)
series = new TimeSeries("Trend of " + quantity + " for " + Arrays.toString(dimensions).replace("[", "").replace("]", ""));
else
series = new TimeSeries("Trend of " + quantity);
dataset.addSeries(series);
DefaultCategoryDataset[] linedatasets = new DefaultCategoryDataset[dimensions.length];
for (Object row : rows) {
Object[] array = (Object[]) row;
Double q = null;
Date timeD = null;
String timel = "" + array[array.length - 1];
try {
q = Double.parseDouble("" + array[array.length - 2]);
timeD = getTime(timel);
} catch (Exception e) {
}
if (q != null && timeD != null) {
FixedMillisecond ms = new FixedMillisecond(timeD);
series.add(ms, q);
for (int dimIdx=0;dimIdx<dimensions.length;dimIdx++) {
String dimensionValue = "";
if (array[dimIdx] != null)
dimensionValue = "" + array[dimIdx];
if (dimensionValue.length() > 0) {
DefaultCategoryDataset lineds = linedatasets[dimIdx];
if (lineds == null){
lineds = new DefaultCategoryDataset();
linedatasets[dimIdx]=lineds;
}
lineds.addValue(q, "singlets", timel + ";[" + dimensionValue + "]");
}
}
}
}
JFreeChart chartTS = TimeSeriesGraph.createStaticChart(dataset, getChartPattern(sampleTime), "Variations of " + quantity);
if (displaychart) {
TimeSeriesGraph tsg = new TimeSeriesGraph("Variations of " + quantity);
tsg.timeseriesformat = getChartPattern(sampleTime);
tsg.render(dataset);
for (int i = 0; i < dimensions.length; i++) {
TransectLineGraph tlg = new TransectLineGraph("Variations of " + dimensions[i]);
tlg.render(linedatasets[i]);
}
}
JFreeChart[] charts = new JFreeChart[1 + dimensions.length];
charts[0] = chartTS;
for (int i = 0; i < dimensions.length; i++) {
JFreeChart linechartTS = LineGraph.createStaticChart(linedatasets[i], "Variations of " + dimensions[i]);
charts[i+1] = linechartTS;
}
return charts;
}
}