gCubeSystem
/
ecological-engine-geospatial-extensions
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@92747 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2014-03-06 11:56:54 +00:00
parent cf54369930
commit 9b76e5e8a6
17 changed files with 608 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

304
src/main/java/org/gcube/dataanalysis/geo/algorithms/OccurrenceEnrichment.java
View File

@ -1,6 +1,8 @@
package org.gcube.dataanalysis.geo.algorithms; package org.gcube.dataanalysis.geo.algorithms;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
@ -11,6 +13,7 @@ import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveTypesList; import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveTypesList;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
@ -21,6 +24,7 @@ import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper; import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory;
import org.gcube.dataanalysis.ecoengine.utils.Tuple; import org.gcube.dataanalysis.ecoengine.utils.Tuple;
import org.gcube.dataanalysis.geo.interfaces.GISDataConnector;
import org.gcube.dataanalysis.geo.matrixmodel.XYExtractor; import org.gcube.dataanalysis.geo.matrixmodel.XYExtractor;
import org.gcube.dataanalysis.geo.utils.VectorOperations; import org.gcube.dataanalysis.geo.utils.VectorOperations;
import org.hibernate.SessionFactory; import org.hibernate.SessionFactory;
@ -28,89 +32,102 @@ import org.hibernate.SessionFactory;
public class OccurrenceEnrichment implements Transducerer { public class OccurrenceEnrichment implements Transducerer {
static String OccurrencesTableNameParameter = "OccurrenceTable"; static String OccurrencesTableNameParameter = "OccurrenceTable";
static String LongitudeColumn= "LongitudeColumn"; static String LongitudeColumn = "LongitudeColumn";
static String LatitudeColumn = "LatitudeColumn"; static String LatitudeColumn = "LatitudeColumn";
static String ScientificNameColumn = "ScientificNameColumn"; static String ScientificNameColumn = "ScientificNameColumn";
static String TimeColumn = "TimeColumn";
static String OutputTableLabelParameter = "OutputTableName"; static String OutputTableLabelParameter = "OutputTableName";
static String OutputTableDBNameParameter = "OutputTableDBName"; static String OutputTableDBNameParameter = "OutputTableDBName";
static String FilterParameter = "OptionalFilter"; static String FilterParameter = "OptionalFilter";
static String Resolution = "Resolution"; static String Resolution = "Resolution";
static String Layers = "Layers"; static String Layers = "Layers";
static String LayersNames = "FeaturesNames";
static String yLL = "BBox_LowerLeftLat"; static String yLL = "BBox_LowerLeftLat";
static String xLL = "BBox_LowerLeftLong"; static String xLL = "BBox_LowerLeftLong";
static String yUR = "BBox_UpperRightLat"; static String yUR = "BBox_UpperRightLat";
static String xUR = "BBox_UpperRightLong"; static String xUR = "BBox_UpperRightLong";
AlgorithmConfiguration config; AlgorithmConfiguration config;
float status; float status;
private String [] layers; private String[] layers;
private String[] layersnames;
private String occurrencesTableName; private String occurrencesTableName;
private String longitudeColumn; private String longitudeColumn;
private String latitudeColumn; private String latitudeColumn;
private String scientificnameColumn; private String scientificnameColumn;
private String timeColumn;
private String filter; private String filter;
private float resolution; private float resolution;
private String outputTableLabel; private String outputTableLabel;
private String outputTableDBName; private String outputTableDBName;
private double BBxLL; public LinkedHashMap<String, String> outputParameters = new LinkedHashMap<String, String>();
private double BByLL;
private double BBxUR;
private double BByUR;
@Override @Override
public List<StatisticalType> getInputParameters() { public List<StatisticalType> getInputParameters() {
List<StatisticalType> inputs = new ArrayList<StatisticalType>(); List<StatisticalType> inputs = new ArrayList<StatisticalType>();
List<TableTemplates> template= new ArrayList<TableTemplates>(); List<TableTemplates> template = new ArrayList<TableTemplates>();
template.add(TableTemplates.OCCURRENCE_SPECIES); template.add(TableTemplates.OCCURRENCE_SPECIES);
InputTable table = new InputTable(template,OccurrencesTableNameParameter ,"A geospatial table containing at least x,y information",""); InputTable table = new InputTable(template, OccurrencesTableNameParameter, "A geospatial table containing occurrence records, following the template of the Species Products Discovery datasets", "");
inputs.add(table); inputs.add(table);
ColumnType p1 = new ColumnType(OccurrencesTableNameParameter , LongitudeColumn, "column with longitude values", "decimallongitude", false); ColumnType p1 = new ColumnType(OccurrencesTableNameParameter, LongitudeColumn, "The column containing longitude values", "decimallongitude", false);
inputs.add(p1); inputs.add(p1);
ColumnType p2 = new ColumnType(OccurrencesTableNameParameter , LatitudeColumn, "column with latitude values", "decimallatitude", false); ColumnType p2 = new ColumnType(OccurrencesTableNameParameter, LatitudeColumn, "The column containing latitude values", "decimallatitude", false);
inputs.add(p2); inputs.add(p2);
ColumnType p3 = new ColumnType(OccurrencesTableNameParameter , ScientificNameColumn, "column with Scientific Names", "scientificname", false); ColumnType p3 = new ColumnType(OccurrencesTableNameParameter, ScientificNameColumn, "The column containing Scientific Names", "scientificname", false);
inputs.add(p3); inputs.add(p3);
ColumnType p4 = new ColumnType(OccurrencesTableNameParameter, TimeColumn, "The column containing time information", "eventdate", false);
IOHelper.addStringInput(inputs, FilterParameter, "A filter on one of the columns (e.g. basisofrecord='HumanObservation')", " "); inputs.add(p4);
IOHelper.addDoubleInput(inputs, Resolution, "The spatial resolution of the association between observations and environmental features.", "0.5");
IOHelper.addRandomStringInput(inputs, OutputTableDBNameParameter, "The db name of the table to produce", "enrich_"); IOHelper.addStringInput(inputs, FilterParameter, "A filter on one of the columns (e.g. basisofrecord='HumanObservation'). Optional", " ");
IOHelper.addDoubleInput(inputs, Resolution, "The spatial resolution in degrees of the association between observations and environmental features", "0.5");
IOHelper.addRandomStringInput(inputs, OutputTableDBNameParameter, "The database name of the table to produce", "enrich_");
IOHelper.addStringInput(inputs, OutputTableLabelParameter, "The name of the output table", "enrich_"); IOHelper.addStringInput(inputs, OutputTableLabelParameter, "The name of the output table", "enrich_");
PrimitiveTypesList listEnvLayers = new PrimitiveTypesList(String.class.getName(), PrimitiveTypes.STRING, Layers, "The list of environmental layers to use for enriching the points. Each entry is a Layer Title or UUID (preferred) of a layer indexed in the e-Infrastructure on GeoNetwork - You can retrieve it from GeoExplorer. Supports several standards (NETCDF, WFS, WCS ASC, GeoTiff )", false); PrimitiveTypesList listEnvLayers = new PrimitiveTypesList(String.class.getName(), PrimitiveTypes.STRING, Layers, "The list of environmental layers to use for enriching the points. Each entry is a layer Title or UUID or HTTP link. E.g. the title or the UUID (preferred) of a layer indexed in the e-Infrastructure on GeoNetwork - You can retrieve it from GeoExplorer. Otherwise you can supply the direct HTTP link of the layer. The format will be guessed from the link. The default is GeoTiff. Supports several standards (NETCDF, WFS, WCS, ASC, GeoTiff )", false);
inputs.add(listEnvLayers); inputs.add(listEnvLayers);
IOHelper.addDoubleInput(inputs, xLL, "Lower Left Longitude of the Bounding Box", "-180"); PrimitiveTypesList listEnvLayersNames = new PrimitiveTypesList(String.class.getName(), PrimitiveTypes.STRING, LayersNames, "The list of names for the columns corresponding to the environmental layers. These will be the column names of the resulting table", false);
IOHelper.addDoubleInput(inputs, yLL, "Lower Left Latitute of the Bounding Box", "-90"); inputs.add(listEnvLayersNames);
IOHelper.addDoubleInput(inputs, xUR, "Upper Right Longitude of the Bounding Box", "180");
IOHelper.addDoubleInput(inputs, yUR, "Upper Right Latitute of the Bounding Box", "90");
DatabaseType.addDefaultDBPars(inputs); DatabaseType.addDefaultDBPars(inputs);
return inputs; return inputs;
} }
protected void getParameters() { protected void getParameters() {
layers = IOHelper.getInputParameter(config, Layers).split(AlgorithmConfiguration.getListSeparator()); layers = IOHelper.getInputParameter(config, Layers).trim().split(AlgorithmConfiguration.getListSeparator());
AnalysisLogger.getLogger().debug("Layers to take " + layers.length); String layernamesS = IOHelper.getInputParameter(config, LayersNames);
occurrencesTableName=IOHelper.getInputParameter(config, OccurrencesTableNameParameter); if (layernamesS == null)
longitudeColumn=IOHelper.getInputParameter(config, LongitudeColumn); layernamesS = "";
latitudeColumn=IOHelper.getInputParameter(config, LatitudeColumn); layersnames = layernamesS.split(AlgorithmConfiguration.getListSeparator());
scientificnameColumn=IOHelper.getInputParameter(config, ScientificNameColumn); AnalysisLogger.getLogger().debug("N. of Layers to take " + layers.length);
filter=IOHelper.getInputParameter(config, FilterParameter); occurrencesTableName = IOHelper.getInputParameter(config, OccurrencesTableNameParameter);
if (filter==null) longitudeColumn = IOHelper.getInputParameter(config, LongitudeColumn);
filter=""; latitudeColumn = IOHelper.getInputParameter(config, LatitudeColumn);
filter=filter.trim(); scientificnameColumn = IOHelper.getInputParameter(config, ScientificNameColumn);
timeColumn = IOHelper.getInputParameter(config, TimeColumn);
resolution=IOHelper.getInputParameter(config, Resolution)==null?0.5f:Float.parseFloat(IOHelper.getInputParameter(config, Resolution)); filter = IOHelper.getInputParameter(config, FilterParameter);
outputTableLabel=IOHelper.getInputParameter(config, OutputTableLabelParameter); if (filter == null)
outputTableDBName=IOHelper.getInputParameter(config, OutputTableDBNameParameter); filter = "";
BBxLL=Double.parseDouble(IOHelper.getInputParameter(config, xLL)); filter = filter.trim();
BByLL=Double.parseDouble(IOHelper.getInputParameter(config, yLL));
BBxUR=Double.parseDouble(IOHelper.getInputParameter(config, xUR)); resolution = IOHelper.getInputParameter(config, Resolution) == null ? 0.5f : Float.parseFloat(IOHelper.getInputParameter(config, Resolution));
BByUR=Double.parseDouble(IOHelper.getInputParameter(config, yUR)); outputTableLabel = IOHelper.getInputParameter(config, OutputTableLabelParameter);
outputTableDBName = IOHelper.getInputParameter(config, OutputTableDBNameParameter);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->layers: " + layers);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->layers names: " + layersnames);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->occurrencesTableName: " + occurrencesTableName);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->longitudeColumn: " + longitudeColumn);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->latitudeColumn: " + latitudeColumn);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->scientificnameColumn: " + scientificnameColumn);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->timeColumn: " + timeColumn);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->filter: " + filter);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->resolution: " + resolution);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->outputTableLabel: " + outputTableLabel);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->outputTableDBName: " + outputTableDBName);
String scope = config.getGcubeScope(); String scope = config.getGcubeScope();
AnalysisLogger.getLogger().debug("Extraction: Externally set scope " + scope); AnalysisLogger.getLogger().debug("Extraction: Externally set scope " + scope);
if (scope == null) { if (scope == null) {
@ -120,6 +137,7 @@ public class OccurrenceEnrichment implements Transducerer {
} }
} }
@Override @Override
public void init() throws Exception { public void init() throws Exception {
AnalysisLogger.getLogger().debug("Occurrence Enrichment Initialization"); AnalysisLogger.getLogger().debug("Occurrence Enrichment Initialization");
@ -132,83 +150,157 @@ public class OccurrenceEnrichment implements Transducerer {
@Override @Override
public String getDescription() { public String getDescription() {
return "An algorithm performing occurrences enrichment. Takes as input one table containing occurrence points for a set of species and a list of environmental layers from the e-infrastructure GeoNetwork (through the GeoExplorer application). Produces one table reporting the set of environmental values associated to the occurrence points."; return "An algorithm performing occurrences enrichment. Takes as input one table containing occurrence points for a set of species and a list of environmental layer, taken either from the e-infrastructure GeoNetwork (through the GeoExplorer application) or from direct HTTP links. Produces one table reporting the set of environmental values associated to the occurrence points.";
} }
@Override @Override
public void compute() throws Exception { public void compute() throws Exception {
//TODO: report times
SessionFactory dbconnection=null; SessionFactory dbconnection = null;
try{ try {
dbconnection=DatabaseUtils.initDBSession(config); long t0 = System.currentTimeMillis();
String columns = longitudeColumn+","+latitudeColumn+","+scientificnameColumn; status = 10;
String columnsToProduce = longitudeColumn+","+latitudeColumn+","+scientificnameColumn; getParameters();
//take the occurrence points dbconnection = DatabaseUtils.initDBSession(config);
List<Object> rows = (List<Object>)DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(occurrencesTableName, columns, filter),dbconnection); String columns = longitudeColumn + "," + latitudeColumn + "," + scientificnameColumn + "," + timeColumn;
if (rows==null || rows.size()==0) // (id serial, csquarecode character varying, x real, y real, z real, t real, fvalue real)
String columnsTypes = "id serial, " + longitudeColumn + " real," + latitudeColumn + " real," + scientificnameColumn + " character varying," + timeColumn + " timestamp without time zone";
// take min_max lat
String query = "select min(" + longitudeColumn + ") as minlong, max(" + longitudeColumn + ") as maxlong,min(" + latitudeColumn + ") as minlat,max(" + latitudeColumn + ") as maxlat from " + occurrencesTableName;
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Retrieving bounding box: " + query);
List<Object> minmaxlonglat = (List<Object>) DatabaseFactory.executeSQLQuery(query, dbconnection);
if (minmaxlonglat == null || minmaxlonglat.size() == 0)
throw new Exception("Could not find min and max for occurrence data");
status = 20;
Object[] minmaxLongLat = (Object[]) minmaxlonglat.get(0);
double BBxLL = Double.parseDouble("" + minmaxLongLat[0]);
double BBxUR = Double.parseDouble("" + minmaxLongLat[1]);
double BByLL = Double.parseDouble("" + minmaxLongLat[2]);
double BByUR = Double.parseDouble("" + minmaxLongLat[3]);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Calculated Bounding Box: [" + BBxLL + "," + BByLL + ";" + BBxUR + "," + BByUR + "]");
// take the occurrence points
List<Object> rows = (List<Object>) DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(occurrencesTableName, columns, filter), dbconnection);
if (rows == null || rows.size() == 0)
throw new Exception("Could not find occurrence data"); throw new Exception("Could not find occurrence data");
status = 30;
int rowsize = rows.size(); AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Rows Retrieved");
List<Tuple<Double>> coordinates4d = new ArrayList<Tuple<Double>>(); List<Tuple<Double>> coordinates4d = new ArrayList<Tuple<Double>>();
List<String[]> enrichment = new ArrayList<String[]>(); List<String[]> enrichment = new ArrayList<String[]>();
int elementstoreport = 4;
int elementsfromoccurrences = 3; int elementsFromOccurrences = 4;
for (Object row:rows){ int elementstoreport = elementsFromOccurrences + layers.length;
for (Object row : rows) {
Object[] elements = (Object[]) row; Object[] elements = (Object[]) row;
double x =elements[0]==null?0:Double.parseDouble(""+elements[0]); double x = elements[0] == null ? 0 : Double.parseDouble("" + elements[0]);
double y =elements[1]==null?0:Double.parseDouble(""+elements[1]); double y = elements[1] == null ? 0 : Double.parseDouble("" + elements[1]);
String species=elements[2]==null?"":""+elements[2]; String species = elements[2] == null ? "" : "" + elements[2];
Tuple<Double> el = new Tuple<Double>(x,y); String time = elements[3] == null ? "NULL" : "" + elements[3];
Tuple<Double> el = new Tuple<Double>(x, y, 0d, 0d);
coordinates4d.add(el); coordinates4d.add(el);
String[] singlerow = new String[elementstoreport]; String[] singlerow = new String[elementstoreport];
singlerow [0]=""+x; singlerow[0] = "" + x;
singlerow[1]=""+y; singlerow[1] = "" + y;
singlerow[2]=""+species; singlerow[2] = species;
singlerow[3] = time;
enrichment.add(singlerow);
} }
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Tuples Created. Assigning grid values to the tuples");
//take the layers matrices status = 40;
int layeridx = 0; // take the layers matrices
for (String layerID:layers){ int layeridx = 0;
//for each layer float statusSteps = 50f / (float) layers.length;
XYExtractor extractor = new XYExtractor(config); for (String layerID : layers) {
extractor.extractXYGrid(layerID, 0, BBxLL, BBxUR, BByLL, BByUR, 0, resolution,resolution); if (layerID.length() == 0)
List<Double> gridValues = extractor.currentTimeValues; continue;
List<Tuple<Double>> grid3d =extractor.currentTuples; AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Assigning layer " + layerID + " Layer enumerator: " + layeridx);
int time = 0; // for each layer
//take the association XYExtractor extractor = new XYExtractor(config);
List<Double> enriched = VectorOperations.assignGridValuesToPoints(grid3d, time , gridValues, coordinates4d, resolution); extractor.correctZ(0, layerID,resolution);
int k=0; double zmin = extractor.zmin;
for (Double value:enriched){ double zmax = extractor.zmax;
String[] singlerow =enrichment.get(k); double bestZ = Math.min(Math.abs(zmin), Math.abs(zmax));
singlerow[layeridx+elementsfromoccurrences] = ""+value;
k++; outputParameters.put("Matching Z value in layer " + (layeridx + 1), "" + bestZ);
outputParameters.put("Min Z value in layer "+ (layeridx + 1), "" + zmin);
outputParameters.put("Max Z value in layer "+ (layeridx + 1), "" + zmax);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Best Z for this reference layer: " + bestZ);
// perform the comparison closest to the surface
extractor.extractXYGrid(layerID, 0, BBxLL, BBxUR, BByLL, BByUR, bestZ, resolution, resolution);
// retrieve the grid time values and tuples
List<Double> gridValues = extractor.currentTimeValues;
List<Tuple<Double>> grid3d = extractor.currentTuples;
String layername = (layersnames.length > (layeridx) && layersnames[layeridx].trim().length() > 0) ? layersnames[layeridx].trim() : "feature" + (layeridx + 1);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Retrieved Layer Name: " + layername);
columns += ",\"" + layername + "\"";
columnsTypes += ",\"" + layername + "\" real";
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Assigning grid points to the occurrences");
// take the association
List<Double> enriched = VectorOperations.assignGridValuesToPoints2D(grid3d, gridValues, coordinates4d, resolution);
int k = 0;
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Assigning values to the column " + (elementsFromOccurrences + layeridx));
for (Double value : enriched) {
String[] singlerow = enrichment.get(k);
if (value == null || Double.isNaN(value) || Double.isInfinite(value))
singlerow[elementsFromOccurrences + layeridx] = "-9999";
else
singlerow[elementsFromOccurrences + layeridx] = "" + value;
k++;
}
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Added values to the row");
layeridx++;
status = status + statusSteps;
} }
layeridx++;
} // write the complete association into the db
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Dropping table " + outputTableDBName);
//write the complete association into the db try {
DatabaseFactory.executeSQLQuery(DatabaseUtils.dropTableStatement(outputTableDBName),dbconnection); DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outputTableDBName), dbconnection);
//TODO: create table } catch (Exception e) {
DatabaseUtils.insertChunksIntoTable(outputTableDBName, columnsToProduce, enrichment, 5000, dbconnection); AnalysisLogger.getLogger().debug("OccurrenceEnrichment->cannot drop table, does not exist: " + outputTableDBName);
}
}catch(Exception e){ String createquery = "create table " + outputTableDBName + " (" + columnsTypes + ")";
if (dbconnection!=null) AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Creating table " + outputTableDBName + " query:" + createquery);
DatabaseFactory.executeSQLUpdate(createquery, dbconnection);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Inserting chunks");
DatabaseUtils.insertChunksIntoTable(outputTableDBName, columns, enrichment, 5000, dbconnection);
AnalysisLogger.getLogger().debug("OccurrenceEnrichment->Whole process complete in " + ((double) (System.currentTimeMillis() - t0) / 1000f) + " s");
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
if (dbconnection != null)
dbconnection.close(); dbconnection.close();
status = 100;
} }
} }
@Override @Override
public StatisticalType getOutput() { public StatisticalType getOutput() {
List<TableTemplates> templateHspec = new ArrayList<TableTemplates>(); List<TableTemplates> templateHspec = new ArrayList<TableTemplates>();
templateHspec.add(TableTemplates.GENERIC); templateHspec.add(TableTemplates.GENERIC);
OutputTable p = new OutputTable(templateHspec, OutputTableLabelParameter, OutputTableDBNameParameter, "Output table"); OutputTable p = new OutputTable(templateHspec, outputTableLabel, outputTableDBName, "Output table");
return p; LinkedHashMap<String, StatisticalType> map = new LinkedHashMap<String, StatisticalType>();
for (String key : outputParameters.keySet()) {
String value = outputParameters.get(key);
PrimitiveType val = new PrimitiveType(String.class.getName(), "" + value, PrimitiveTypes.STRING, key, key);
map.put(key, val);
}
map.put("OutputTable", p);
PrimitiveType outputm = new PrimitiveType(HashMap.class.getName(), map, PrimitiveTypes.MAP, "ResultsMap", "Results Map");
return outputm;
} }
@Override @Override
@ -237,5 +329,5 @@ public class OccurrenceEnrichment implements Transducerer {
public String getResources() { public String getResources() {
return ResourceFactory.getResources(100f); return ResourceFactory.getResources(100f);
} }
} }

31
src/main/java/org/gcube/dataanalysis/geo/algorithms/TimeExtraction.java
View File

@ -21,6 +21,7 @@ import org.gcube.dataanalysis.ecoengine.signals.SignalConverter;
import org.gcube.dataanalysis.ecoengine.signals.SignalProcessing; import org.gcube.dataanalysis.ecoengine.signals.SignalProcessing;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper; import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
import org.gcube.dataanalysis.ecoengine.utils.Tuple; import org.gcube.dataanalysis.ecoengine.utils.Tuple;
import org.gcube.dataanalysis.geo.interfaces.GISDataConnector;
import org.gcube.dataanalysis.geo.matrixmodel.RasterTable; import org.gcube.dataanalysis.geo.matrixmodel.RasterTable;
import org.gcube.dataanalysis.geo.matrixmodel.TimeSeriesExtractor; import org.gcube.dataanalysis.geo.matrixmodel.TimeSeriesExtractor;
@ -63,9 +64,9 @@ public class TimeExtraction extends XYExtraction{
IOHelper.addDoubleInput(inputs, resolution, "Extraction point resolution", "0.5"); IOHelper.addDoubleInput(inputs, resolution, "Extraction point resolution", "0.5");
IOHelper.addIntegerInput(inputs, samplingFrequency, "Sampling frequency in Hz. Leave it to -1 if unknown or under 1", "-1"); IOHelper.addIntegerInput(inputs, samplingFrequency, "Sampling frequency in Hz. Leave it to -1 if unknown or under 1", "-1");
IOHelper.addDoubleInput(inputs, minFrequency, "Minimum expected frequency in Hz. Can be decimal.", "-1"); IOHelper.addDoubleInput(inputs, minFrequency, "Minimum expected frequency in Hz. Can be decimal", "-1");
IOHelper.addDoubleInput(inputs, maxFrequency, "Maximum expected frequency in Hz. Can be decimal.", "-1"); IOHelper.addDoubleInput(inputs, maxFrequency, "Maximum expected frequency in Hz. Can be decimal", "-1");
IOHelper.addDoubleInput(inputs, expectedFrequencyError, "Expected precision on periodicity detection in Hz. Can be decimal and depends on the signal length.", "0.1"); IOHelper.addDoubleInput(inputs, expectedFrequencyError, "Expected precision on periodicity detection in Hz or 1/samples. Can be decimal and depends on the signal length. Default is 0.1", "0.1");
DatabaseType.addDefaultDBPars(inputs); DatabaseType.addDefaultDBPars(inputs);
@ -120,6 +121,14 @@ public class TimeExtraction extends XYExtraction{
AnalysisLogger.getLogger().debug("Extracting Time Series from layer"); AnalysisLogger.getLogger().debug("Extracting Time Series from layer");
TimeSeriesExtractor intersector = new TimeSeriesExtractor(config); TimeSeriesExtractor intersector = new TimeSeriesExtractor(config);
long t0 = System.currentTimeMillis(); long t0 = System.currentTimeMillis();
//take best z
zValue = intersector.correctZ(zValue, layerNameValue, resolutionValue);
AnalysisLogger.getLogger().debug("TimeExtraction->Best Z for this reference layer: " + zValue);
outputParameters.put("Matching Z value in the layer", ""+zValue);
outputParameters.put("Min Z value in the Layer", ""+intersector.zmin);
outputParameters.put("Max Z value in the Layer", ""+intersector.zmax);
AnalysisLogger.getLogger().debug("Z allowed to be: "+zValue);
signal = intersector.extractT(layerNameValue, xValue,yValue, zValue, resolutionValue); signal = intersector.extractT(layerNameValue, xValue,yValue, zValue, resolutionValue);
AnalysisLogger.getLogger().debug("ELAPSED TIME: "+(System.currentTimeMillis()-t0)); AnalysisLogger.getLogger().debug("ELAPSED TIME: "+(System.currentTimeMillis()-t0));
@ -188,10 +197,12 @@ public class TimeExtraction extends XYExtraction{
raster.deleteTable(); raster.deleteTable();
raster.dumpGeoTable(); raster.dumpGeoTable();
signalimage = SignalProcessing.renderSignalWithGenericTime(signal, timeline, "Signal");
// spectrogramImage = SignalProcessing.renderSignalSpectrogram(signal, timeline, pd.currentSamplingRate, pd.currentWindowAnalysisSamples, pd.currentWindowShiftSamples);
spectrogramImage = SignalProcessing.renderSignalSpectrogram2(pd.currentspectrum);
// spectrogramImage = SignalProcessing.renderSignalSpectrogram(signal, timeline, pd.currentSamplingRate, pd.currentWindowAnalysisSamples, pd.currentWindowShiftSamples);
if (pd!=null && pd.currentspectrum!=null && pd.currentspectrum.length>0){
signalimage = SignalProcessing.renderSignalWithGenericTime(signal, timeline, "Signal");
spectrogramImage = SignalProcessing.renderSignalSpectrogram2(pd.currentspectrum);
}
AnalysisLogger.getLogger().debug("Extractor: Map was dumped in table: " + tableNameValue); AnalysisLogger.getLogger().debug("Extractor: Map was dumped in table: " + tableNameValue);
status = 80; status = 80;
AnalysisLogger.getLogger().debug("Extractor: Elapsed: Whole operation completed in " + ((double) (System.currentTimeMillis() - t0) / 1000d) + "s"); AnalysisLogger.getLogger().debug("Extractor: Elapsed: Whole operation completed in " + ((double) (System.currentTimeMillis() - t0) / 1000d) + "s");
@ -217,8 +228,10 @@ public class TimeExtraction extends XYExtraction{
map.put("OutputTable", p); map.put("OutputTable", p);
if (pd!=null && signal!=null && signal.length>0){ if (pd!=null && signal!=null && signal.length>0){
HashMap<String, Image> producedImages = new HashMap<String, Image>(); HashMap<String, Image> producedImages = new HashMap<String, Image>();
if (signalimage!=null)
producedImages.put("Time Series Visualization", signalimage); producedImages.put("Time Series Visualization", signalimage);
producedImages.put("Spectrogram", spectrogramImage); if (spectrogramImage!=null)
producedImages.put("Spectrogram", spectrogramImage);
/* /*
try { try {
ImageIO.write(ImageTools.toBufferedImage(signalimage), "png", new File("signal.png")); ImageIO.write(ImageTools.toBufferedImage(signalimage), "png", new File("signal.png"));
@ -239,7 +252,9 @@ public class TimeExtraction extends XYExtraction{
map.put("Images", images); map.put("Images", images);
} }
else
map.put("Note", new PrimitiveType(String.class.getName(), "The signal contains only one point. The charts will not be displayed.", PrimitiveTypes.STRING,"Note","Note about the signal"));
// generate a primitive type for the collection // generate a primitive type for the collection
PrimitiveType outputm = new PrimitiveType(HashMap.class.getName(), map, PrimitiveTypes.MAP, "ResultsMap", "Results Map"); PrimitiveType outputm = new PrimitiveType(HashMap.class.getName(), map, PrimitiveTypes.MAP, "ResultsMap", "Results Map");

2
src/main/java/org/gcube/dataanalysis/geo/algorithms/TimeExtractionTable.java
View File

@ -36,7 +36,7 @@ public class TimeExtractionTable extends TimeExtraction{
inputs.add(columnx); inputs.add(columnx);
ColumnType columny = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.yDimensionColumnParameter, "The column containing y (latitude) information", "y", false); ColumnType columny = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.yDimensionColumnParameter, "The column containing y (latitude) information", "y", false);
inputs.add(columny); inputs.add(columny);
ColumnType columnt = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.timeDimensionColumnParameter, "The column containing y (latitude) information", "y", false); ColumnType columnt = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.timeDimensionColumnParameter, "The column containing time information", "datetime", false);
inputs.add(columnt); inputs.add(columnt);
ColumnType columnvalue = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.valueDimensionColumnParameter, "A column containing real valued features", "value", false); ColumnType columnvalue = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.valueDimensionColumnParameter, "A column containing real valued features", "value", false);
inputs.add(columnvalue); inputs.add(columnvalue);

34
src/main/java/org/gcube/dataanalysis/geo/algorithms/XYExtraction.java
View File

@ -1,6 +1,7 @@
package org.gcube.dataanalysis.geo.algorithms; package org.gcube.dataanalysis.geo.algorithms;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
@ -10,11 +11,14 @@ import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper; import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory;
import org.gcube.dataanalysis.geo.interfaces.GISDataConnector;
import org.gcube.dataanalysis.geo.matrixmodel.RasterTable; import org.gcube.dataanalysis.geo.matrixmodel.RasterTable;
import org.gcube.dataanalysis.geo.matrixmodel.XYExtractor; import org.gcube.dataanalysis.geo.matrixmodel.XYExtractor;
@ -59,13 +63,13 @@ public class XYExtraction implements Transducerer {
@Override @Override
public String getDescription() { public String getDescription() {
return "An algorithm to extract values associated to an environmental feature repository (e.g. NETCDF, ASC, GeoTiff files etc. ). A grid of points at a certain resolution is specified by the user and values are associated to the points from the environmental repository. " + "It accepts as one geospatial repository ID (via their UUIDs in the infrastructure spatial data repository - recoverable through the Geoexplorer portlet) " + "and the specification about time and space. The algorithm produces one table containing the values associated to the selected bounding box."; return "An algorithm to extract values associated to an environmental feature repository (e.g. NETCDF, ASC, GeoTiff files etc. ). A grid of points at a certain resolution is specified by the user and values are associated to the points from the environmental repository. " + "It accepts as one geospatial repository ID (via their UUIDs in the infrastructure spatial data repository - recoverable through the Geoexplorer portlet) or a direct link to a file " + "and the specification about time and space. The algorithm produces one table containing the values associated to the selected bounding box.";
} }
@Override @Override
public List<StatisticalType> getInputParameters() { public List<StatisticalType> getInputParameters() {
IOHelper.addStringInput(inputs, layerName, "Layer Title or UUID: The title or the UUID (preferred) of a layer indexed in the e-Infrastructure on GeoNetwork - You can retrieve it from GeoExplorer. Supports several standards (NETCDF, WFS, WCS ASC, GeoTiff )", ""); IOHelper.addStringInput(inputs, layerName, "Layer Title or UUID or HTTP link. E.g. the title or the UUID (preferred) of a layer indexed in the e-Infrastructure on GeoNetwork - You can retrieve it from GeoExplorer. Otherwise you can supply the direct HTTP link of the layer. The format will be guessed from the link. The default is GeoTiff. Supports several standards (NETCDF, WFS, WCS ASC, GeoTiff )", "");
IOHelper.addDoubleInput(inputs, yLL, "Lower Left Latitute of the Bounding Box", "-60"); IOHelper.addDoubleInput(inputs, yLL, "Lower Left Latitute of the Bounding Box", "-60");
IOHelper.addDoubleInput(inputs, xLL, "Lower Left Longitude of the Bounding Box", "-50"); IOHelper.addDoubleInput(inputs, xLL, "Lower Left Longitude of the Bounding Box", "-50");
IOHelper.addDoubleInput(inputs, yUR, "Upper Right Latitute of the Bounding Box", "60"); IOHelper.addDoubleInput(inputs, yUR, "Upper Right Latitute of the Bounding Box", "60");
@ -73,8 +77,8 @@ public class XYExtraction implements Transducerer {
IOHelper.addRandomStringInput(inputs, tableName, "The db name of the table to produce", "extr_"); IOHelper.addRandomStringInput(inputs, tableName, "The db name of the table to produce", "extr_");
IOHelper.addStringInput(inputs, tableLabel, "The name of the table to produce", "extr_"); IOHelper.addStringInput(inputs, tableLabel, "The name of the table to produce", "extr_");
IOHelper.addDoubleInput(inputs, z, "Value of Z. Default is 0, that means processing will be at surface level", "0"); IOHelper.addDoubleInput(inputs, z, "Value of Z. Default is 0, that means processing will be at surface level or at the first avaliable Z value in the layer", "0");
IOHelper.addIntegerInput(inputs, t, "Time Index. The default is the first", "0"); IOHelper.addIntegerInput(inputs, t, "Time Index. The default is the first time indexed dataset", "0");
IOHelper.addDoubleInput(inputs, xRes, "Projection resolution on the X axis", "0.5"); IOHelper.addDoubleInput(inputs, xRes, "Projection resolution on the X axis", "0.5");
IOHelper.addDoubleInput(inputs, yRes, "Projection resolution on the Y axis", "0.5"); IOHelper.addDoubleInput(inputs, yRes, "Projection resolution on the Y axis", "0.5");
@ -144,12 +148,20 @@ public class XYExtraction implements Transducerer {
AnalysisLogger.getLogger().debug("Extractor: MatrixExtractor initialized"); AnalysisLogger.getLogger().debug("Extractor: MatrixExtractor initialized");
long t0 = System.currentTimeMillis(); long t0 = System.currentTimeMillis();
XYExtractor extractor = new XYExtractor(config); XYExtractor extractor = new XYExtractor(config);
zValue = extractor.correctZ(zValue, layerNameValue, xResValue);
AnalysisLogger.getLogger().debug("XYExtraction->Best Z for this reference layer: " + zValue);
outputParameters.put("Matching Z value in the layer", ""+zValue);
outputParameters.put("Min Z value in the Layer", ""+extractor.zmin);
outputParameters.put("Max Z value in the Layer", ""+extractor.zmax);
double[][] matrix = extractor.extractXYGrid(layerNameValue, time, BBxLL, BBxUR, BByLL, BByUR, zValue, xResValue, yResValue); double[][] matrix = extractor.extractXYGrid(layerNameValue, time, BBxLL, BBxUR, BByLL, BByUR, zValue, xResValue, yResValue);
System.out.println("ELAPSED TIME: " + (System.currentTimeMillis() - t0)); System.out.println("ELAPSED TIME: " + (System.currentTimeMillis() - t0));
AnalysisLogger.getLogger().debug("Extractor: Matrix Extracted"); AnalysisLogger.getLogger().debug("Extractor: Matrix Extracted");
AnalysisLogger.getLogger().debug("Extractor: ****Rasterizing grid into table****"); AnalysisLogger.getLogger().debug("Extractor: ****Rasterizing grid into table****");
status = 30; status = 30;
RasterTable raster = new RasterTable(BBxLL, BBxUR, BByLL, BByUR, zValue, xResValue, yResValue, matrix, config); RasterTable raster = new RasterTable(BBxLL, BBxUR, BByLL, BByUR, zValue, time, xResValue, yResValue, matrix, config);
raster.setTablename(tableNameValue); raster.setTablename(tableNameValue);
raster.deleteTable(); raster.deleteTable();
raster.dumpGeoTable(); raster.dumpGeoTable();
@ -177,7 +189,17 @@ public class XYExtraction implements Transducerer {
List<TableTemplates> templateHspec = new ArrayList<TableTemplates>(); List<TableTemplates> templateHspec = new ArrayList<TableTemplates>();
templateHspec.add(TableTemplates.GENERIC); templateHspec.add(TableTemplates.GENERIC);
OutputTable p = new OutputTable(templateHspec, tableLabelValue, tableNameValue, "Output table"); OutputTable p = new OutputTable(templateHspec, tableLabelValue, tableNameValue, "Output table");
return p; LinkedHashMap<String, StatisticalType> map = new LinkedHashMap<String, StatisticalType>();
for (String key:outputParameters.keySet()){
String value = outputParameters.get(key);
PrimitiveType val = new PrimitiveType(String.class.getName(), "" + value, PrimitiveTypes.STRING, key, key);
map.put(key, val);
}
map.put("OutputTable", p);
PrimitiveType outputm = new PrimitiveType(HashMap.class.getName(), map, PrimitiveTypes.MAP, "ResultsMap", "Results Map");
return outputm;
} }
@Override @Override

2
src/main/java/org/gcube/dataanalysis/geo/algorithms/XYExtractionTable.java
View File

@ -48,6 +48,8 @@ public class XYExtractionTable extends XYExtraction{
inputs.add(previnputs.get(4)); inputs.add(previnputs.get(4));
inputs.add(previnputs.get(5)); inputs.add(previnputs.get(5));
inputs.add(previnputs.get(6)); inputs.add(previnputs.get(6));
inputs.add(previnputs.get(7));
inputs.add(previnputs.get(8));
inputs.add(previnputs.get(9)); inputs.add(previnputs.get(9));
inputs.add(previnputs.get(10)); inputs.add(previnputs.get(10));

5
src/main/java/org/gcube/dataanalysis/geo/algorithms/ZExtraction.java
View File

@ -100,6 +100,8 @@ public class ZExtraction extends XYExtraction{
AnalysisLogger.getLogger().debug("Extracting Time Series from layer"); AnalysisLogger.getLogger().debug("Extracting Time Series from layer");
ZExtractor extractor = new ZExtractor(config); ZExtractor extractor = new ZExtractor(config);
extractor.correctZ(0, layerNameValue,resolutionValue);
long t0 = System.currentTimeMillis(); long t0 = System.currentTimeMillis();
signal = extractor.extractZ(layerNameValue, xValue,yValue, time, resolutionValue); signal = extractor.extractZ(layerNameValue, xValue,yValue, time, resolutionValue);
@ -117,7 +119,7 @@ public class ZExtraction extends XYExtraction{
double matrix[][] = new double[1][]; double matrix[][] = new double[1][];
matrix[0] = signal; matrix[0] = signal;
RasterTable raster = new RasterTable(xValue, xValue, yValue, yValue, zValue, resolutionValue, resolutionValue, matrix, config); RasterTable raster = new RasterTable(xValue, xValue, yValue, yValue, zValue, time,resolutionValue, resolutionValue, matrix, config);
int signalRate = 1; int signalRate = 1;
@ -161,6 +163,7 @@ public class ZExtraction extends XYExtraction{
List<TableTemplates> templateHspec = new ArrayList<TableTemplates>(); List<TableTemplates> templateHspec = new ArrayList<TableTemplates>();
templateHspec.add(TableTemplates.GENERIC); templateHspec.add(TableTemplates.GENERIC);
OutputTable p = new OutputTable(templateHspec, tableLabelValue, tableNameValue, "Output table"); OutputTable p = new OutputTable(templateHspec, tableLabelValue, tableNameValue, "Output table");
map.put("OutputTable", p); map.put("OutputTable", p);
if (signalimage!=null){ if (signalimage!=null){

2
src/main/java/org/gcube/dataanalysis/geo/algorithms/ZExtractionTable.java
View File

@ -36,7 +36,7 @@ public class ZExtractionTable extends ZExtraction{
inputs.add(columnx); inputs.add(columnx);
ColumnType columny = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.yDimensionColumnParameter, "The column containing y (latitude) information", "y", false); ColumnType columny = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.yDimensionColumnParameter, "The column containing y (latitude) information", "y", false);
inputs.add(columny); inputs.add(columny);
ColumnType columnt = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.zDimensionColumnParameter, "The column containing z information", "datetime", false); ColumnType columnt = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.zDimensionColumnParameter, "The column containing z information", "z", false);
inputs.add(columnt); inputs.add(columnt);
ColumnType columnvalue = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.valueDimensionColumnParameter, "A column containing real valued features", "value", false); ColumnType columnvalue = new ColumnType(TableMatrixRepresentation.tableNameParameter, TableMatrixRepresentation.valueDimensionColumnParameter, "A column containing real valued features", "value", false);
inputs.add(columnvalue); inputs.add(columnvalue);

11
src/main/java/org/gcube/dataanalysis/geo/connectors/asc/AscRasterReader.java
View File

@ -11,6 +11,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.tools.ant.types.CommandlineJava.SysProperties; import org.apache.tools.ant.types.CommandlineJava.SysProperties;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
/** /**
@ -44,17 +45,19 @@ public class AscRasterReader
BufferedReader input = null; BufferedReader input = null;
URLConnection urlConn =null; URLConnection urlConn =null;
if (filename.startsWith("http")){ if (filename.startsWith("http")){
AnalysisLogger.getLogger().debug("Getting file from http");
URL fileurl = new URL(filename); URL fileurl = new URL(filename);
urlConn = fileurl.openConnection(); urlConn = fileurl.openConnection();
urlConn.setConnectTimeout(60000); urlConn.setConnectTimeout(120000);
urlConn.setReadTimeout(60000); urlConn.setReadTimeout(1200000);
urlConn.setAllowUserInteraction(false); urlConn.setAllowUserInteraction(false);
urlConn.setDoOutput(true); urlConn.setDoOutput(true);
input = new BufferedReader(new InputStreamReader(urlConn.getInputStream())); input = new BufferedReader(new InputStreamReader(urlConn.getInputStream()));
} }
else else {
AnalysisLogger.getLogger().debug("Getting file from local file");
input = new BufferedReader( new FileReader( filename ) ); input = new BufferedReader( new FileReader( filename ) );
}
while( input.ready() ) while( input.ready() )
{ {
String line = input.readLine(); String line = input.readLine();

5
src/main/java/org/gcube/dataanalysis/geo/connectors/netcdf/NetCDFDataExplorer.java
View File

@ -98,8 +98,9 @@ public class NetCDFDataExplorer {
List<GridDatatype> gridTypes = gds.getGrids(); List<GridDatatype> gridTypes = gds.getGrids();
for (GridDatatype gdt : gridTypes) { for (GridDatatype gdt : gridTypes) {
AnalysisLogger.getLogger().debug("Inside File - layer name: " + gdt.getName()); AnalysisLogger.getLogger().debug("Inside File - layer name: " + gdt.getName()+" layer to find "+layer);
if (layer.equalsIgnoreCase(gdt.getName())) { //if the layer is an HTTP link then take the first innser layer
if (layer.equalsIgnoreCase(gdt.getName()) || layer.toLowerCase().startsWith("http:")) {
AnalysisLogger.getLogger().debug("Found layer " + layer + " inside file"); AnalysisLogger.getLogger().debug("Found layer " + layer + " inside file");
GridDatatype grid = gds.findGridDatatype(gdt.getName()); GridDatatype grid = gds.findGridDatatype(gdt.getName());
CoordinateAxis zAxis = gdt.getCoordinateSystem().getVerticalAxis(); CoordinateAxis zAxis = gdt.getCoordinateSystem().getVerticalAxis();

1
src/main/java/org/gcube/dataanalysis/geo/connectors/table/Table.java
View File

@ -41,6 +41,7 @@ public class Table implements GISDataConnector {
throw new Exception("Error in getting elements for time " + time); throw new Exception("Error in getting elements for time " + time);
} }
AnalysisLogger.getLogger().debug("Getting elements for time " + time);
// check z: if there is at least one point inside the z boundary then it is ok // check z: if there is at least one point inside the z boundary then it is ok
boolean outsideZ = true; boolean outsideZ = true;

12
src/main/java/org/gcube/dataanalysis/geo/connectors/table/TableMatrixRepresentation.java
View File

@ -46,23 +46,23 @@ public class TableMatrixRepresentation {
String dbtuple = ""; String dbtuple = "";
if (xField != null && xField.trim().length()>0) if (xField != null && xField.trim().length()>0)
dbtuple += xField + ","; dbtuple += "\""+xField + "\" as x,";
else else
dbtuple += "0 as x,"; dbtuple += "0 as x,";
if (yField != null && yField.trim().length()>0) if (yField != null && yField.trim().length()>0)
dbtuple += yField + ","; dbtuple += "\""+yField + "\" as y,";
else else
dbtuple += "0 as y,"; dbtuple += "0 as y,";
if (zField != null && zField.trim().length()>0) if (zField != null && zField.trim().length()>0)
dbtuple += zField + ","; dbtuple += "\""+zField + "\" as z,";
else else
dbtuple += "0 as z,"; dbtuple += "0 as z,";
if (tField != null && tField.trim().length()>0) if (tField != null && tField.trim().length()>0)
dbtuple += tField + " as time,"; dbtuple += "\""+tField + "\" as time,";
else else
dbtuple += "0 as time,"; dbtuple += "0 as time,";
if (valueField != null && valueField.trim().length()>0) if (valueField != null && valueField.trim().length()>0)
dbtuple += valueField; dbtuple += "\""+valueField+"\" as v";
else else
dbtuple += "0 as v"; dbtuple += "0 as v";
@ -73,7 +73,7 @@ public class TableMatrixRepresentation {
//find maxZ //find maxZ
if (zField!=null && zField.trim().length()>0){ if (zField!=null && zField.trim().length()>0){
String maxzq = "select max("+zField+"),min("+zField+") from "+tableName; String maxzq = "select max("+zField+") as max,min("+zField+") as min from "+tableName;
Object [] maxzr = (Object [] )DatabaseFactory.executeSQLQuery(maxzq, dbconnection).get(0); Object [] maxzr = (Object [] )DatabaseFactory.executeSQLQuery(maxzq, dbconnection).get(0);
maxZ = Double.parseDouble(""+maxzr[0]); maxZ = Double.parseDouble(""+maxzr[0]);
minZ = Double.parseDouble(""+maxzr[1]); minZ = Double.parseDouble(""+maxzr[1]);

146
src/main/java/org/gcube/dataanalysis/geo/matrixmodel/MatrixExtractor.java
View File

@ -1,6 +1,7 @@
package org.gcube.dataanalysis.geo.matrixmodel; package org.gcube.dataanalysis.geo.matrixmodel;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
@ -16,6 +17,7 @@ import org.gcube.dataanalysis.geo.connectors.wcs.WCS;
import org.gcube.dataanalysis.geo.connectors.wfs.WFS; import org.gcube.dataanalysis.geo.connectors.wfs.WFS;
import org.gcube.dataanalysis.geo.infrastructure.GeoNetworkInspector; import org.gcube.dataanalysis.geo.infrastructure.GeoNetworkInspector;
import org.gcube.dataanalysis.geo.interfaces.GISDataConnector; import org.gcube.dataanalysis.geo.interfaces.GISDataConnector;
import org.gcube.dataanalysis.geo.meta.GenericLayerMetadata;
import org.opengis.metadata.Metadata; import org.opengis.metadata.Metadata;
import org.opengis.metadata.identification.Identification; import org.opengis.metadata.identification.Identification;
@ -26,13 +28,14 @@ public class MatrixExtractor {
public static int maxSignalLength = 100000; public static int maxSignalLength = 100000;
public static int maxzLength = 100000; public static int maxzLength = 100000;
boolean log = true; boolean log = true;
protected double currentResolution=0.5; protected double currentResolution = 0.5;
public List<Double> currentTimeValues; public List<Double> currentTimeValues;
public List<Tuple<Double>> currentTuples; public List<Tuple<Double>> currentTuples;
public MatrixExtractor(AlgorithmConfiguration configuration) { public MatrixExtractor(AlgorithmConfiguration configuration) {
gnInspector = new GeoNetworkInspector(); gnInspector = new GeoNetworkInspector();
gnInspector.setScope(configuration.getGcubeScope()); gnInspector.setScope(configuration.getGcubeScope());
this.configuration=configuration; this.configuration = configuration;
} }
public GeoNetworkInspector getFeaturer() { public GeoNetworkInspector getFeaturer() {
@ -49,12 +52,11 @@ public class MatrixExtractor {
protected List<Double> getRawValuesInTimeInstantAndBoundingBox(String layerTitle, int time, List<Tuple<Double>> coordinates3d, double xL, double xR, double yL, double yR, double resolution) throws Exception { protected List<Double> getRawValuesInTimeInstantAndBoundingBox(String layerTitle, int time, List<Tuple<Double>> coordinates3d, double xL, double xR, double yL, double yR, double resolution) throws Exception {
return getRawValuesInTimeInstantAndBoundingBox(layerTitle, time, coordinates3d, xL, xR, yL, yR, resolution, false); return getRawValuesInTimeInstantAndBoundingBox(layerTitle, time, coordinates3d, xL, xR, yL, yR, resolution, false);
} }
public GISDataConnector currentconnector; public GISDataConnector currentconnector;
public String layerName; public String layerName;
public String layerURL; public String layerURL;
public GISDataConnector getConnector(String layerTitle, double resolution) throws Exception { public GISDataConnector getConnector(String layerTitle, double resolution) throws Exception {
// get the layer // get the layer
Metadata meta = null; Metadata meta = null;
@ -64,62 +66,87 @@ public class MatrixExtractor {
else { else {
if (isTable()) { if (isTable()) {
AnalysisLogger.getLogger().debug("Extracting grid from table " + configuration.getParam(TableMatrixRepresentation.tableNameParameter)); AnalysisLogger.getLogger().debug("Extracting grid from table " + configuration.getParam(TableMatrixRepresentation.tableNameParameter));
connector = new Table(configuration,resolution); connector = new Table(configuration, resolution);
currentconnector = connector; currentconnector = connector;
} else } else {
meta = gnInspector.getGNInfobyUUIDorName(layerTitle); meta = gnInspector.getGNInfobyUUIDorName(layerTitle);
}
// if the layer is good // if the layer is not on GeoNetwork
if (meta != null) { if (meta == null) {
layerName = gnInspector.getLayerName(meta); String[] urls = { layerTitle };
if (layerName == null) String[] protocols = { "HTTP" };
layerName = layerTitle; meta = new GenericLayerMetadata().createBasicMeta(urls, protocols);
layerURL = ""; }
if (gnInspector.isNetCDFFile(meta)) { layerName = gnInspector.getLayerName(meta);
Identification id = meta.getIdentificationInfo().iterator().next(); if (layerName == null)
String title = id.getCitation().getTitle().toString(); layerName = layerTitle;
AnalysisLogger.getLogger().debug("found a netCDF file with title " + title + " and layer name " + layerName); layerURL = "";
layerURL = gnInspector.getOpenDapLink(meta); if (gnInspector.isNetCDFFile(meta)) {
connector = new NetCDF(layerURL, layerName); Identification id = meta.getIdentificationInfo().iterator().next();
} else if (gnInspector.isAscFile(meta)) { String title = id.getCitation().getTitle().toString();
AnalysisLogger.getLogger().debug("managing ASC File"); AnalysisLogger.getLogger().debug("found a netCDF file with title " + title + " and layer name " + layerName);
layerURL = gnInspector.getHttpLink(meta); layerURL = gnInspector.getOpenDapLink(meta);
connector = new ASC(); connector = new NetCDF(layerURL, layerName);
} else if (gnInspector.isWFS(meta)) { } else if (gnInspector.isAscFile(meta)) {
AnalysisLogger.getLogger().debug("found a Geo Layer with reference " + layerTitle + " and layer name " + layerName); AnalysisLogger.getLogger().debug("managing ASC File");
layerURL = gnInspector.getGeoserverLink(meta); layerURL = gnInspector.getHttpLink(meta);
connector = new WFS(); connector = new ASC();
} else if (gnInspector.isWCS(meta)) { } else if (gnInspector.isWFS(meta)) {
AnalysisLogger.getLogger().debug("found a WCS Layer with reference " + layerTitle + " and layer name " + layerName); AnalysisLogger.getLogger().debug("found a Geo Layer with reference " + layerURL + " and layer name " + layerName);
layerURL = gnInspector.getWCSLink(meta); layerURL = gnInspector.getGeoserverLink(meta);
connector = new WCS(configuration,layerURL); connector = new WFS();
} else if (gnInspector.isGeoTiff(meta)) { } else if (gnInspector.isWCS(meta)) {
layerURL = gnInspector.getGeoTiffLink(meta); AnalysisLogger.getLogger().debug("found a WCS Layer with reference " + layerURL + " and layer name " + layerName);
AnalysisLogger.getLogger().debug("found a GeoTiff with reference " + layerTitle + " and layer name " + layerName); layerURL = gnInspector.getWCSLink(meta);
connector = new GeoTiff(configuration,layerURL); connector = new WCS(configuration, layerURL);
} else if (gnInspector.isGeoTiff(meta)) {
layerURL = gnInspector.getGeoTiffLink(meta);
AnalysisLogger.getLogger().debug("found a GeoTiff with reference " + layerURL + " and layer name " + layerName);
connector = new GeoTiff(configuration, layerURL);
} else {
// treat as geotiff
layerURL = layerTitle;
AnalysisLogger.getLogger().debug("guessing a GeoTiff with reference " + layerURL + " and layer name " + layerName);
connector = new GeoTiff(configuration, layerURL);
}
} }
currentconnector = connector;
} }
currentconnector = connector;
return currentconnector; return currentconnector;
} }
//4D Extraction // 4D Extraction
/** /**
* Extract raw values in a time instant according to a set of grid points and a bounding box * Extract raw values in a time instant according to a set of grid points and a bounding box
*/ */
public List<Double> getRawValuesInTimeInstantAndBoundingBox(String layerTitle, int time, List<Tuple<Double>> coordinates3d, double xL, double xR, double yL, double yR, double resolution, boolean saveLayer) throws Exception { public List<Double> getRawValuesInTimeInstantAndBoundingBox(String layerTitle, int time, List<Tuple<Double>> coordinates3d, double xL, double xR, double yL, double yR, double resolution, boolean saveLayer) throws Exception {
GISDataConnector connector = getConnector(layerTitle,resolution); GISDataConnector connector = getConnector(layerTitle, resolution);
//execute connector // execute connector
if (connector != null) if (connector != null)
return connector.getFeaturesInTimeInstantAndArea(layerURL, layerName, time, coordinates3d, xL, xR, yL, yR); return connector.getFeaturesInTimeInstantAndArea(layerURL, layerName, time, coordinates3d, xL, xR, yL, yR);
else else
throw new Exception("ERROR: Connector not found for layer " + layerTitle + " - Cannot Rasterize!"); throw new Exception("ERROR: Connector not found for layer " + layerTitle + " - Cannot Rasterize!");
} }
public double zmin;
public double zmax;
public double correctZ(double zValue, String layerURL, double resolution) throws Exception{
GISDataConnector connector = getConnector(layerURL, resolution);
zmin = connector.getMinZ(layerURL, layerName);
zmax = connector.getMaxZ(layerURL, layerName);
if (zValue<zmin)
zValue = zmin;
else if (zValue>zmax)
zValue = zmax;
return zValue;
}
/** /**
* Extract a grid of XY points with fixed time and z * Extract a grid of XY points with fixed time and z
*
* @param layerTitle * @param layerTitle
* @param timeInstant * @param timeInstant
* @param x1 * @param x1
@ -135,11 +162,11 @@ public class MatrixExtractor {
*/ */
public double[][] extractXYGridWithFixedTZ(String layerTitle, int timeInstant, double x1, double x2, double y1, double y2, double z, double xResolution, double yResolution, boolean cachelayer) throws Exception { public double[][] extractXYGridWithFixedTZ(String layerTitle, int timeInstant, double x1, double x2, double y1, double y2, double z, double xResolution, double yResolution, boolean cachelayer) throws Exception {
currentResolution = (double)(xResolution+yResolution)/2d; currentResolution = (double) (xResolution + yResolution) / 2d;
boolean faolayer = false; boolean faolayer = false;
if (layerTitle==null) if (layerTitle == null)
layerTitle=""; layerTitle = "";
if (layerTitle.toLowerCase().contains("MatrixExtractor->FAO aquatic species distribution map")) { if (layerTitle.toLowerCase().contains("MatrixExtractor->FAO aquatic species distribution map")) {
AnalysisLogger.getLogger().debug("MatrixExtractor->FAO DISTRIBUTION LAYER ... TO APPY PATCH!"); AnalysisLogger.getLogger().debug("MatrixExtractor->FAO DISTRIBUTION LAYER ... TO APPY PATCH!");
faolayer = true; faolayer = true;
@ -153,12 +180,12 @@ public class MatrixExtractor {
double[][] slice = new double[ysteps + 1][xsteps + 1]; double[][] slice = new double[ysteps + 1][xsteps + 1];
List<Tuple<Double>> tuples = new ArrayList<Tuple<Double>>(); List<Tuple<Double>> tuples = new ArrayList<Tuple<Double>>();
if (log){ if (log) {
AnalysisLogger.getLogger().debug("MatrixExtractor->Building the points grid according to YRes:" + yResolution + " and XRes:" + xResolution); AnalysisLogger.getLogger().debug("MatrixExtractor->Building the points grid according to YRes:" + yResolution + " and XRes:" + xResolution);
AnalysisLogger.getLogger().debug("MatrixExtractor->Points to reassign:" + (ysteps * xsteps)); AnalysisLogger.getLogger().debug("MatrixExtractor->Points to reassign:" + (ysteps * xsteps));
} }
// build the tuples according to the desired resolution // build the tuples according to the desired resolution
for (int i = 0; i < ysteps + 1; i++) { for (int i = 0; i < ysteps + 1; i++) {
double y = (i * yResolution) + y1; double y = (i * yResolution) + y1;
@ -187,16 +214,15 @@ public class MatrixExtractor {
} }
if (log) if (log)
AnalysisLogger.getLogger().debug("Bounding box: (" + x1 + "," + x2 + ";" + y1 + "," + y2 + ")"); AnalysisLogger.getLogger().debug("Bounding box: (" + x1 + "," + x2 + ";" + y1 + "," + y2 + ")");
// long t0=System.currentTimeMillis(); // long t0=System.currentTimeMillis();
currentTimeValues = getRawValuesInTimeInstantAndBoundingBox(layerTitle, timeInstant, tuples, x1, x2, y1, y2, currentResolution, cachelayer); currentTimeValues = getRawValuesInTimeInstantAndBoundingBox(layerTitle, timeInstant, tuples, x1, x2, y1, y2, currentResolution, cachelayer);
currentTuples=tuples; currentTuples = tuples;
// AnalysisLogger.getLogger().debug("Elapsed:"+(System.currentTimeMillis()-t0)); // AnalysisLogger.getLogger().debug("Elapsed:"+(System.currentTimeMillis()-t0));
if (log) if (log)
AnalysisLogger.getLogger().debug("Taken " + currentTimeValues.size() + " values"); AnalysisLogger.getLogger().debug("Taken " + currentTimeValues.size() + " values");
// build back the values matrix // build back the values matrix
int k = 0; int k = 0;
int g = 0; int g = 0;
@ -224,10 +250,10 @@ public class MatrixExtractor {
} }
// applyNearestNeighbor(); // applyNearestNeighbor();
if (log) if (log)
AnalysisLogger.getLogger().debug("Features map: rows " + slice.length + ", cols " + slice[0].length); AnalysisLogger.getLogger().debug("Features map: rows " + slice.length + ", cols " + slice[0].length);
return slice; return slice;
} }
} }

67
src/main/java/org/gcube/dataanalysis/geo/test/infra/TestExtraction.java
View File

@ -12,8 +12,9 @@ import org.gcube.dataanalysis.geo.connectors.table.TableMatrixRepresentation;
public class TestExtraction { public class TestExtraction {
// static AlgorithmConfiguration[] configs = { testXYExtractionNetCDF(),testXYExtractionAquaMaps(),testXYExtractionTable()}; static AlgorithmConfiguration[] configs = { testXYExtractionNetCDF(),testXYExtractionAquaMaps(),testXYExtractionTable(),testXYExtractionTable2()};
static AlgorithmConfiguration[] configs = { testXYExtractionTable()}; // static AlgorithmConfiguration[] configs = { testXYExtractionTable2()};
// static AlgorithmConfiguration[] configs = { testDirectExtraction()};
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
System.out.println("TEST 1"); System.out.println("TEST 1");
@ -35,7 +36,7 @@ public class TestExtraction {
AlgorithmConfiguration config = new AlgorithmConfiguration(); AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("XYEXTRACTION"); config.setAgent("XYEXTRACTOR");
config.setConfigPath("./cfg/"); config.setConfigPath("./cfg/");
config.setPersistencePath("./"); config.setPersistencePath("./");
config.setParam("DatabaseUserName","gcube"); config.setParam("DatabaseUserName","gcube");
@ -59,6 +60,34 @@ public class TestExtraction {
return config; return config;
} }
private static AlgorithmConfiguration testDirectExtraction() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("XYEXTRACTOR");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName","gcube");
config.setParam("DatabasePassword","d4science2");
config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb");
config.setParam("DatabaseDriver","org.postgresql.Driver");
config.setGcubeScope("/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab");
config.setParam("Layer","https://dl.dropboxusercontent.com/u/12809149/geoserver-GetCoverage.image.asc");
config.setParam("Z","0");
config.setParam("TimeIndex","0");
config.setParam("BBox_LowerLeftLat","-60");
config.setParam("BBox_LowerLeftLong","-50");
config.setParam("BBox_UpperRightLat","60");
config.setParam("BBox_UpperRightLong","50");
config.setParam("XResolution","0.5");
config.setParam("YResolution","0.5");
config.setParam("OutputTableName","testextraction");
config.setParam("OutputTableLabel","testextraction");
return config;
}
private static AlgorithmConfiguration testXYExtractionAquaMaps() { private static AlgorithmConfiguration testXYExtractionAquaMaps() {
@ -98,4 +127,36 @@ public class TestExtraction {
return config; return config;
} }
private static AlgorithmConfiguration testXYExtractionTable2() {
AlgorithmConfiguration config = testXYExtractionNetCDF();
config.setAgent("XYEXTRACTOR_TABLE");
config.setParam("OutputTableName","testextractiontable");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
/*
config.setParam(TableMatrixRepresentation.tableNameParameter, "generic_id037d302d_2ba0_4e43_b6e4_1a797bb91728");
config.setParam(TableMatrixRepresentation.xDimensionColumnParameter, "x");
config.setParam(TableMatrixRepresentation.yDimensionColumnParameter, "y");
config.setParam(TableMatrixRepresentation.timeDimensionColumnParameter, "datetime");
config.setParam(TableMatrixRepresentation.valueDimensionColumnParameter, "speed");
config.setParam(TableMatrixRepresentation.filterParameter, "speed<2");
*/
config.setParam(TableMatrixRepresentation.tableNameParameter, "occurrence_species_id0045886b_2a7c_4ede_afc4_3157c694b893");
config.setParam(TableMatrixRepresentation.xDimensionColumnParameter, "decimallongitude");
config.setParam(TableMatrixRepresentation.yDimensionColumnParameter, "decimallatitude");
config.setParam(TableMatrixRepresentation.timeDimensionColumnParameter, "modified");
config.setParam(TableMatrixRepresentation.valueDimensionColumnParameter, "decimallatitude");
config.setParam(TableMatrixRepresentation.filterParameter, " ");
config.setParam("Z","0");
config.setParam("TimeIndex","1");
return config;
}
} }

71
src/main/java/org/gcube/dataanalysis/geo/test/infra/TestOccurrenceEnrichment.java Normal file
View File

@ -0,0 +1,71 @@
package org.gcube.dataanalysis.geo.test.infra;
import java.util.ArrayList;
import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveTypesList;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
public class TestOccurrenceEnrichment {
static AlgorithmConfiguration[] configs = { testOccEnrichment()};
public static void main(String[] args) throws Exception {
System.out.println("TEST 1");
for (int i = 0; i < configs.length; i++) {
AnalysisLogger.getLogger().debug("Executing: "+configs[i].getAgent());
List<ComputationalAgent> trans = null;
trans = TransducerersFactory.getTransducerers(configs[i]);
trans.get(0).init();
Regressor.process(trans.get(0));
StatisticalType st = trans.get(0).getOutput();
AnalysisLogger.getLogger().debug("ST:" + st);
trans = null;
}
}
private static AlgorithmConfiguration testOccEnrichment() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("OCCURRENCE_ENRICHMENT");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
config.setGcubeScope("/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab");
config.setParam("OccurrenceTable","occurrence_species_id0045886b_2a7c_4ede_afc4_3157c694b893");
config.setParam("LongitudeColumn","decimallongitude");
config.setParam("LatitudeColumn","decimallatitude");
config.setParam("ScientificNameColumn","scientificname");
config.setParam("TimeColumn","eventdate");
config.setParam("OptionalFilter","");
config.setParam("Resolution","0.5");
config.setParam("OutputTableDBName","testenrichment");
config.setParam("OutputTableName","testenrichment");
String sep=AlgorithmConfiguration.getListSeparator();
config.setParam("Layers","http://thredds.research-infrastructures.eu/thredds/dodsC/public/netcdf/global-reanalysis-phys-001-004-b-ref-fr-mjm95-gridt__ENVIRONMENT_OCEANS_ELEVATION_1366210702774.nc"+sep+"4d597da9-dbfa-4a65-9de6-9bbff69eac19"+sep+"2c2304d1-681a-4f3a-8409-e8cdb5ed447f");
// config.setParam("Layers","8f5d883f-95bf-4b7c-8252-aaf0b2e6fd81"+sep+"4d597da9-dbfa-4a65-9de6-9bbff69eac19"+sep+"2c2304d1-681a-4f3a-8409-e8cdb5ed447f");
config.setParam("FeaturesNames","temperature"+sep+"chlorophyll"+sep+"ph");
// config.setParam("Layers","4d597da9-dbfa-4a65-9de6-9bbff69eac19"+sep+"2c2304d1-681a-4f3a-8409-e8cdb5ed447f");
// config.setParam("FeaturesNames","chlorophyll"+sep+"ph");
return config;
}
}

12
src/main/java/org/gcube/dataanalysis/geo/test/infra/TestTimeExtraction.java
View File

@ -13,8 +13,8 @@ import org.gcube.dataanalysis.geo.connectors.table.TableMatrixRepresentation;
public class TestTimeExtraction { public class TestTimeExtraction {
static AlgorithmConfiguration[] configs = { testTimeExtractionTable(), testTimeExtractionNetCDF(),testTimeExtractionTable2()}; // static AlgorithmConfiguration[] configs = { testTimeExtractionTable(), testTimeExtractionNetCDF(),testTimeExtractionTable2()};
static AlgorithmConfiguration[] configs = { testTimeExtractionNetCDF()};
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
System.out.println("TEST 1"); System.out.println("TEST 1");
@ -43,11 +43,13 @@ public class TestTimeExtraction {
config.setParam("DatabasePassword","d4science2"); config.setParam("DatabasePassword","d4science2");
config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb"); config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb");
config.setParam("DatabaseDriver","org.postgresql.Driver"); config.setParam("DatabaseDriver","org.postgresql.Driver");
config.setGcubeScope("/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab"); // config.setGcubeScope("/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab");
config.setGcubeScope("/gcube/devsec/devVRE");
// config.setParam("Layer","c565e32c-c5b3-4964-b44f-06dc620563e9");
config.setParam("Layer","c565e32c-c5b3-4964-b44f-06dc620563e9");
// config.setParam("Layer","dffa504b-dbc8-4553-896e-002549f8f5d3"); // config.setParam("Layer","dffa504b-dbc8-4553-896e-002549f8f5d3");
//wind
config.setParam("Layer","21715b2e-28de-4646-acce-d4f16b59d6d0");
config.setParam("OutputTableName","testtimeextraction"); config.setParam("OutputTableName","testtimeextraction");
config.setParam("OutputTableLabel","testtimeextraction"); config.setParam("OutputTableLabel","testtimeextraction");

70
src/main/java/org/gcube/dataanalysis/geo/test/regression/RegressionOccurrenceEnrichment.java Normal file
View File

@ -0,0 +1,70 @@
package org.gcube.dataanalysis.geo.test.regression;
import java.util.ArrayList;
import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveTypesList;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
public class RegressionOccurrenceEnrichment {
static AlgorithmConfiguration[] configs = { testOccEnrichment()};
public static void main(String[] args) throws Exception {
System.out.println("TEST 1");
for (int i = 0; i < configs.length; i++) {
AnalysisLogger.getLogger().debug("Executing: "+configs[i].getAgent());
List<ComputationalAgent> trans = null;
trans = TransducerersFactory.getTransducerers(configs[i]);
trans.get(0).init();
Regressor.process(trans.get(0));
StatisticalType st = trans.get(0).getOutput();
AnalysisLogger.getLogger().debug("ST:" + st);
trans = null;
}
}
private static AlgorithmConfiguration testOccEnrichment() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("OCCURRENCE_ENRICHMENT");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
config.setGcubeScope("/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab");
config.setParam("OccurrenceTable","occurrence_species_id0045886b_2a7c_4ede_afc4_3157c694b893");
config.setParam("LongitudeColumn","decimallongitude");
config.setParam("LatitudeColumn","decimallatitude");
config.setParam("ScientificNameColumn","scientificname");
config.setParam("TimeColumn","eventdate");
config.setParam("OptionalFilter","");
config.setParam("Resolution","0.5");
config.setParam("OutputTableDBName","testenrichment");
config.setParam("OutputTableName","testenrichment");
String sep=AlgorithmConfiguration.getListSeparator();
config.setParam("Layers","8f5d883f-95bf-4b7c-8252-aaf0b2e6fd81"+sep+"4d597da9-dbfa-4a65-9de6-9bbff69eac19"+sep+"2c2304d1-681a-4f3a-8409-e8cdb5ed447f");
config.setParam("FeaturesNames","temperature"+sep+"chlorophyll"+sep+"ph");
// config.setParam("Layers","4d597da9-dbfa-4a65-9de6-9bbff69eac19"+sep+"2c2304d1-681a-4f3a-8409-e8cdb5ed447f");
// config.setParam("FeaturesNames","chlorophyll"+sep+"ph");
return config;
}
}

37
src/main/java/org/gcube/dataanalysis/geo/utils/VectorOperations.java
View File

@ -92,6 +92,42 @@ public class VectorOperations {
return valuesForGrid; return valuesForGrid;
} }
public static List<Double> assignGridValuesToPoints2D(List<Tuple<Double>> grid3d, List<Double> gridValues, List<Tuple<Double>> coordinates4d, double tolerance) {
List<Double> valuesForPoints = new ArrayList<Double>();
int gridSize = coordinates4d.size();
for (int i = 0; i < gridSize; i++) {
valuesForPoints.add(Double.NaN);
}
int foundmatches = 0;
int points=0;
for (Tuple<Double> coord4d : coordinates4d) {
double rx = coord4d.getElements().get(0);
double ry = coord4d.getElements().get(1);
int gridIdx = 0;
for (Tuple<Double> gridElement : grid3d) {
double x = gridElement.getElements().get(0);
double y = gridElement.getElements().get(1);
double d = distance(x, y, 0, 0, rx, ry, 0, 0);
if (d <= tolerance) {
// AnalysisLogger.getLogger().debug("Association: distance between grid:("+x+","+y+","+z+","+gridTimeInstant+") and point:("+rx+","+ry+","+rz+","+rt+") is "+d);
valuesForPoints.set(points, gridValues.get(gridIdx));
foundmatches++;
break;
}
gridIdx++;
}
points++;
}
AnalysisLogger.getLogger().debug("Association: Found "+foundmatches+" matches between the points and the grid");
return valuesForPoints;
}
public static List<Double> assignGridValuesToPoints(List<Tuple<Double>> grid3d, int gridTimeInstant, List<Double> gridValues, List<Tuple<Double>> coordinates4d, double tolerance) { public static List<Double> assignGridValuesToPoints(List<Tuple<Double>> grid3d, int gridTimeInstant, List<Double> gridValues, List<Tuple<Double>> coordinates4d, double tolerance) {
List<Double> valuesForPoints = new ArrayList<Double>(); List<Double> valuesForPoints = new ArrayList<Double>();
@ -116,7 +152,6 @@ public class VectorOperations {
double z = gridElement.getElements().get(2); double z = gridElement.getElements().get(2);
double d = distance(x, y, z, gridTimeInstant, rx, ry, rz, rt); double d = distance(x, y, z, gridTimeInstant, rx, ry, rz, rt);
if (d <= tolerance) { if (d <= tolerance) {
// AnalysisLogger.getLogger().debug("Association: distance between grid:("+x+","+y+","+z+","+gridTimeInstant+") and point:("+rx+","+ry+","+rz+","+rt+") is "+d); // AnalysisLogger.getLogger().debug("Association: distance between grid:("+x+","+y+","+z+","+gridTimeInstant+") and point:("+rx+","+ry+","+rz+","+rt+") is "+d);
valuesForPoints.set(points, gridValues.get(gridIdx)); valuesForPoints.set(points, gridValues.get(gridIdx));