git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@92162 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
677cabb537
commit
0d9bb0366f
|
@ -0,0 +1,195 @@
|
||||||
|
package org.gcube.dataanalysis.geo.matrixmodel;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.utils.Tuple;
|
||||||
|
import org.gcube.dataanalysis.geo.connectors.asc.ASC;
|
||||||
|
import org.gcube.dataanalysis.geo.connectors.interfaces.GISDataConnector;
|
||||||
|
import org.gcube.dataanalysis.geo.connectors.netcdf.NetCDF;
|
||||||
|
import org.gcube.dataanalysis.geo.connectors.wfs.WFS;
|
||||||
|
import org.gcube.dataanalysis.geo.infrastructure.GeoNetworkInspector;
|
||||||
|
import org.opengis.metadata.Metadata;
|
||||||
|
import org.opengis.metadata.identification.Identification;
|
||||||
|
|
||||||
|
public class MatrixExtractor {
|
||||||
|
|
||||||
|
private GeoNetworkInspector gnInspector;
|
||||||
|
private String configDir;
|
||||||
|
|
||||||
|
public MatrixExtractor(String scope, String cfgDir) {
|
||||||
|
gnInspector = new GeoNetworkInspector();
|
||||||
|
gnInspector.setScope(scope);
|
||||||
|
this.configDir=cfgDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
public GeoNetworkInspector getFeaturer(){
|
||||||
|
return gnInspector;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static List<Tuple<Double>> generateCoordinateTripletsInBoundingBox(double x1, double x2, double y1, double y2, double z, double xResolution, double yResolution){
|
||||||
|
|
||||||
|
int ysteps = (int) ((y2 - y1) / yResolution);
|
||||||
|
int xsteps = (int) ((x2 - x1) / xResolution);
|
||||||
|
List<Tuple<Double>> tuples = new ArrayList<Tuple<Double>>();
|
||||||
|
AnalysisLogger.getLogger().debug("Building the points grid according to YRes:" + yResolution + " and XRes:" + xResolution);
|
||||||
|
// build the tuples according to the desired resolution
|
||||||
|
for (int i = 0; i < ysteps + 1; i++) {
|
||||||
|
double y = (i * yResolution) + y1;
|
||||||
|
if (i == ysteps)
|
||||||
|
y = y2;
|
||||||
|
for (int j = 0; j < xsteps + 1; j++) {
|
||||||
|
double x = (j * xResolution) + x1;
|
||||||
|
if (j == xsteps)
|
||||||
|
x = x2;
|
||||||
|
tuples.add(new Tuple<Double>(x, y, z));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tuples;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected List<Double> getRawValuesInTimeInstantAndBoundingBox(String layerTitle, int time, List<Tuple<Double>> coordinates3d, double xL,double xR, double yL, double yR) throws Exception {
|
||||||
|
// get the layer
|
||||||
|
Metadata meta = gnInspector.getGNInfobyUUIDorName(layerTitle);
|
||||||
|
// if the layer is good
|
||||||
|
if (meta != null) {
|
||||||
|
String layer = gnInspector.getLayerName(meta);
|
||||||
|
if (layer == null)
|
||||||
|
layer = layerTitle;
|
||||||
|
String layerURL = "";
|
||||||
|
GISDataConnector connector = null;
|
||||||
|
if (gnInspector.isNetCDFFile(meta)) {
|
||||||
|
Identification id = meta.getIdentificationInfo().iterator().next();
|
||||||
|
String title = id.getCitation().getTitle().toString();
|
||||||
|
AnalysisLogger.getLogger().debug("found a netCDF file with title " + title + " and layer name " + layer);
|
||||||
|
layerURL = gnInspector.getOpenDapLink(meta);
|
||||||
|
connector = new NetCDF();
|
||||||
|
}
|
||||||
|
else if (gnInspector.isAscFile(meta)){
|
||||||
|
AnalysisLogger.getLogger().debug("managing ASC File");
|
||||||
|
layerURL = gnInspector.getHttpLink(meta);
|
||||||
|
connector = new ASC();
|
||||||
|
}
|
||||||
|
else if (gnInspector.isWFS(meta)){
|
||||||
|
AnalysisLogger.getLogger().debug("found a Geo Layer with reference " + layerTitle + " and layer name " + layer);
|
||||||
|
layerURL = gnInspector.getGeoserverLink(meta);
|
||||||
|
connector = new WFS();
|
||||||
|
}
|
||||||
|
if (connector!=null)
|
||||||
|
return connector.getFeaturesInTimeInstantAndArea(layerURL, layer, time, coordinates3d, xL, xR, yL, yR);
|
||||||
|
else
|
||||||
|
throw new Exception("ERROR: Connector not found for layer "+layerTitle+" - Cannot Rasterize!");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
throw new Exception("ERROR: Metadata not found for layer "+layerTitle+" - Cannot Rasterize!");
|
||||||
|
}
|
||||||
|
|
||||||
|
public double[][] takeTimeInstantMatrix(String layerTitle, int timeInstant, double x1, double x2, double y1, double y2, double z, double xResolution, double yResolution) throws Exception {
|
||||||
|
AnalysisLogger.getLogger().debug("Bounding box: (" + x1 + "," + x2 + ";" + y1 + "," + y2 + ")");
|
||||||
|
boolean faolayer = false;
|
||||||
|
if (layerTitle.toLowerCase().contains("MatrixExtractor->FAO aquatic species distribution map") )
|
||||||
|
{
|
||||||
|
AnalysisLogger.getLogger().debug("MatrixExtractor->FAO DISTRIBUTION LAYER ... TO APPY PATCH!");
|
||||||
|
faolayer=true;
|
||||||
|
}
|
||||||
|
if ((x2 < x1) || (y2 < y1)) {
|
||||||
|
AnalysisLogger.getLogger().debug("MatrixExtractor->ERROR: BAD BOUNDING BOX!!!");
|
||||||
|
return new double[0][0];
|
||||||
|
}
|
||||||
|
|
||||||
|
int ysteps = (int) ((y2 - y1) / yResolution);
|
||||||
|
int xsteps = (int) ((x2 - x1) / xResolution);
|
||||||
|
double[][] slice = new double[ysteps + 1][xsteps + 1];
|
||||||
|
List<Tuple<Double>> tuples = new ArrayList<Tuple<Double>>();
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("MatrixExtractor->Building the points grid according to YRes:" + yResolution + " and XRes:" + xResolution);
|
||||||
|
AnalysisLogger.getLogger().debug("MatrixExtractor->Points to reassign:" + (ysteps*xsteps));
|
||||||
|
|
||||||
|
// build the tuples according to the desired resolution
|
||||||
|
for (int i = 0; i < ysteps + 1; i++) {
|
||||||
|
double y = (i * yResolution) + y1;
|
||||||
|
if (i == ysteps)
|
||||||
|
y = y2;
|
||||||
|
for (int j = 0; j < xsteps + 1; j++) {
|
||||||
|
double x = (j * xResolution) + x1;
|
||||||
|
if (j == xsteps)
|
||||||
|
x = x2;
|
||||||
|
tuples.add(new Tuple<Double>(x, y, z));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("Taking " + ysteps + " values per "+xsteps+"="+(ysteps*xsteps)+ "...");
|
||||||
|
|
||||||
|
List<Double> timeValues = getRawValuesInTimeInstantAndBoundingBox(layerTitle, timeInstant, tuples, x1, x2, y1,y2);
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("Taken " + timeValues.size() + " values");
|
||||||
|
|
||||||
|
// build back the values matrix
|
||||||
|
int k = 0;
|
||||||
|
int g = 0;
|
||||||
|
int ntriplets = timeValues.size();
|
||||||
|
//cycle on all the triplets to recontruct the matrix
|
||||||
|
for (int t = 0; t < ntriplets; t++) {
|
||||||
|
//take the corresponding (time,value) pair
|
||||||
|
Double value = timeValues.get(t);
|
||||||
|
//if there is value, then set it, otherwise set NaN
|
||||||
|
//the layer is undefined in that point and a value must be generated
|
||||||
|
//assign a value to the matrix
|
||||||
|
|
||||||
|
//WARNING: PATCH FOR FAO LAYERS:. Probability can be equal to 2 for uncertainty
|
||||||
|
if (faolayer && (value>1)){
|
||||||
|
AnalysisLogger.getLogger().debug("APPLYING FAO PATCH!");
|
||||||
|
slice[k][g] = 0.5;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
slice[k][g] = value;
|
||||||
|
//increase the x step according to the matrix
|
||||||
|
if (g == xsteps) {
|
||||||
|
g = 0;
|
||||||
|
k++;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
g++;
|
||||||
|
}
|
||||||
|
|
||||||
|
//applyNearestNeighbor();
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("Features map: "+slice.length+","+slice[0].length);
|
||||||
|
return slice;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void applyNearestNeighbor(){
|
||||||
|
|
||||||
|
/*
|
||||||
|
AnalysisLogger.getLogger().debug("Applying nearest Neighbor to all the rows");
|
||||||
|
//apply nearest neighbor to each row
|
||||||
|
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||||
|
config.setConfigPath(configDir);
|
||||||
|
boolean rapidinit = false;
|
||||||
|
|
||||||
|
|
||||||
|
for (int i=0;i<slice.length;i++){
|
||||||
|
// AnalysisLogger.getLogger().debug("Checking for unfilled values");
|
||||||
|
boolean tofill = false;
|
||||||
|
for (int j=0;j<slice[i].length;j++) {
|
||||||
|
if (new Double(slice[i][j]).equals(Double.NaN))
|
||||||
|
tofill = true;
|
||||||
|
}
|
||||||
|
if (tofill){
|
||||||
|
if (!rapidinit){
|
||||||
|
config.initRapidMiner();
|
||||||
|
rapidinit=true;
|
||||||
|
}
|
||||||
|
AnalysisLogger.getLogger().debug("Filling signal");
|
||||||
|
double[] ssliced = SignalProcessing.fillSignal(slice[i]);
|
||||||
|
slice[i] = ssliced;
|
||||||
|
}
|
||||||
|
// else
|
||||||
|
// AnalysisLogger.getLogger().debug("Signal yet complete");
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,148 @@
|
||||||
|
package org.gcube.dataanalysis.geo.matrixmodel;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||||
|
import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.utils.Tuple;
|
||||||
|
import org.gcube.dataanalysis.geo.utils.CSquareCodesConverter;
|
||||||
|
import org.hibernate.SessionFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* transforms a raster map into a table
|
||||||
|
*
|
||||||
|
* @author coro
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class RasterTable {
|
||||||
|
|
||||||
|
private double valuesMatrix[][];
|
||||||
|
double x1;
|
||||||
|
double x2;
|
||||||
|
double y1;
|
||||||
|
double y2;
|
||||||
|
double z;
|
||||||
|
double xResolution;
|
||||||
|
double yResolution;
|
||||||
|
|
||||||
|
private AlgorithmConfiguration configuration;
|
||||||
|
private String tablename = "rstr" + ("" + UUID.randomUUID()).replace("-", "");
|
||||||
|
static String createTableStatement = "CREATE TABLE %1$s (id serial, csquarecode character varying, x real, y real, z real, probability real)";
|
||||||
|
static String columnsnames = "csquarecode, x , y , z , probability";
|
||||||
|
public static String csquareColumn = "csquarecode";
|
||||||
|
public static String probabilityColumn = "probability";
|
||||||
|
public static String idColumn = "id";
|
||||||
|
|
||||||
|
public String getTablename() {
|
||||||
|
return tablename;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTablename(String tablename) {
|
||||||
|
this.tablename = tablename;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RasterTable(double x1, double x2, double y1, double y2, double z, double xResolution, double yResolution, double[][] values, AlgorithmConfiguration configuration) {
|
||||||
|
this.valuesMatrix = values;
|
||||||
|
this.configuration = configuration;
|
||||||
|
this.x1 = x1;
|
||||||
|
this.x2 = x2;
|
||||||
|
this.y1 = y1;
|
||||||
|
this.y2 = y2;
|
||||||
|
this.z = z;
|
||||||
|
this.xResolution = xResolution;
|
||||||
|
this.yResolution = yResolution;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void dumpGeoTable() {
|
||||||
|
|
||||||
|
// open the connection to the db
|
||||||
|
SessionFactory dbconnection = DatabaseUtils.initDBSession(configuration);
|
||||||
|
try {
|
||||||
|
AnalysisLogger.getLogger().debug("Database Initialized");
|
||||||
|
// create a table
|
||||||
|
DatabaseFactory.executeSQLUpdate(String.format(createTableStatement, tablename), dbconnection);
|
||||||
|
AnalysisLogger.getLogger().debug("Table " + tablename + " created");
|
||||||
|
List<Tuple<Double>> coordinates = MatrixExtractor.generateCoordinateTripletsInBoundingBox(x1, x2, y1, y2, z, xResolution, yResolution);
|
||||||
|
int triplets = coordinates.size();
|
||||||
|
AnalysisLogger.getLogger().debug("Generated " + triplets + " coordinates triples");
|
||||||
|
List<Double> values = associateValueToCoordinates(coordinates, valuesMatrix);
|
||||||
|
AnalysisLogger.getLogger().debug("Association to values completed - fulfilling buffer");
|
||||||
|
// for each element in the matrix, build the corresponding csquare code
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
for (int i = 0; i < triplets; i++) {
|
||||||
|
// save the string in a buffer
|
||||||
|
Tuple<Double> cset = coordinates.get(i);
|
||||||
|
double x = cset.getElements().get(0);
|
||||||
|
double y = cset.getElements().get(1);
|
||||||
|
String csquare = CSquareCodesConverter.convertAtResolution(y,x, xResolution);
|
||||||
|
Double value = values.get(i);
|
||||||
|
//we do not use NaNs in this case every value will be filled
|
||||||
|
if (value.isNaN())
|
||||||
|
value = 0d;
|
||||||
|
|
||||||
|
sb.append("('" + csquare + "'," + x + "," + y + "," + z + ",'" + value + "')");
|
||||||
|
|
||||||
|
if (i % 5000 == 0) {
|
||||||
|
// AnalysisLogger.getLogger().debug("Partial Inserting Buffer of " + sb.length() + " Values");
|
||||||
|
String insertStatement = DatabaseUtils.insertFromBuffer(tablename, columnsnames, sb);
|
||||||
|
DatabaseFactory.executeSQLUpdate(insertStatement, dbconnection);
|
||||||
|
// AnalysisLogger.getLogger().debug("Partial Insertion completed with Success!");
|
||||||
|
sb = new StringBuffer();
|
||||||
|
} else if (i < triplets - 1)
|
||||||
|
sb.append(",");
|
||||||
|
}
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("Inserting Final Buffer of " + sb.length() + " Values");
|
||||||
|
// save all the strings on the table
|
||||||
|
if (sb.length() > 0) {
|
||||||
|
String insertStatement = DatabaseUtils.insertFromBuffer(tablename, columnsnames, sb);
|
||||||
|
DatabaseFactory.executeSQLUpdate(insertStatement, dbconnection);
|
||||||
|
AnalysisLogger.getLogger().debug("Insertion completed with Success!");
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
AnalysisLogger.getLogger().debug("Error in dumping table: " + e.getLocalizedMessage());
|
||||||
|
} finally {
|
||||||
|
// close the connection
|
||||||
|
DatabaseUtils.closeDBConnection(dbconnection);
|
||||||
|
AnalysisLogger.getLogger().debug("Raster Geo Table DB closed!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deleteTable() {
|
||||||
|
SessionFactory dbconnection = null;
|
||||||
|
try {
|
||||||
|
dbconnection = DatabaseUtils.initDBSession(configuration);
|
||||||
|
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(tablename), dbconnection);
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} finally {
|
||||||
|
DatabaseUtils.closeDBConnection(dbconnection);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<Double> associateValueToCoordinates(List<Tuple<Double>> coordinates, double[][] data){
|
||||||
|
List<Double> values = new ArrayList<Double>();
|
||||||
|
|
||||||
|
int k = 0;
|
||||||
|
int g = 0;
|
||||||
|
int ntriplets = coordinates.size();
|
||||||
|
int xsteps = data[0].length-1;
|
||||||
|
for (int t = 0; t < ntriplets; t++) {
|
||||||
|
values.add(data[k][g]);
|
||||||
|
if (g == xsteps) {
|
||||||
|
g = 0;
|
||||||
|
k++;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
g++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue