Algorithms for FAO Ocean Area codes

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@111820 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2015-02-09 16:24:11 +00:00
parent 7c779c0634
commit 6baac3c87d
6 changed files with 389 additions and 33 deletions

View File

@ -26,8 +26,8 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm {
static String yDim = "Latitude_Column";
static String inputTableParameter = "InputTable";
static String outputTableParameter = "OutputTableName";
static String resolutionParameter = "CSquare_Resolution";
static String csquarecolumnName = "csquare_code";
String resolutionParameter = "CSquare_Resolution";
String codecolumnName = "csquare_code";
String outTable = "";
String outTableLabel = "";
double resolution = 0.1;
@ -79,7 +79,7 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm {
String x = IOHelper.getInputParameter(config, xDim);
String y = IOHelper.getInputParameter(config, yDim);
String table = IOHelper.getInputParameter(config, inputTableParameter);
outTable = ("csq_" + UUID.randomUUID()).replace("-", "");
outTable = ("code_" + UUID.randomUUID()).replace("-", "");
resolution = Double.parseDouble(IOHelper.getInputParameter(config, resolutionParameter));
outTableLabel = IOHelper.getInputParameter(config, outputTableParameter);
@ -88,23 +88,58 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm {
status = 10;
if (x == null || x.trim().length() == 0 || y == null || y.trim().length() == 0)
throw new Exception("Error please provide information for the input layers");
try{
addCodeColumToTable(table);
AnalysisLogger.getLogger().debug("CSquareCreator: finished");
} catch (Throwable e) {
throw new Exception(e.getMessage());
} finally {
status = 100;
}
}
AnalysisLogger.getLogger().debug("CSquareCreator: initializing connection");
public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){
String x = IOHelper.getInputParameter(config, xDim);
String y = IOHelper.getInputParameter(config, yDim);
String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + offset;
return select;
}
public String rowToCode (Object[] rowArray){
// take x and y
String xValue = "" + rowArray[rowArray.length - 2];
String yValue = "" + rowArray[rowArray.length - 1];
// generate csquarecodes
String csquare = "";
try {
double xV = Double.parseDouble(xValue);
double yV = Double.parseDouble(yValue);
csquare = CSquareCodesConverter.convertAtResolution(yV, xV, resolution);
} catch (Exception e) {
}
return csquare;
}
public void addCodeColumToTable(String table) throws Exception{
AnalysisLogger.getLogger().debug("CodeCreator: initializing connection");
long t0 = System.currentTimeMillis();
try {
connection = DatabaseUtils.initDBSession(config);
AnalysisLogger.getLogger().debug("CSquareCreator: database: " + config.getDatabaseURL());
AnalysisLogger.getLogger().debug("CodeCreator: database: " + config.getDatabaseURL());
// create a new output table
AnalysisLogger.getLogger().debug("CSquareCreator: dropping table " + outTable + " if exists");
AnalysisLogger.getLogger().debug("CodeCreator: dropping table " + outTable + " if exists");
try {
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection);
} catch (Exception e) {
AnalysisLogger.getLogger().debug("CSquareCreator: table " + outTable + " does not exist yet");
AnalysisLogger.getLogger().debug("CodeCreator: table " + outTable + " does not exist yet");
}
AnalysisLogger.getLogger().debug("CSquareCreator: creating the new table " + outTable);
AnalysisLogger.getLogger().debug("CodeCreator: creating the new table " + outTable);
DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(table, outTable), connection);
AnalysisLogger.getLogger().debug("CSquareCreator: adding new column to " + outTable);
DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, csquarecolumnName, "character varying"), connection);
AnalysisLogger.getLogger().debug("CSquareCreator: getting columns from " + outTable);
AnalysisLogger.getLogger().debug("CodeCreator: adding new column to " + outTable);
DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, codecolumnName, "character varying"), connection);
AnalysisLogger.getLogger().debug("CodeCreator: getting columns from " + outTable);
// get columns names
List<Object> names = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(outTable), connection);
StringBuffer colnames = new StringBuffer();
@ -114,42 +149,33 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm {
if (i < nnames - 1)
colnames.append(",");
}
AnalysisLogger.getLogger().debug("CSquareCreator: columns are: " + colnames.toString());
AnalysisLogger.getLogger().debug("CSquareCreator: taking chunks ... ");
AnalysisLogger.getLogger().debug("CodeCreator: columns are: " + colnames.toString());
AnalysisLogger.getLogger().debug("CodeCreator: taking chunks ... ");
// take chunks of the table
int initIdx = 0;
int limit = 5000;
long maxRows = DatabaseUtils.estimateNumberofRows(table, connection);
AnalysisLogger.getLogger().debug("CSquareCreator: estimated number of rows " + maxRows);
AnalysisLogger.getLogger().debug("CodeCreator: estimated number of rows " + maxRows);
status = 20;
while (true) {
String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + initIdx;
AnalysisLogger.getLogger().debug("CSquareCreator: from " + initIdx + " to " + (initIdx + limit));
String select = selectInformationForTransformation(config, table, limit, initIdx);
AnalysisLogger.getLogger().debug("CodeCreator: from " + initIdx + " to " + (initIdx + limit));
List<Object> rows = DatabaseFactory.executeSQLQuery(select, connection);
if (rows == null || rows.size() == 0) {
AnalysisLogger.getLogger().debug("CSquareCreator: no more rows");
AnalysisLogger.getLogger().debug("CodeCreator: no more rows");
break;
}
AnalysisLogger.getLogger().debug("CSquareCreator: transforming");
AnalysisLogger.getLogger().debug("CodeCreator: transforming");
// take x and y
List<String[]> stringrows = new ArrayList<String[]>();
for (Object row : rows) {
Object[] rowArray = (Object[]) row;
// take x and y
String xValue = "" + rowArray[rowArray.length - 2];
String yValue = "" + rowArray[rowArray.length - 1];
// generate csquarecodes
String csquare = "";
try {
double xV = Double.parseDouble(xValue);
double yV = Double.parseDouble(yValue);
csquare = CSquareCodesConverter.convertAtResolution(yV, xV, resolution);
} catch (Exception e) {
}
rowArray[rowArray.length - 2] = csquare;
String[] stringArray = new String[rowArray.length - 1];
String code = rowToCode(rowArray);
rowArray[nnames-1] = code;
String[] stringArray = new String[nnames];
// convert all the objects into Strings
for (int k = 0; k < stringArray.length; k++) {
stringArray[k] = "" + rowArray[k];
@ -180,10 +206,9 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm {
} finally {
shutdown();
AnalysisLogger.getLogger().debug("CSquareCreator finished in " + (System.currentTimeMillis() - t0) + " ms");
status = 100;
}
}
@Override
public void shutdown() {
AnalysisLogger.getLogger().debug("CSquareCreator shutdown");

View File

@ -0,0 +1,71 @@
package org.gcube.dataanalysis.geo.algorithms;
import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
import org.gcube.dataanalysis.geo.utils.FAOOceanAreaConverter;
public class FAOOceanAreaCreator extends CSquaresCreator {
public FAOOceanAreaCreator(){
resolutionParameter = "Resolution";
codecolumnName = "fao_ocean_area";
}
@Override
protected void setInputParameters() {
List<TableTemplates> templates = new ArrayList<TableTemplates>();
templates.add(TableTemplates.GENERIC);
InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm adds the csquare column");
inputs.add(tinput);
ColumnType xDimension = new ColumnType(inputTableParameter, xDim, "The column containing Longitude information", "x", false);
ColumnType yDimension = new ColumnType(inputTableParameter, yDim, "The column containing Latitude information", "y", false);
inputs.add(xDimension);
inputs.add(yDimension);
IOHelper.addDoubleInput(inputs, resolutionParameter, "The resolution of the FAO Ocean Area codes", "5");
IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "faooceanarea_");
DatabaseType.addDefaultDBPars(inputs);
}
@Override
public String getDescription() {
return "An algorithm that adds a column containing the FAO Ocean Area codes associated to longitude and latitude columns.";
}
public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){
String x = IOHelper.getInputParameter(config, xDim);
String y = IOHelper.getInputParameter(config, yDim);
String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + offset;
return select;
}
public String rowToCode (Object[] rowArray){
// take x and y
String xValue = "" + rowArray[rowArray.length - 2];
String yValue = "" + rowArray[rowArray.length - 1];
// generate csquarecodes
String code = "";
try {
double xV = Double.parseDouble(xValue);
double yV = Double.parseDouble(yValue);
FAOOceanAreaConverter converter = new FAOOceanAreaConverter();
code = converter.FAOOceanArea(-1, xV, yV, (int)resolution);
} catch (Exception e) {
}
return code;
}
}

View File

@ -0,0 +1,81 @@
package org.gcube.dataanalysis.geo.algorithms;
import java.util.ArrayList;
import java.util.List;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
import org.gcube.dataanalysis.geo.utils.FAOOceanAreaConverter;
public class FAOOceanAreaCreatorQuadrant extends FAOOceanAreaCreator {
static String quadrantDim = "Quadrant_column";
public FAOOceanAreaCreatorQuadrant(){
super();
}
@Override
protected void setInputParameters() {
List<TableTemplates> templates = new ArrayList<TableTemplates>();
templates.add(TableTemplates.GENERIC);
InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm adds the csquare column");
inputs.add(tinput);
ColumnType xDimension = new ColumnType(inputTableParameter, xDim, "The column containing Longitude information", "x", false);
ColumnType yDimension = new ColumnType(inputTableParameter, yDim, "The column containing Latitude information", "y", false);
ColumnType quadrantDimension = new ColumnType(inputTableParameter, quadrantDim, "The column containing Quadrant information", "quadrant", false);
inputs.add(xDimension);
inputs.add(yDimension);
inputs.add(quadrantDimension);
IOHelper.addDoubleInput(inputs, resolutionParameter, "The resolution of the FAO Ocean Area codes", "5");
IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "faooceanarea_");
DatabaseType.addDefaultDBPars(inputs);
}
@Override
public String getDescription() {
return "An algorithm that adds a column containing the FAO Ocean Area codes associated to longitude, latitude and quadrant columns.";
}
@Override
public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){
String x = IOHelper.getInputParameter(config, xDim);
String y = IOHelper.getInputParameter(config, yDim);
String quadrant = IOHelper.getInputParameter(config, quadrantDim);
String select = "select *," + x + "," + y +","+quadrant+ " from " + table + " order by " + x + " limit " + limit + " offset " + offset;
return select;
}
@Override
public String rowToCode (Object[] rowArray){
// take x and y
String xValue = "" + rowArray[rowArray.length - 3];
String yValue = "" + rowArray[rowArray.length - 2];
String quadrantValue = "" + rowArray[rowArray.length - 1];
// generate csquarecodes
String code = "";
try {
double xV = Double.parseDouble(xValue);
double yV = Double.parseDouble(yValue);
int quadrantV = (int)(Double.parseDouble(quadrantValue));
FAOOceanAreaConverter converter = new FAOOceanAreaConverter();
code = converter.FAOOceanArea(quadrantV, xV, yV, (int)resolution);
} catch (Exception e) {
}
return code;
}
}

View File

@ -0,0 +1,41 @@
package org.gcube.dataanalysis.geo.test;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.geo.algorithms.FAOOceanAreaCreator;
public class TestFAOAreaCodesCreator {
public static void main(String[] args) throws Exception{
AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile);
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// config.setParam("Longitude_Column", "centerlong");
// config.setParam("Latitude_Column", "centerlat");
config.setParam("Longitude_Column", "long");
config.setParam("Latitude_Column", "lat");
// config.setParam("InputTable", "interp_2024_linear_01355325354899");
// config.setParam("InputTable", "interp_2036_linear_11384851795640");
config.setParam("InputTable", "generic_id6ef3e4fa_6a06_4df1_9445_553f2e918102");
config.setParam("OutputTableName", "csqout");
config.setParam("Resolution", "5");
config.setGcubeScope("/gcube/devsec/devVRE");
FAOOceanAreaCreator cscreator = new FAOOceanAreaCreator();
cscreator.setConfiguration(config);
cscreator.compute();
System.out.println("DONE! "+cscreator.getOutput());
}
}

View File

@ -0,0 +1,41 @@
package org.gcube.dataanalysis.geo.test;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.geo.algorithms.FAOOceanAreaCreator;
import org.gcube.dataanalysis.geo.algorithms.FAOOceanAreaCreatorQuadrant;
public class TestFAOAreaCodesQuadrantCreator {
public static void main(String[] args) throws Exception{
AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile);
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
config.setParam("Quadrant_column", "quadrant");
config.setParam("Longitude_Column", "long");
config.setParam("Latitude_Column", "lat");
config.setParam("InputTable", "generic_id6ef3e4fa_6a06_4df1_9445_553f2e918102");
// config.setParam("InputTable", "interp_2036_linear_11384851795640");
// config.setParam("InputTable", "generic_id35e6ded3_4adc_48ba_a575_35a02b67514a");
config.setParam("OutputTableName", "csqout");
config.setParam("Resolution", "5");
config.setGcubeScope("/gcube/devsec/devVRE");
FAOOceanAreaCreatorQuadrant cscreator = new FAOOceanAreaCreatorQuadrant();
cscreator.setConfiguration(config);
cscreator.compute();
System.out.println("DONE! "+cscreator.getOutput());
}
}

View File

@ -0,0 +1,97 @@
package org.gcube.dataanalysis.geo.utils;
public class FAOOceanAreaConverter {
public static void main(String[] args) {
FAOOceanAreaConverter converter = new FAOOceanAreaConverter();
int quadrant = -1;
//double longitude = 12.5;
//double latitude = 12.5;
//double longitude = 129;
//double latitude = -29;
double longitude = -1;
double latitude = -56.1;
int resolution = 5;
String conv = converter.FAOOceanArea(quadrant, longitude, latitude, resolution);
System.out.println(conv);
}
public String padLongitude(int longitude){
String longi = ""+longitude;
if (longi.length()==1)
longi = "00"+longi;
else if (longi.length()==2)
longi = "0"+longi;
return longi;
}
public String padLatitude(int latitude){
String lati = ""+latitude;
if (lati.length()==1)
lati = "0"+lati;
return lati;
}
public int getQuadrant(double longitude,double latitude){
if (longitude>=0 && latitude>=0)
return 1;
else if (longitude>=0 && latitude<0)
return 2;
else if (longitude<0 && latitude<0)
return 3;
else
return 4;
}
public int[] getBounding(int resolution, double dimension){
int lowDim = (int)(resolution*Math.floor(dimension/resolution));
int upDim = (int) (resolution*Math.ceil(dimension/resolution));
int[] bounding = {lowDim,upDim};
return bounding;
}
public double getDistanceFromCenter(double x, double y){
return Math.sqrt(x*x+y*y);
}
public String FAOOceanArea(int quadrant, double longitude,double latitude, int resolution){
if (quadrant<=0||quadrant>4){
quadrant = getQuadrant(longitude, latitude);
}
if (longitude>180 || longitude<-180)
return null;
if (latitude>90 || latitude<-90)
return null;
if (resolution<=0 || resolution>90)
return null;
double alongitude = Math.abs(longitude);
double alatitude = Math.abs(latitude);
int [] bblons = getBounding(resolution, alongitude);
int [] bblats = getBounding(resolution, alatitude);
int bestlon = -1;
int bestlat = -1;
double bestdist = Double.MAX_VALUE;
for (int bblon:bblons){
for (int bblat:bblats){
double dist = getDistanceFromCenter(bblon, bblat);
if (dist<bestdist){
bestdist=dist;
bestlon = bblon;
bestlat = bblat;
}
}
}
String FaoOceanArea = quadrant+padLatitude(bestlat)+padLongitude(bestlon);
if (FaoOceanArea.length()!=6)
return null;
return FaoOceanArea;
}
}