diff --git a/src/main/java/org/gcube/dataanalysis/geo/algorithms/CSquaresCreator.java b/src/main/java/org/gcube/dataanalysis/geo/algorithms/CSquaresCreator.java index c21aa80..14396f6 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/algorithms/CSquaresCreator.java +++ b/src/main/java/org/gcube/dataanalysis/geo/algorithms/CSquaresCreator.java @@ -26,8 +26,8 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm { static String yDim = "Latitude_Column"; static String inputTableParameter = "InputTable"; static String outputTableParameter = "OutputTableName"; - static String resolutionParameter = "CSquare_Resolution"; - static String csquarecolumnName = "csquare_code"; + String resolutionParameter = "CSquare_Resolution"; + String codecolumnName = "csquare_code"; String outTable = ""; String outTableLabel = ""; double resolution = 0.1; @@ -79,7 +79,7 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm { String x = IOHelper.getInputParameter(config, xDim); String y = IOHelper.getInputParameter(config, yDim); String table = IOHelper.getInputParameter(config, inputTableParameter); - outTable = ("csq_" + UUID.randomUUID()).replace("-", ""); + outTable = ("code_" + UUID.randomUUID()).replace("-", ""); resolution = Double.parseDouble(IOHelper.getInputParameter(config, resolutionParameter)); outTableLabel = IOHelper.getInputParameter(config, outputTableParameter); @@ -88,23 +88,58 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm { status = 10; if (x == null || x.trim().length() == 0 || y == null || y.trim().length() == 0) throw new Exception("Error please provide information for the input layers"); + try{ + addCodeColumToTable(table); + AnalysisLogger.getLogger().debug("CSquareCreator: finished"); + } catch (Throwable e) { + throw new Exception(e.getMessage()); + } finally { + status = 100; + } + } - AnalysisLogger.getLogger().debug("CSquareCreator: initializing connection"); + public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){ + + String x = IOHelper.getInputParameter(config, xDim); + String y = IOHelper.getInputParameter(config, yDim); + + String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + offset; + return select; + } + + public String rowToCode (Object[] rowArray){ + // take x and y + String xValue = "" + rowArray[rowArray.length - 2]; + String yValue = "" + rowArray[rowArray.length - 1]; + // generate csquarecodes + String csquare = ""; + try { + double xV = Double.parseDouble(xValue); + double yV = Double.parseDouble(yValue); + csquare = CSquareCodesConverter.convertAtResolution(yV, xV, resolution); + } catch (Exception e) { + } + return csquare; + } + + public void addCodeColumToTable(String table) throws Exception{ + AnalysisLogger.getLogger().debug("CodeCreator: initializing connection"); + long t0 = System.currentTimeMillis(); try { connection = DatabaseUtils.initDBSession(config); - AnalysisLogger.getLogger().debug("CSquareCreator: database: " + config.getDatabaseURL()); + AnalysisLogger.getLogger().debug("CodeCreator: database: " + config.getDatabaseURL()); // create a new output table - AnalysisLogger.getLogger().debug("CSquareCreator: dropping table " + outTable + " if exists"); + AnalysisLogger.getLogger().debug("CodeCreator: dropping table " + outTable + " if exists"); try { DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection); } catch (Exception e) { - AnalysisLogger.getLogger().debug("CSquareCreator: table " + outTable + " does not exist yet"); + AnalysisLogger.getLogger().debug("CodeCreator: table " + outTable + " does not exist yet"); } - AnalysisLogger.getLogger().debug("CSquareCreator: creating the new table " + outTable); + AnalysisLogger.getLogger().debug("CodeCreator: creating the new table " + outTable); DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(table, outTable), connection); - AnalysisLogger.getLogger().debug("CSquareCreator: adding new column to " + outTable); - DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, csquarecolumnName, "character varying"), connection); - AnalysisLogger.getLogger().debug("CSquareCreator: getting columns from " + outTable); + AnalysisLogger.getLogger().debug("CodeCreator: adding new column to " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, codecolumnName, "character varying"), connection); + AnalysisLogger.getLogger().debug("CodeCreator: getting columns from " + outTable); // get columns names List names = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(outTable), connection); StringBuffer colnames = new StringBuffer(); @@ -114,42 +149,33 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm { if (i < nnames - 1) colnames.append(","); } - AnalysisLogger.getLogger().debug("CSquareCreator: columns are: " + colnames.toString()); - AnalysisLogger.getLogger().debug("CSquareCreator: taking chunks ... "); + AnalysisLogger.getLogger().debug("CodeCreator: columns are: " + colnames.toString()); + AnalysisLogger.getLogger().debug("CodeCreator: taking chunks ... "); // take chunks of the table int initIdx = 0; int limit = 5000; long maxRows = DatabaseUtils.estimateNumberofRows(table, connection); - AnalysisLogger.getLogger().debug("CSquareCreator: estimated number of rows " + maxRows); + AnalysisLogger.getLogger().debug("CodeCreator: estimated number of rows " + maxRows); status = 20; while (true) { - String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + initIdx; - AnalysisLogger.getLogger().debug("CSquareCreator: from " + initIdx + " to " + (initIdx + limit)); + + String select = selectInformationForTransformation(config, table, limit, initIdx); + AnalysisLogger.getLogger().debug("CodeCreator: from " + initIdx + " to " + (initIdx + limit)); List rows = DatabaseFactory.executeSQLQuery(select, connection); if (rows == null || rows.size() == 0) { - AnalysisLogger.getLogger().debug("CSquareCreator: no more rows"); + AnalysisLogger.getLogger().debug("CodeCreator: no more rows"); break; } - AnalysisLogger.getLogger().debug("CSquareCreator: transforming"); + AnalysisLogger.getLogger().debug("CodeCreator: transforming"); // take x and y List stringrows = new ArrayList(); for (Object row : rows) { Object[] rowArray = (Object[]) row; - // take x and y - String xValue = "" + rowArray[rowArray.length - 2]; - String yValue = "" + rowArray[rowArray.length - 1]; - // generate csquarecodes - String csquare = ""; - try { - double xV = Double.parseDouble(xValue); - double yV = Double.parseDouble(yValue); - csquare = CSquareCodesConverter.convertAtResolution(yV, xV, resolution); - } catch (Exception e) { - } - rowArray[rowArray.length - 2] = csquare; - String[] stringArray = new String[rowArray.length - 1]; + String code = rowToCode(rowArray); + rowArray[nnames-1] = code; + String[] stringArray = new String[nnames]; // convert all the objects into Strings for (int k = 0; k < stringArray.length; k++) { stringArray[k] = "" + rowArray[k]; @@ -180,10 +206,9 @@ public class CSquaresCreator extends StandardLocalExternalAlgorithm { } finally { shutdown(); AnalysisLogger.getLogger().debug("CSquareCreator finished in " + (System.currentTimeMillis() - t0) + " ms"); - status = 100; } } - + @Override public void shutdown() { AnalysisLogger.getLogger().debug("CSquareCreator shutdown"); diff --git a/src/main/java/org/gcube/dataanalysis/geo/algorithms/FAOOceanAreaCreator.java b/src/main/java/org/gcube/dataanalysis/geo/algorithms/FAOOceanAreaCreator.java new file mode 100644 index 0000000..066a940 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/algorithms/FAOOceanAreaCreator.java @@ -0,0 +1,71 @@ +package org.gcube.dataanalysis.geo.algorithms; + +import java.util.ArrayList; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.utils.IOHelper; +import org.gcube.dataanalysis.geo.utils.FAOOceanAreaConverter; + +public class FAOOceanAreaCreator extends CSquaresCreator { + + public FAOOceanAreaCreator(){ + resolutionParameter = "Resolution"; + codecolumnName = "fao_ocean_area"; + } + @Override + protected void setInputParameters() { + + List templates = new ArrayList(); + templates.add(TableTemplates.GENERIC); + InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm adds the csquare column"); + inputs.add(tinput); + + ColumnType xDimension = new ColumnType(inputTableParameter, xDim, "The column containing Longitude information", "x", false); + ColumnType yDimension = new ColumnType(inputTableParameter, yDim, "The column containing Latitude information", "y", false); + + inputs.add(xDimension); + inputs.add(yDimension); + IOHelper.addDoubleInput(inputs, resolutionParameter, "The resolution of the FAO Ocean Area codes", "5"); + IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "faooceanarea_"); + DatabaseType.addDefaultDBPars(inputs); + + } + + @Override + public String getDescription() { + return "An algorithm that adds a column containing the FAO Ocean Area codes associated to longitude and latitude columns."; + } + + public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){ + + String x = IOHelper.getInputParameter(config, xDim); + String y = IOHelper.getInputParameter(config, yDim); + + String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + offset; + return select; + } + + public String rowToCode (Object[] rowArray){ + // take x and y + String xValue = "" + rowArray[rowArray.length - 2]; + String yValue = "" + rowArray[rowArray.length - 1]; + + // generate csquarecodes + String code = ""; + try { + double xV = Double.parseDouble(xValue); + double yV = Double.parseDouble(yValue); + FAOOceanAreaConverter converter = new FAOOceanAreaConverter(); + code = converter.FAOOceanArea(-1, xV, yV, (int)resolution); + + } catch (Exception e) { + } + return code; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/algorithms/FAOOceanAreaCreatorQuadrant.java b/src/main/java/org/gcube/dataanalysis/geo/algorithms/FAOOceanAreaCreatorQuadrant.java new file mode 100644 index 0000000..c0c38b2 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/algorithms/FAOOceanAreaCreatorQuadrant.java @@ -0,0 +1,81 @@ +package org.gcube.dataanalysis.geo.algorithms; + +import java.util.ArrayList; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.utils.IOHelper; +import org.gcube.dataanalysis.geo.utils.FAOOceanAreaConverter; + +public class FAOOceanAreaCreatorQuadrant extends FAOOceanAreaCreator { + + static String quadrantDim = "Quadrant_column"; + + public FAOOceanAreaCreatorQuadrant(){ + super(); + } + + @Override + protected void setInputParameters() { + + List templates = new ArrayList(); + templates.add(TableTemplates.GENERIC); + InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm adds the csquare column"); + inputs.add(tinput); + + ColumnType xDimension = new ColumnType(inputTableParameter, xDim, "The column containing Longitude information", "x", false); + ColumnType yDimension = new ColumnType(inputTableParameter, yDim, "The column containing Latitude information", "y", false); + ColumnType quadrantDimension = new ColumnType(inputTableParameter, quadrantDim, "The column containing Quadrant information", "quadrant", false); + + inputs.add(xDimension); + inputs.add(yDimension); + inputs.add(quadrantDimension); + IOHelper.addDoubleInput(inputs, resolutionParameter, "The resolution of the FAO Ocean Area codes", "5"); + IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "faooceanarea_"); + DatabaseType.addDefaultDBPars(inputs); + + } + + @Override + public String getDescription() { + return "An algorithm that adds a column containing the FAO Ocean Area codes associated to longitude, latitude and quadrant columns."; + } + + @Override + public String selectInformationForTransformation (AlgorithmConfiguration config, String table, int limit, int offset){ + + String x = IOHelper.getInputParameter(config, xDim); + String y = IOHelper.getInputParameter(config, yDim); + String quadrant = IOHelper.getInputParameter(config, quadrantDim); + + String select = "select *," + x + "," + y +","+quadrant+ " from " + table + " order by " + x + " limit " + limit + " offset " + offset; + return select; + } + @Override + public String rowToCode (Object[] rowArray){ + // take x and y + String xValue = "" + rowArray[rowArray.length - 3]; + String yValue = "" + rowArray[rowArray.length - 2]; + String quadrantValue = "" + rowArray[rowArray.length - 1]; + + // generate csquarecodes + String code = ""; + try { + double xV = Double.parseDouble(xValue); + double yV = Double.parseDouble(yValue); + int quadrantV = (int)(Double.parseDouble(quadrantValue)); + + FAOOceanAreaConverter converter = new FAOOceanAreaConverter(); + code = converter.FAOOceanArea(quadrantV, xV, yV, (int)resolution); + + } catch (Exception e) { + + } + return code; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/TestFAOAreaCodesCreator.java b/src/main/java/org/gcube/dataanalysis/geo/test/TestFAOAreaCodesCreator.java new file mode 100644 index 0000000..c92316e --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/test/TestFAOAreaCodesCreator.java @@ -0,0 +1,41 @@ +package org.gcube.dataanalysis.geo.test; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.geo.algorithms.FAOOceanAreaCreator; + +public class TestFAOAreaCodesCreator { + + public static void main(String[] args) throws Exception{ + AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + +// config.setParam("Longitude_Column", "centerlong"); +// config.setParam("Latitude_Column", "centerlat"); + config.setParam("Longitude_Column", "long"); + config.setParam("Latitude_Column", "lat"); +// config.setParam("InputTable", "interp_2024_linear_01355325354899"); +// config.setParam("InputTable", "interp_2036_linear_11384851795640"); + config.setParam("InputTable", "generic_id6ef3e4fa_6a06_4df1_9445_553f2e918102"); + + + config.setParam("OutputTableName", "csqout"); + config.setParam("Resolution", "5"); + + config.setGcubeScope("/gcube/devsec/devVRE"); + + FAOOceanAreaCreator cscreator = new FAOOceanAreaCreator(); + cscreator.setConfiguration(config); + cscreator.compute(); + + System.out.println("DONE! "+cscreator.getOutput()); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/TestFAOAreaCodesQuadrantCreator.java b/src/main/java/org/gcube/dataanalysis/geo/test/TestFAOAreaCodesQuadrantCreator.java new file mode 100644 index 0000000..39185dd --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/test/TestFAOAreaCodesQuadrantCreator.java @@ -0,0 +1,41 @@ +package org.gcube.dataanalysis.geo.test; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.geo.algorithms.FAOOceanAreaCreator; +import org.gcube.dataanalysis.geo.algorithms.FAOOceanAreaCreatorQuadrant; + +public class TestFAOAreaCodesQuadrantCreator { + + public static void main(String[] args) throws Exception{ + AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + + config.setParam("Quadrant_column", "quadrant"); + config.setParam("Longitude_Column", "long"); + config.setParam("Latitude_Column", "lat"); + config.setParam("InputTable", "generic_id6ef3e4fa_6a06_4df1_9445_553f2e918102"); +// config.setParam("InputTable", "interp_2036_linear_11384851795640"); +// config.setParam("InputTable", "generic_id35e6ded3_4adc_48ba_a575_35a02b67514a"); + + + config.setParam("OutputTableName", "csqout"); + config.setParam("Resolution", "5"); + + config.setGcubeScope("/gcube/devsec/devVRE"); + + FAOOceanAreaCreatorQuadrant cscreator = new FAOOceanAreaCreatorQuadrant(); + cscreator.setConfiguration(config); + cscreator.compute(); + + System.out.println("DONE! "+cscreator.getOutput()); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/utils/FAOOceanAreaConverter.java b/src/main/java/org/gcube/dataanalysis/geo/utils/FAOOceanAreaConverter.java new file mode 100644 index 0000000..8b3d98f --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/utils/FAOOceanAreaConverter.java @@ -0,0 +1,97 @@ +package org.gcube.dataanalysis.geo.utils; + +public class FAOOceanAreaConverter { + + + public static void main(String[] args) { + FAOOceanAreaConverter converter = new FAOOceanAreaConverter(); + int quadrant = -1; + //double longitude = 12.5; + //double latitude = 12.5; + //double longitude = 129; + //double latitude = -29; + double longitude = -1; + double latitude = -56.1; + int resolution = 5; + + String conv = converter.FAOOceanArea(quadrant, longitude, latitude, resolution); + System.out.println(conv); + } + + public String padLongitude(int longitude){ + String longi = ""+longitude; + if (longi.length()==1) + longi = "00"+longi; + else if (longi.length()==2) + longi = "0"+longi; + return longi; + } + + public String padLatitude(int latitude){ + String lati = ""+latitude; + if (lati.length()==1) + lati = "0"+lati; + return lati; + } + + public int getQuadrant(double longitude,double latitude){ + if (longitude>=0 && latitude>=0) + return 1; + else if (longitude>=0 && latitude<0) + return 2; + else if (longitude<0 && latitude<0) + return 3; + else + return 4; + } + + public int[] getBounding(int resolution, double dimension){ + + int lowDim = (int)(resolution*Math.floor(dimension/resolution)); + int upDim = (int) (resolution*Math.ceil(dimension/resolution)); + int[] bounding = {lowDim,upDim}; + return bounding; + } + + public double getDistanceFromCenter(double x, double y){ + return Math.sqrt(x*x+y*y); + } + + public String FAOOceanArea(int quadrant, double longitude,double latitude, int resolution){ + + if (quadrant<=0||quadrant>4){ + quadrant = getQuadrant(longitude, latitude); + } + if (longitude>180 || longitude<-180) + return null; + if (latitude>90 || latitude<-90) + return null; + if (resolution<=0 || resolution>90) + return null; + + double alongitude = Math.abs(longitude); + double alatitude = Math.abs(latitude); + int [] bblons = getBounding(resolution, alongitude); + int [] bblats = getBounding(resolution, alatitude); + int bestlon = -1; + int bestlat = -1; + double bestdist = Double.MAX_VALUE; + for (int bblon:bblons){ + for (int bblat:bblats){ + double dist = getDistanceFromCenter(bblon, bblat); + if (dist