diff --git a/src/main/java/org/gcube/dataanalysis/geo/algorithms/CSquaresCreator.java b/src/main/java/org/gcube/dataanalysis/geo/algorithms/CSquaresCreator.java new file mode 100644 index 0000000..905234f --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/algorithms/CSquaresCreator.java @@ -0,0 +1,195 @@ +package org.gcube.dataanalysis.geo.algorithms; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.UUID; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.gcube.dataanalysis.ecoengine.utils.IOHelper; +import org.gcube.dataanalysis.geo.utils.CSquareCodesConverter; +import org.hibernate.SessionFactory; + +public class CSquaresCreator extends StandardLocalExternalAlgorithm { + + static String xDim = "Longitude_Column"; + static String yDim = "Latitude_Column"; + static String inputTableParameter = "InputTable"; + static String outputTableParameter = "OutputTableName"; + static String resolutionParameter = "CSquare_Resolution"; + static String csquarecolumnName = "csquare_code"; + String outTable = ""; + String outTableLabel = ""; + double resolution = 0.1; + SessionFactory connection = null; + + float status; + + @Override + protected void setInputParameters() { + + List templates = new ArrayList(); + templates.add(TableTemplates.GENERIC); + InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm adds the csquare column"); + inputs.add(tinput); + + ColumnType xDimension = new ColumnType(inputTableParameter, xDim, "The column containing Longitude information", "x", false); + ColumnType yDimension = new ColumnType(inputTableParameter, yDim, "The column containing Latitude information", "y", false); + + inputs.add(xDimension); + inputs.add(yDimension); + IOHelper.addDoubleInput(inputs, resolutionParameter, "The resolution of the CSquare codes", "0.1"); + IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "csquaretbl_"); + DatabaseType.addDefaultDBPars(inputs); + + } + + @Override + public StatisticalType getOutput() { + List template = new ArrayList(); + template.add(TableTemplates.GENERIC); + OutputTable p = new OutputTable(template, outTableLabel, outTable, "Output table"); + return p; + } + + @Override + public void init() throws Exception { + AnalysisLogger.getLogger().debug("CSquareCreator Initialized"); + } + + @Override + public String getDescription() { + return "An algorithm that adds a column containing the CSquare codes associated to longitude and latitude columns."; + } + + @Override + protected void process() throws Exception { + status = 0; + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + long t0 = System.currentTimeMillis(); + String x = IOHelper.getInputParameter(config, xDim); + String y = IOHelper.getInputParameter(config, yDim); + String table = IOHelper.getInputParameter(config, inputTableParameter); + outTable = ("csq_" + UUID.randomUUID()).replace("-", ""); + resolution = Double.parseDouble(IOHelper.getInputParameter(config, resolutionParameter)); + outTableLabel = IOHelper.getInputParameter(config, outputTableParameter); + + AnalysisLogger.getLogger().debug("CSquareCreator: received parameters: x " + x + ", y " + y + ", table " + table + ", outputTable " + outTable + ", res " + resolution + " outLabel " + outTableLabel); + + status = 10; + if (x == null || x.trim().length() == 0 || y == null || y.trim().length() == 0) + throw new Exception("Error please provide information for the input layers"); + + AnalysisLogger.getLogger().debug("CSquareCreator: initializing connection"); + try { + connection = DatabaseUtils.initDBSession(config); + AnalysisLogger.getLogger().debug("CSquareCreator: database: " + config.getDatabaseURL()); + // create a new output table + AnalysisLogger.getLogger().debug("CSquareCreator: dropping table " + outTable + " if exists"); + try { + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection); + } catch (Exception e) { + AnalysisLogger.getLogger().debug("CSquareCreator: table " + outTable + " does not exist yet"); + } + AnalysisLogger.getLogger().debug("CSquareCreator: creating the new table " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(table, outTable), connection); + AnalysisLogger.getLogger().debug("CSquareCreator: adding new column to " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, csquarecolumnName, "character varying"), connection); + AnalysisLogger.getLogger().debug("CSquareCreator: getting columns from " + outTable); + // get columns names + List names = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(outTable), connection); + StringBuffer colnames = new StringBuffer(); + int nnames = names.size(); + for (int i = 0; i < nnames; i++) { + colnames.append(names.get(i)); + if (i < nnames - 1) + colnames.append(","); + } + AnalysisLogger.getLogger().debug("CSquareCreator: columns are: " + colnames.toString()); + AnalysisLogger.getLogger().debug("CSquareCreator: taking chunks ... "); + // take chunks of the table + int initIdx = 0; + int limit = 5000; + + long maxRows = DatabaseUtils.estimateNumberofRows(table, connection); + AnalysisLogger.getLogger().debug("CSquareCreator: estimated number of rows " + maxRows); + status = 20; + while (true) { + String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + initIdx; + AnalysisLogger.getLogger().debug("CSquareCreator: from " + initIdx + " to " + (initIdx + limit)); + List rows = DatabaseFactory.executeSQLQuery(select, connection); + + if (rows == null || rows.size() == 0) { + AnalysisLogger.getLogger().debug("CSquareCreator: no more rows"); + break; + } + AnalysisLogger.getLogger().debug("CSquareCreator: transforming"); + // take x and y + List stringrows = new ArrayList(); + for (Object row : rows) { + Object[] rowArray = (Object[]) row; + // take x and y + String xValue = "" + rowArray[rowArray.length - 2]; + String yValue = "" + rowArray[rowArray.length - 1]; + // generate csquarecodes + String csquare = ""; + try { + double xV = Double.parseDouble(xValue); + double yV = Double.parseDouble(yValue); + csquare = CSquareCodesConverter.convertAtResolution(yV, xV, resolution); + } catch (Exception e) { + } + rowArray[rowArray.length - 2] = csquare; + String[] stringArray = new String[rowArray.length - 1]; + // convert all the objects into Strings + for (int k = 0; k < stringArray.length; k++) { + stringArray[k] = "" + rowArray[k]; + } + + stringrows.add(stringArray); + } + + AnalysisLogger.getLogger().debug("CSquareCreator: inserting chunks into the table"); + // write the vector into the table + DatabaseUtils.insertChunksIntoTable(outTable, colnames.toString(), stringrows, limit, connection, true); + initIdx += limit; + status = Math.min(90, 20 + (70 * initIdx / maxRows)); + AnalysisLogger.getLogger().debug("CSquareCreator: status " + status); + } + + AnalysisLogger.getLogger().debug("CSquareCreator: finished"); + } catch (Throwable e) { + e.printStackTrace(); + AnalysisLogger.getLogger().debug("CSquareCreator : ERROR!: " + e.getLocalizedMessage()); + try { + AnalysisLogger.getLogger().debug("CSquareCreator: dropping " + outTable); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection); + } catch (Exception e1) { + AnalysisLogger.getLogger().debug("CSquareCreator: could not drop " + outTable); + } + throw new Exception(e.getMessage()); + } finally { + shutdown(); + AnalysisLogger.getLogger().debug("CSquareCreator finished in " + (System.currentTimeMillis() - t0) + " ms"); + status = 100; + } + } + + @Override + public void shutdown() { + AnalysisLogger.getLogger().debug("CSquareCreator shutdown"); + DatabaseUtils.closeDBConnection(connection); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/test/TestCSquareCodesCreator.java b/src/main/java/org/gcube/dataanalysis/geo/test/TestCSquareCodesCreator.java new file mode 100644 index 0000000..a08941f --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/test/TestCSquareCodesCreator.java @@ -0,0 +1,42 @@ +package org.gcube.dataanalysis.geo.test; + +import java.io.File; +import java.io.FileWriter; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.geo.algorithms.CSquaresCreator; +import org.gcube.dataanalysis.geo.matrixmodel.XYExtractor; +import org.gcube.dataanalysis.geo.utils.MapUtils; + +public class TestCSquareCodesCreator { + + public static void main(String[] args) throws Exception{ + AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + + config.setParam("Longitude_Column", "centerlong"); + config.setParam("Latitude_Column", "centerlat"); +// config.setParam("InputTable", "interp_2024_linear_01355325354899"); + config.setParam("InputTable", "interp_2036_linear_11384851795640"); + + config.setParam("OutputTableName", "csqout"); + config.setParam("CSquare_Resolution", "0.5"); + + config.setGcubeScope("/gcube/devsec/devVRE"); + + CSquaresCreator cscreator = new CSquaresCreator(); + cscreator.setConfiguration(config); + cscreator.compute(); + + System.out.println("DONE! "+cscreator.getOutput()); + } + +}