git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@111621 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
0ec883c920
commit
ed9826fde9
|
@ -0,0 +1,195 @@
|
||||||
|
package org.gcube.dataanalysis.geo.algorithms;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
|
||||||
|
import org.gcube.dataanalysis.geo.utils.CSquareCodesConverter;
|
||||||
|
import org.hibernate.SessionFactory;
|
||||||
|
|
||||||
|
public class CSquaresCreator extends StandardLocalExternalAlgorithm {
|
||||||
|
|
||||||
|
static String xDim = "Longitude_Column";
|
||||||
|
static String yDim = "Latitude_Column";
|
||||||
|
static String inputTableParameter = "InputTable";
|
||||||
|
static String outputTableParameter = "OutputTableName";
|
||||||
|
static String resolutionParameter = "CSquare_Resolution";
|
||||||
|
static String csquarecolumnName = "csquare_code";
|
||||||
|
String outTable = "";
|
||||||
|
String outTableLabel = "";
|
||||||
|
double resolution = 0.1;
|
||||||
|
SessionFactory connection = null;
|
||||||
|
|
||||||
|
float status;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void setInputParameters() {
|
||||||
|
|
||||||
|
List<TableTemplates> templates = new ArrayList<TableTemplates>();
|
||||||
|
templates.add(TableTemplates.GENERIC);
|
||||||
|
InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm adds the csquare column");
|
||||||
|
inputs.add(tinput);
|
||||||
|
|
||||||
|
ColumnType xDimension = new ColumnType(inputTableParameter, xDim, "The column containing Longitude information", "x", false);
|
||||||
|
ColumnType yDimension = new ColumnType(inputTableParameter, yDim, "The column containing Latitude information", "y", false);
|
||||||
|
|
||||||
|
inputs.add(xDimension);
|
||||||
|
inputs.add(yDimension);
|
||||||
|
IOHelper.addDoubleInput(inputs, resolutionParameter, "The resolution of the CSquare codes", "0.1");
|
||||||
|
IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "csquaretbl_");
|
||||||
|
DatabaseType.addDefaultDBPars(inputs);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public StatisticalType getOutput() {
|
||||||
|
List<TableTemplates> template = new ArrayList<TableTemplates>();
|
||||||
|
template.add(TableTemplates.GENERIC);
|
||||||
|
OutputTable p = new OutputTable(template, outTableLabel, outTable, "Output table");
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init() throws Exception {
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator Initialized");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getDescription() {
|
||||||
|
return "An algorithm that adds a column containing the CSquare codes associated to longitude and latitude columns.";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void process() throws Exception {
|
||||||
|
status = 0;
|
||||||
|
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||||
|
|
||||||
|
long t0 = System.currentTimeMillis();
|
||||||
|
String x = IOHelper.getInputParameter(config, xDim);
|
||||||
|
String y = IOHelper.getInputParameter(config, yDim);
|
||||||
|
String table = IOHelper.getInputParameter(config, inputTableParameter);
|
||||||
|
outTable = ("csq_" + UUID.randomUUID()).replace("-", "");
|
||||||
|
resolution = Double.parseDouble(IOHelper.getInputParameter(config, resolutionParameter));
|
||||||
|
outTableLabel = IOHelper.getInputParameter(config, outputTableParameter);
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: received parameters: x " + x + ", y " + y + ", table " + table + ", outputTable " + outTable + ", res " + resolution + " outLabel " + outTableLabel);
|
||||||
|
|
||||||
|
status = 10;
|
||||||
|
if (x == null || x.trim().length() == 0 || y == null || y.trim().length() == 0)
|
||||||
|
throw new Exception("Error please provide information for the input layers");
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: initializing connection");
|
||||||
|
try {
|
||||||
|
connection = DatabaseUtils.initDBSession(config);
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: database: " + config.getDatabaseURL());
|
||||||
|
// create a new output table
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: dropping table " + outTable + " if exists");
|
||||||
|
try {
|
||||||
|
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection);
|
||||||
|
} catch (Exception e) {
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: table " + outTable + " does not exist yet");
|
||||||
|
}
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: creating the new table " + outTable);
|
||||||
|
DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(table, outTable), connection);
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: adding new column to " + outTable);
|
||||||
|
DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, csquarecolumnName, "character varying"), connection);
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: getting columns from " + outTable);
|
||||||
|
// get columns names
|
||||||
|
List<Object> names = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(outTable), connection);
|
||||||
|
StringBuffer colnames = new StringBuffer();
|
||||||
|
int nnames = names.size();
|
||||||
|
for (int i = 0; i < nnames; i++) {
|
||||||
|
colnames.append(names.get(i));
|
||||||
|
if (i < nnames - 1)
|
||||||
|
colnames.append(",");
|
||||||
|
}
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: columns are: " + colnames.toString());
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: taking chunks ... ");
|
||||||
|
// take chunks of the table
|
||||||
|
int initIdx = 0;
|
||||||
|
int limit = 5000;
|
||||||
|
|
||||||
|
long maxRows = DatabaseUtils.estimateNumberofRows(table, connection);
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: estimated number of rows " + maxRows);
|
||||||
|
status = 20;
|
||||||
|
while (true) {
|
||||||
|
String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + initIdx;
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: from " + initIdx + " to " + (initIdx + limit));
|
||||||
|
List<Object> rows = DatabaseFactory.executeSQLQuery(select, connection);
|
||||||
|
|
||||||
|
if (rows == null || rows.size() == 0) {
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: no more rows");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: transforming");
|
||||||
|
// take x and y
|
||||||
|
List<String[]> stringrows = new ArrayList<String[]>();
|
||||||
|
for (Object row : rows) {
|
||||||
|
Object[] rowArray = (Object[]) row;
|
||||||
|
// take x and y
|
||||||
|
String xValue = "" + rowArray[rowArray.length - 2];
|
||||||
|
String yValue = "" + rowArray[rowArray.length - 1];
|
||||||
|
// generate csquarecodes
|
||||||
|
String csquare = "";
|
||||||
|
try {
|
||||||
|
double xV = Double.parseDouble(xValue);
|
||||||
|
double yV = Double.parseDouble(yValue);
|
||||||
|
csquare = CSquareCodesConverter.convertAtResolution(yV, xV, resolution);
|
||||||
|
} catch (Exception e) {
|
||||||
|
}
|
||||||
|
rowArray[rowArray.length - 2] = csquare;
|
||||||
|
String[] stringArray = new String[rowArray.length - 1];
|
||||||
|
// convert all the objects into Strings
|
||||||
|
for (int k = 0; k < stringArray.length; k++) {
|
||||||
|
stringArray[k] = "" + rowArray[k];
|
||||||
|
}
|
||||||
|
|
||||||
|
stringrows.add(stringArray);
|
||||||
|
}
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: inserting chunks into the table");
|
||||||
|
// write the vector into the table
|
||||||
|
DatabaseUtils.insertChunksIntoTable(outTable, colnames.toString(), stringrows, limit, connection, true);
|
||||||
|
initIdx += limit;
|
||||||
|
status = Math.min(90, 20 + (70 * initIdx / maxRows));
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: status " + status);
|
||||||
|
}
|
||||||
|
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: finished");
|
||||||
|
} catch (Throwable e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator : ERROR!: " + e.getLocalizedMessage());
|
||||||
|
try {
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: dropping " + outTable);
|
||||||
|
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection);
|
||||||
|
} catch (Exception e1) {
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator: could not drop " + outTable);
|
||||||
|
}
|
||||||
|
throw new Exception(e.getMessage());
|
||||||
|
} finally {
|
||||||
|
shutdown();
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator finished in " + (System.currentTimeMillis() - t0) + " ms");
|
||||||
|
status = 100;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void shutdown() {
|
||||||
|
AnalysisLogger.getLogger().debug("CSquareCreator shutdown");
|
||||||
|
DatabaseUtils.closeDBConnection(connection);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,42 @@
|
||||||
|
package org.gcube.dataanalysis.geo.test;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileWriter;
|
||||||
|
|
||||||
|
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||||
|
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||||
|
import org.gcube.dataanalysis.geo.algorithms.CSquaresCreator;
|
||||||
|
import org.gcube.dataanalysis.geo.matrixmodel.XYExtractor;
|
||||||
|
import org.gcube.dataanalysis.geo.utils.MapUtils;
|
||||||
|
|
||||||
|
public class TestCSquareCodesCreator {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception{
|
||||||
|
AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile);
|
||||||
|
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||||
|
|
||||||
|
config.setConfigPath("./cfg/");
|
||||||
|
config.setPersistencePath("./");
|
||||||
|
config.setParam("DatabaseUserName", "utente");
|
||||||
|
config.setParam("DatabasePassword", "d4science");
|
||||||
|
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
|
||||||
|
config.setParam("DatabaseDriver", "org.postgresql.Driver");
|
||||||
|
|
||||||
|
config.setParam("Longitude_Column", "centerlong");
|
||||||
|
config.setParam("Latitude_Column", "centerlat");
|
||||||
|
// config.setParam("InputTable", "interp_2024_linear_01355325354899");
|
||||||
|
config.setParam("InputTable", "interp_2036_linear_11384851795640");
|
||||||
|
|
||||||
|
config.setParam("OutputTableName", "csqout");
|
||||||
|
config.setParam("CSquare_Resolution", "0.5");
|
||||||
|
|
||||||
|
config.setGcubeScope("/gcube/devsec/devVRE");
|
||||||
|
|
||||||
|
CSquaresCreator cscreator = new CSquaresCreator();
|
||||||
|
cscreator.setConfiguration(config);
|
||||||
|
cscreator.compute();
|
||||||
|
|
||||||
|
System.out.println("DONE! "+cscreator.getOutput());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue