git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@111621 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
0ec883c920
commit
ed9826fde9
|
|
@ -0,0 +1,195 @@
|
|||
package org.gcube.dataanalysis.geo.algorithms;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.IOHelper;
|
||||
import org.gcube.dataanalysis.geo.utils.CSquareCodesConverter;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class CSquaresCreator extends StandardLocalExternalAlgorithm {
|
||||
|
||||
static String xDim = "Longitude_Column";
|
||||
static String yDim = "Latitude_Column";
|
||||
static String inputTableParameter = "InputTable";
|
||||
static String outputTableParameter = "OutputTableName";
|
||||
static String resolutionParameter = "CSquare_Resolution";
|
||||
static String csquarecolumnName = "csquare_code";
|
||||
String outTable = "";
|
||||
String outTableLabel = "";
|
||||
double resolution = 0.1;
|
||||
SessionFactory connection = null;
|
||||
|
||||
float status;
|
||||
|
||||
@Override
|
||||
protected void setInputParameters() {
|
||||
|
||||
List<TableTemplates> templates = new ArrayList<TableTemplates>();
|
||||
templates.add(TableTemplates.GENERIC);
|
||||
InputTable tinput = new InputTable(templates, inputTableParameter, "The table to which the algorithm adds the csquare column");
|
||||
inputs.add(tinput);
|
||||
|
||||
ColumnType xDimension = new ColumnType(inputTableParameter, xDim, "The column containing Longitude information", "x", false);
|
||||
ColumnType yDimension = new ColumnType(inputTableParameter, yDim, "The column containing Latitude information", "y", false);
|
||||
|
||||
inputs.add(xDimension);
|
||||
inputs.add(yDimension);
|
||||
IOHelper.addDoubleInput(inputs, resolutionParameter, "The resolution of the CSquare codes", "0.1");
|
||||
IOHelper.addStringInput(inputs, outputTableParameter, "The name of the output table", "csquaretbl_");
|
||||
DatabaseType.addDefaultDBPars(inputs);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public StatisticalType getOutput() {
|
||||
List<TableTemplates> template = new ArrayList<TableTemplates>();
|
||||
template.add(TableTemplates.GENERIC);
|
||||
OutputTable p = new OutputTable(template, outTableLabel, outTable, "Output table");
|
||||
return p;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() throws Exception {
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator Initialized");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "An algorithm that adds a column containing the CSquare codes associated to longitude and latitude columns.";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void process() throws Exception {
|
||||
status = 0;
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
long t0 = System.currentTimeMillis();
|
||||
String x = IOHelper.getInputParameter(config, xDim);
|
||||
String y = IOHelper.getInputParameter(config, yDim);
|
||||
String table = IOHelper.getInputParameter(config, inputTableParameter);
|
||||
outTable = ("csq_" + UUID.randomUUID()).replace("-", "");
|
||||
resolution = Double.parseDouble(IOHelper.getInputParameter(config, resolutionParameter));
|
||||
outTableLabel = IOHelper.getInputParameter(config, outputTableParameter);
|
||||
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: received parameters: x " + x + ", y " + y + ", table " + table + ", outputTable " + outTable + ", res " + resolution + " outLabel " + outTableLabel);
|
||||
|
||||
status = 10;
|
||||
if (x == null || x.trim().length() == 0 || y == null || y.trim().length() == 0)
|
||||
throw new Exception("Error please provide information for the input layers");
|
||||
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: initializing connection");
|
||||
try {
|
||||
connection = DatabaseUtils.initDBSession(config);
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: database: " + config.getDatabaseURL());
|
||||
// create a new output table
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: dropping table " + outTable + " if exists");
|
||||
try {
|
||||
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection);
|
||||
} catch (Exception e) {
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: table " + outTable + " does not exist yet");
|
||||
}
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: creating the new table " + outTable);
|
||||
DatabaseFactory.executeSQLUpdate(DatabaseUtils.createBlankTableFromAnotherStatement(table, outTable), connection);
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: adding new column to " + outTable);
|
||||
DatabaseFactory.executeSQLUpdate(DatabaseUtils.addColumnStatement(outTable, csquarecolumnName, "character varying"), connection);
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: getting columns from " + outTable);
|
||||
// get columns names
|
||||
List<Object> names = DatabaseFactory.executeSQLQuery(DatabaseUtils.getColumnsNamesStatement(outTable), connection);
|
||||
StringBuffer colnames = new StringBuffer();
|
||||
int nnames = names.size();
|
||||
for (int i = 0; i < nnames; i++) {
|
||||
colnames.append(names.get(i));
|
||||
if (i < nnames - 1)
|
||||
colnames.append(",");
|
||||
}
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: columns are: " + colnames.toString());
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: taking chunks ... ");
|
||||
// take chunks of the table
|
||||
int initIdx = 0;
|
||||
int limit = 5000;
|
||||
|
||||
long maxRows = DatabaseUtils.estimateNumberofRows(table, connection);
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: estimated number of rows " + maxRows);
|
||||
status = 20;
|
||||
while (true) {
|
||||
String select = "select *," + x + "," + y + " from " + table + " order by " + x + " limit " + limit + " offset " + initIdx;
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: from " + initIdx + " to " + (initIdx + limit));
|
||||
List<Object> rows = DatabaseFactory.executeSQLQuery(select, connection);
|
||||
|
||||
if (rows == null || rows.size() == 0) {
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: no more rows");
|
||||
break;
|
||||
}
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: transforming");
|
||||
// take x and y
|
||||
List<String[]> stringrows = new ArrayList<String[]>();
|
||||
for (Object row : rows) {
|
||||
Object[] rowArray = (Object[]) row;
|
||||
// take x and y
|
||||
String xValue = "" + rowArray[rowArray.length - 2];
|
||||
String yValue = "" + rowArray[rowArray.length - 1];
|
||||
// generate csquarecodes
|
||||
String csquare = "";
|
||||
try {
|
||||
double xV = Double.parseDouble(xValue);
|
||||
double yV = Double.parseDouble(yValue);
|
||||
csquare = CSquareCodesConverter.convertAtResolution(yV, xV, resolution);
|
||||
} catch (Exception e) {
|
||||
}
|
||||
rowArray[rowArray.length - 2] = csquare;
|
||||
String[] stringArray = new String[rowArray.length - 1];
|
||||
// convert all the objects into Strings
|
||||
for (int k = 0; k < stringArray.length; k++) {
|
||||
stringArray[k] = "" + rowArray[k];
|
||||
}
|
||||
|
||||
stringrows.add(stringArray);
|
||||
}
|
||||
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: inserting chunks into the table");
|
||||
// write the vector into the table
|
||||
DatabaseUtils.insertChunksIntoTable(outTable, colnames.toString(), stringrows, limit, connection, true);
|
||||
initIdx += limit;
|
||||
status = Math.min(90, 20 + (70 * initIdx / maxRows));
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: status " + status);
|
||||
}
|
||||
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: finished");
|
||||
} catch (Throwable e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator : ERROR!: " + e.getLocalizedMessage());
|
||||
try {
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: dropping " + outTable);
|
||||
DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(outTable), connection);
|
||||
} catch (Exception e1) {
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator: could not drop " + outTable);
|
||||
}
|
||||
throw new Exception(e.getMessage());
|
||||
} finally {
|
||||
shutdown();
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator finished in " + (System.currentTimeMillis() - t0) + " ms");
|
||||
status = 100;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
AnalysisLogger.getLogger().debug("CSquareCreator shutdown");
|
||||
DatabaseUtils.closeDBConnection(connection);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package org.gcube.dataanalysis.geo.test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.geo.algorithms.CSquaresCreator;
|
||||
import org.gcube.dataanalysis.geo.matrixmodel.XYExtractor;
|
||||
import org.gcube.dataanalysis.geo.utils.MapUtils;
|
||||
|
||||
public class TestCSquareCodesCreator {
|
||||
|
||||
public static void main(String[] args) throws Exception{
|
||||
AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile);
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setParam("DatabaseUserName", "utente");
|
||||
config.setParam("DatabasePassword", "d4science");
|
||||
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
|
||||
config.setParam("DatabaseDriver", "org.postgresql.Driver");
|
||||
|
||||
config.setParam("Longitude_Column", "centerlong");
|
||||
config.setParam("Latitude_Column", "centerlat");
|
||||
// config.setParam("InputTable", "interp_2024_linear_01355325354899");
|
||||
config.setParam("InputTable", "interp_2036_linear_11384851795640");
|
||||
|
||||
config.setParam("OutputTableName", "csqout");
|
||||
config.setParam("CSquare_Resolution", "0.5");
|
||||
|
||||
config.setGcubeScope("/gcube/devsec/devVRE");
|
||||
|
||||
CSquaresCreator cscreator = new CSquaresCreator();
|
||||
cscreator.setConfiguration(config);
|
||||
cscreator.compute();
|
||||
|
||||
System.out.println("DONE! "+cscreator.getOutput());
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue