2015-02-03 16:37:47 +01:00
package org.gcube.dataanalysis.geo.algorithms ;
import java.util.ArrayList ;
import java.util.LinkedHashMap ;
import java.util.List ;
import java.util.UUID ;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger ;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration ;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType ;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType ;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable ;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable ;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType ;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates ;
2016-09-27 18:00:11 +02:00
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm ;
2015-02-03 16:37:47 +01:00
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory ;
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils ;
import org.gcube.dataanalysis.ecoengine.utils.IOHelper ;
import org.gcube.dataanalysis.geo.utils.CSquareCodesConverter ;
import org.hibernate.SessionFactory ;
2016-09-27 18:00:11 +02:00
public class CSquaresCreator extends StandardLocalInfraAlgorithm {
2015-02-03 16:37:47 +01:00
static String xDim = " Longitude_Column " ;
static String yDim = " Latitude_Column " ;
static String inputTableParameter = " InputTable " ;
static String outputTableParameter = " OutputTableName " ;
2015-02-09 17:24:11 +01:00
String resolutionParameter = " CSquare_Resolution " ;
String codecolumnName = " csquare_code " ;
2015-02-03 16:37:47 +01:00
String outTable = " " ;
String outTableLabel = " " ;
double resolution = 0 . 1 ;
SessionFactory connection = null ;
@Override
protected void setInputParameters ( ) {
List < TableTemplates > templates = new ArrayList < TableTemplates > ( ) ;
templates . add ( TableTemplates . GENERIC ) ;
InputTable tinput = new InputTable ( templates , inputTableParameter , " The table to which the algorithm adds the csquare column " ) ;
inputs . add ( tinput ) ;
ColumnType xDimension = new ColumnType ( inputTableParameter , xDim , " The column containing Longitude information " , " x " , false ) ;
ColumnType yDimension = new ColumnType ( inputTableParameter , yDim , " The column containing Latitude information " , " y " , false ) ;
inputs . add ( xDimension ) ;
inputs . add ( yDimension ) ;
IOHelper . addDoubleInput ( inputs , resolutionParameter , " The resolution of the CSquare codes " , " 0.1 " ) ;
IOHelper . addStringInput ( inputs , outputTableParameter , " The name of the output table " , " csquaretbl_ " ) ;
DatabaseType . addDefaultDBPars ( inputs ) ;
}
@Override
public StatisticalType getOutput ( ) {
List < TableTemplates > template = new ArrayList < TableTemplates > ( ) ;
template . add ( TableTemplates . GENERIC ) ;
OutputTable p = new OutputTable ( template , outTableLabel , outTable , " Output table " ) ;
return p ;
}
@Override
public void init ( ) throws Exception {
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator Initialized " ) ;
}
@Override
public String getDescription ( ) {
return " An algorithm that adds a column containing the CSquare codes associated to longitude and latitude columns. " ;
}
@Override
protected void process ( ) throws Exception {
status = 0 ;
AnalysisLogger . setLogger ( config . getConfigPath ( ) + AlgorithmConfiguration . defaultLoggerFile ) ;
long t0 = System . currentTimeMillis ( ) ;
String x = IOHelper . getInputParameter ( config , xDim ) ;
String y = IOHelper . getInputParameter ( config , yDim ) ;
String table = IOHelper . getInputParameter ( config , inputTableParameter ) ;
2015-02-09 17:24:11 +01:00
outTable = ( " code_ " + UUID . randomUUID ( ) ) . replace ( " - " , " " ) ;
2015-02-03 16:37:47 +01:00
resolution = Double . parseDouble ( IOHelper . getInputParameter ( config , resolutionParameter ) ) ;
outTableLabel = IOHelper . getInputParameter ( config , outputTableParameter ) ;
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator: received parameters: x " + x + " , y " + y + " , table " + table + " , outputTable " + outTable + " , res " + resolution + " outLabel " + outTableLabel ) ;
status = 10 ;
if ( x = = null | | x . trim ( ) . length ( ) = = 0 | | y = = null | | y . trim ( ) . length ( ) = = 0 )
2015-10-30 12:45:51 +01:00
throw new Exception ( " Error please provide information for the input table " ) ;
2015-02-09 17:24:11 +01:00
try {
addCodeColumToTable ( table ) ;
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator: finished " ) ;
} catch ( Throwable e ) {
throw new Exception ( e . getMessage ( ) ) ;
} finally {
status = 100 ;
}
}
public String selectInformationForTransformation ( AlgorithmConfiguration config , String table , int limit , int offset ) {
String x = IOHelper . getInputParameter ( config , xDim ) ;
String y = IOHelper . getInputParameter ( config , yDim ) ;
2015-02-10 19:50:16 +01:00
String select = " select *, " + x + " as loforcs01, " + y + " as laforcs01 from " + table + " limit " + limit + " offset " + offset ;
2015-02-09 17:24:11 +01:00
return select ;
}
public String rowToCode ( Object [ ] rowArray ) {
// take x and y
String xValue = " " + rowArray [ rowArray . length - 2 ] ;
String yValue = " " + rowArray [ rowArray . length - 1 ] ;
// generate csquarecodes
String csquare = " " ;
try {
double xV = Double . parseDouble ( xValue ) ;
double yV = Double . parseDouble ( yValue ) ;
csquare = CSquareCodesConverter . convertAtResolution ( yV , xV , resolution ) ;
} catch ( Exception e ) {
}
return csquare ;
}
2015-02-03 16:37:47 +01:00
2015-02-09 17:24:11 +01:00
public void addCodeColumToTable ( String table ) throws Exception {
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: initializing connection " ) ;
long t0 = System . currentTimeMillis ( ) ;
2015-02-03 16:37:47 +01:00
try {
connection = DatabaseUtils . initDBSession ( config ) ;
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: database: " + config . getDatabaseURL ( ) ) ;
2015-02-03 16:37:47 +01:00
// create a new output table
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: dropping table " + outTable + " if exists " ) ;
2015-02-03 16:37:47 +01:00
try {
DatabaseFactory . executeSQLUpdate ( DatabaseUtils . dropTableStatement ( outTable ) , connection ) ;
} catch ( Exception e ) {
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: table " + outTable + " does not exist yet " ) ;
2015-02-03 16:37:47 +01:00
}
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: creating the new table " + outTable ) ;
2015-02-03 16:37:47 +01:00
DatabaseFactory . executeSQLUpdate ( DatabaseUtils . createBlankTableFromAnotherStatement ( table , outTable ) , connection ) ;
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: adding new column to " + outTable ) ;
DatabaseFactory . executeSQLUpdate ( DatabaseUtils . addColumnStatement ( outTable , codecolumnName , " character varying " ) , connection ) ;
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: getting columns from " + outTable ) ;
2015-02-03 16:37:47 +01:00
// get columns names
List < Object > names = DatabaseFactory . executeSQLQuery ( DatabaseUtils . getColumnsNamesStatement ( outTable ) , connection ) ;
StringBuffer colnames = new StringBuffer ( ) ;
int nnames = names . size ( ) ;
for ( int i = 0 ; i < nnames ; i + + ) {
colnames . append ( names . get ( i ) ) ;
if ( i < nnames - 1 )
colnames . append ( " , " ) ;
}
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: columns are: " + colnames . toString ( ) ) ;
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: taking chunks ... " ) ;
2015-02-03 16:37:47 +01:00
// take chunks of the table
int initIdx = 0 ;
int limit = 5000 ;
long maxRows = DatabaseUtils . estimateNumberofRows ( table , connection ) ;
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: estimated number of rows " + maxRows ) ;
2015-02-03 16:37:47 +01:00
status = 20 ;
while ( true ) {
2015-02-09 17:24:11 +01:00
String select = selectInformationForTransformation ( config , table , limit , initIdx ) ;
2015-04-16 11:22:04 +02:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: executing query: " + select ) ;
2015-02-10 19:50:16 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: from " + initIdx + " to " + ( initIdx + limit ) + " limit " + limit ) ;
2015-02-03 16:37:47 +01:00
List < Object > rows = DatabaseFactory . executeSQLQuery ( select , connection ) ;
2015-02-10 19:50:16 +01:00
2015-02-03 16:37:47 +01:00
if ( rows = = null | | rows . size ( ) = = 0 ) {
2015-02-09 17:24:11 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: no more rows " ) ;
2015-02-03 16:37:47 +01:00
break ;
}
2015-02-10 19:50:16 +01:00
AnalysisLogger . getLogger ( ) . debug ( " CodeCreator: transforming " ) ;
2015-02-03 16:37:47 +01:00
// take x and y
List < String [ ] > stringrows = new ArrayList < String [ ] > ( ) ;
for ( Object row : rows ) {
Object [ ] rowArray = ( Object [ ] ) row ;
2015-02-09 17:24:11 +01:00
String code = rowToCode ( rowArray ) ;
rowArray [ nnames - 1 ] = code ;
String [ ] stringArray = new String [ nnames ] ;
2015-02-03 16:37:47 +01:00
// convert all the objects into Strings
for ( int k = 0 ; k < stringArray . length ; k + + ) {
stringArray [ k ] = " " + rowArray [ k ] ;
}
stringrows . add ( stringArray ) ;
}
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator: inserting chunks into the table " ) ;
// write the vector into the table
DatabaseUtils . insertChunksIntoTable ( outTable , colnames . toString ( ) , stringrows , limit , connection , true ) ;
2015-02-10 19:50:16 +01:00
initIdx = initIdx + limit ;
2015-02-03 16:37:47 +01:00
status = Math . min ( 90 , 20 + ( 70 * initIdx / maxRows ) ) ;
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator: status " + status ) ;
}
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator: finished " ) ;
} catch ( Throwable e ) {
e . printStackTrace ( ) ;
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator : ERROR!: " + e . getLocalizedMessage ( ) ) ;
try {
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator: dropping " + outTable ) ;
DatabaseFactory . executeSQLUpdate ( DatabaseUtils . dropTableStatement ( outTable ) , connection ) ;
} catch ( Exception e1 ) {
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator: could not drop " + outTable ) ;
}
throw new Exception ( e . getMessage ( ) ) ;
} finally {
shutdown ( ) ;
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator finished in " + ( System . currentTimeMillis ( ) - t0 ) + " ms " ) ;
}
}
2015-02-09 17:24:11 +01:00
2015-02-03 16:37:47 +01:00
@Override
public void shutdown ( ) {
AnalysisLogger . getLogger ( ) . debug ( " CSquareCreator shutdown " ) ;
DatabaseUtils . closeDBConnection ( connection ) ;
}
}
2015-11-02 11:22:07 +01:00