2012-08-23 14:15:56 +02:00
package org.gcube.dataanalysis.ecoengine.utils ;
2012-10-19 18:36:47 +02:00
import java.io.FileInputStream ;
2013-06-24 18:15:24 +02:00
import java.io.FileWriter ;
2012-10-19 18:36:47 +02:00
import java.sql.Connection ;
2012-08-23 14:15:56 +02:00
import java.util.List ;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration ;
import org.hibernate.SessionFactory ;
2012-10-19 18:36:47 +02:00
import org.postgresql.copy.CopyManager ;
import org.postgresql.core.BaseConnection ;
2017-07-06 18:16:42 +02:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
2012-08-23 14:15:56 +02:00
public class DatabaseUtils {
2017-07-06 18:16:42 +02:00
private static Logger LOGGER = LoggerFactory . getLogger ( DatabaseUtils . class ) ;
2012-08-23 14:15:56 +02:00
static String queryDesc = " SELECT column_name,data_type, character_maximum_length, is_nullable FROM information_schema.COLUMNS WHERE table_name ='%1$s' " ;
2012-09-14 18:10:52 +02:00
static String queryColumns = " SELECT column_name FROM information_schema.COLUMNS WHERE table_name ='%1$s' " ;
2012-08-23 14:15:56 +02:00
static String queryForKeys = " SELECT b.column_name as name, a.constraint_type as type FROM information_schema.table_constraints as a join information_schema.key_column_usage as b on a.table_name ='%1$s' and a.constraint_name = b.constraint_name " ;
static String genCreationStatement = " CREATE TABLE %1$s ( %2$s %3$s ); " ;
static String updateColValues = " UPDATE %1$s SET %2$s = %3$s.%4$s FROM %3$s WHERE %1$s.%5$s = %3$s.%6$s ; " ;
static String addColumn = " ALTER TABLE %1$s ADD COLUMN %2$s %3$s; " ;
SessionFactory referencedbConnection ;
public DatabaseUtils ( SessionFactory referencedbConnection ) {
this . referencedbConnection = referencedbConnection ;
}
public static String createUpdateStatement ( String tableToUpdate , String fieldToUpdate , String tableFromUpdate , String columnFromUpdate , String tableToUpKey , String tableFromUpKey ) {
return String . format ( updateColValues , tableToUpdate , fieldToUpdate , tableFromUpdate , columnFromUpdate , tableFromUpdate , tableToUpdate , tableToUpKey , tableFromUpdate , tableFromUpKey ) ;
}
2013-06-24 18:15:24 +02:00
public static String addColumnStatement ( String tableToUpdate , String columnName , String columnKey ) {
return String . format ( addColumn , tableToUpdate , columnName , columnKey ) ;
2012-08-23 14:15:56 +02:00
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
private String primaryK ;
private String primaryKColName ;
2017-07-06 18:16:42 +02:00
2015-02-03 15:03:24 +01:00
public static long estimateNumberofRows ( String table , SessionFactory dbconnection ) throws Exception {
List < Object > explain = DatabaseFactory . executeSQLQuery ( " EXPLAIN SELECT * FROM " + table , dbconnection ) ;
String explained = " " + explain . get ( 0 ) ;
explained = explained . substring ( explained . lastIndexOf ( " rows= " ) ) ;
explained = explained . substring ( explained . indexOf ( '=' ) + 1 , explained . indexOf ( ' ' ) ) ;
return Long . parseLong ( explained ) ;
}
2017-07-06 18:16:42 +02:00
2012-08-23 14:15:56 +02:00
private void getPrimaryKeys ( List < Object > keys , String table ) {
2013-06-24 18:15:24 +02:00
int keynum = 0 ;
if ( keys ! = null )
keynum = keys . size ( ) ;
2012-08-23 14:15:56 +02:00
StringBuffer pkeybuffer = new StringBuffer ( ) ;
primaryKColName = " " ;
for ( int i = 0 ; i < keynum ; i + + ) {
Object [ ] valueKey = ( Object [ ] ) keys . get ( i ) ;
String colname = ( String ) valueKey [ 0 ] ;
String type = ( String ) valueKey [ 1 ] ;
if ( type . equals ( " PRIMARY KEY " ) ) {
if ( pkeybuffer . length ( ) > 0 )
pkeybuffer . append ( " , " ) ;
pkeybuffer . append ( colname ) ;
if ( primaryKColName . length ( ) > 0 )
primaryKColName = " , " ;
primaryKColName = primaryKColName + colname ;
}
}
if ( pkeybuffer . length ( ) > 0 )
primaryK = " , CONSTRAINT " + table + " _idx PRIMARY KEY ( " + pkeybuffer . toString ( ) + " ) " ;
}
public String getPrimaryKey ( ) {
return primaryKColName ;
}
private String columnDescrs ;
private void getColumnsDesc ( List < Object > columnsDescs ) {
int num = columnsDescs . size ( ) ;
StringBuffer colbuffer = new StringBuffer ( ) ;
for ( int i = 0 ; i < num ; i + + ) {
Object [ ] descriptions = ( Object [ ] ) columnsDescs . get ( i ) ;
String colname = " " + descriptions [ 0 ] ;
String type = " " + descriptions [ 1 ] ;
String len = " " + descriptions [ 2 ] ;
String isnullable = " " + descriptions [ 3 ] ;
if ( len ! = null & & len . length ( ) > 0 & & ! len . equals ( " null " ) )
type = type + " ( " + len + " ) " ;
if ( isnullable ! = null & & isnullable . equalsIgnoreCase ( " NO " ) )
isnullable = " NOT NULL " ;
else
isnullable = " " ;
colbuffer . append ( colname + " " + type + " " + isnullable ) ;
if ( i < num - 1 ) {
colbuffer . append ( " , " ) ;
}
}
columnDescrs = colbuffer . toString ( ) ;
}
2012-09-14 18:10:52 +02:00
public List < Object > columns ;
2012-08-23 14:15:56 +02:00
public List < Object > getColumnDecriptions ( ) {
return columns ;
}
public String getColumnName ( int index ) {
return " " + ( ( Object [ ] ) getColumnDecriptions ( ) . get ( index ) ) [ 0 ] ;
}
public String getColumnType ( int index ) {
return " " + ( ( Object [ ] ) getColumnDecriptions ( ) . get ( index ) ) [ 1 ] ;
}
public static String duplicateTableStatement ( String tableFrom , String tableTo ) {
return " select * into " + tableTo + " from " + tableFrom ;
}
2012-09-20 14:47:28 +02:00
public static String createBlankTableFromAnotherStatement ( String tableFrom , String tableTo ) {
2013-06-24 18:15:24 +02:00
return " select * into " + tableTo + " from (select * from " + tableFrom + " limit 0) a " ;
2012-09-20 14:47:28 +02:00
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
public static String dropTableStatement ( String table ) {
return " drop table " + table ;
}
2013-06-24 18:15:24 +02:00
public static String getDinstictElements ( String table , String columns , String filter ) {
return " select distinct " + columns + " from " + table + " " + filter + " order by " + columns ;
2012-08-23 14:15:56 +02:00
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
public static String getOrderedElements ( String table , String key , String column ) {
return " select " + key + " , " + column + " from " + table + " order by " + key ;
}
2013-06-24 18:15:24 +02:00
public static String sumElementsStatement ( String table , String column ) {
2012-08-23 14:15:56 +02:00
return " select sum( " + column + " ) from " + table ;
}
2013-06-24 18:15:24 +02:00
public static String getColumnsElementsStatement ( String table , String columns , String filter ) {
return " select " + columns + " from " + table + " " + filter ;
2012-08-23 14:15:56 +02:00
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
public static String countElementsStatement ( String table ) {
return " select count(*) from " + table ;
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
public static String update ( String table , String valueColumnName , String value , String keyColumnName , String key ) {
return " UPDATE " + table + " SET " + valueColumnName + " = ' " + value + " ' " + " WHERE " + keyColumnName + " = ' " + key + " ' " ;
}
public static String insertIntoColumn ( String table , String keyColumnName , String valueColumnName , List < Object > couplesColumnAndKeys ) {
StringBuffer buffer = new StringBuffer ( ) ;
int ncols = couplesColumnAndKeys . size ( ) ;
if ( ncols > 0 ) {
buffer . append ( " insert into " + table + " ( " + keyColumnName + " , " + valueColumnName + " ) values " ) ;
for ( int i = 0 ; i < ncols ; i + + ) {
Object [ ] couples = ( Object [ ] ) couplesColumnAndKeys . get ( i ) ;
String key = " " + couples [ 0 ] ;
String value = " " + couples [ 1 ] ;
buffer . append ( " ( " + key + " , " + value + " ) " ) ;
if ( i < ncols - 1 )
buffer . append ( " , " ) ;
}
}
return buffer . toString ( ) ;
}
public static String insertFromBuffer ( String table , String columnsNames , StringBuffer values ) {
2013-06-24 18:15:24 +02:00
return " insert into " + table + " ( " + columnsNames + " ) values " + values ;
2012-08-23 14:15:56 +02:00
}
2012-12-20 15:50:26 +01:00
2013-11-05 11:21:03 +01:00
public static void insertChunksIntoTable ( String table , String columnsNames , List < String [ ] > values , int chunkSize , SessionFactory dbconnection ) throws Exception {
2014-05-09 16:38:10 +02:00
insertChunksIntoTable ( table , columnsNames , values , chunkSize , dbconnection , true ) ;
}
2017-07-06 18:16:42 +02:00
2014-05-09 16:38:10 +02:00
public static void insertChunksIntoTable ( String table , String columnsNames , List < String [ ] > values , int chunkSize , SessionFactory dbconnection , boolean correctApos ) throws Exception {
2017-07-06 18:16:42 +02:00
2013-11-05 11:21:03 +01:00
int valuesize = values . size ( ) ;
StringBuffer sb = new StringBuffer ( ) ;
int stopIndex = 0 ;
for ( int i = 0 ; i < valuesize ; i + + ) {
String [ ] row = values . get ( i ) ;
sb . append ( " ( " ) ;
for ( int j = 0 ; j < row . length ; j + + ) {
2013-11-12 15:16:28 +01:00
String preprow = row [ j ] . replaceAll ( " ^' " , " " ) . replaceAll ( " '$ " , " " ) ;
2014-05-09 16:38:10 +02:00
if ( correctApos )
preprow = preprow . replace ( " ' " , " " + ( char ) 96 ) ;
2014-03-14 14:53:37 +01:00
if ( preprow . equalsIgnoreCase ( " NULL " ) )
sb . append ( preprow ) ;
else
sb . append ( " ' " + preprow + " ' " ) ;
2013-11-05 11:21:03 +01:00
if ( j < row . length - 1 )
sb . append ( " , " ) ;
}
sb . append ( " ) " ) ;
2013-11-05 17:50:28 +01:00
if ( stopIndex > 0 & & stopIndex % chunkSize = = 0 ) {
2013-11-05 11:21:03 +01:00
DatabaseFactory . executeSQLUpdate ( insertFromBuffer ( table , columnsNames , sb ) , dbconnection ) ;
stopIndex = chunkSize ;
sb = new StringBuffer ( ) ;
}
else if ( i < valuesize - 1 )
sb . append ( " , " ) ;
}
2017-07-06 18:16:42 +02:00
2013-11-05 11:21:03 +01:00
if ( stopIndex < valuesize - 1 ) {
2013-11-05 17:50:28 +01:00
if ( sb . length ( ) > 0 ) {
2017-07-06 18:16:42 +02:00
// System.out.println(sb);
2013-11-05 17:50:28 +01:00
try {
DatabaseFactory . executeSQLUpdate ( insertFromBuffer ( table , columnsNames , sb ) , dbconnection ) ;
} catch ( Exception e ) {
System . out . println ( " Query: " + sb ) ;
throw e ;
}
2017-07-06 18:16:42 +02:00
2013-11-05 17:50:28 +01:00
}
2013-11-05 11:21:03 +01:00
}
2017-07-06 18:16:42 +02:00
2013-11-05 11:21:03 +01:00
}
2017-07-06 18:16:42 +02:00
2012-12-20 15:50:26 +01:00
public static String insertFromString ( String table , String columnsNames , String values ) {
2013-06-24 18:15:24 +02:00
return " insert into " + table + " ( " + columnsNames + " ) values " + values ;
2012-12-20 15:50:26 +01:00
}
2013-06-24 18:15:24 +02:00
2012-09-19 18:04:35 +02:00
public static String deleteFromBuffer ( String table , StringBuffer couples ) {
2013-06-24 18:15:24 +02:00
return " delete from " + table + " where " + couples ;
2012-09-19 18:04:35 +02:00
}
2013-06-24 18:15:24 +02:00
public static String copyFileToTableStatement ( String file , String table ) {
return " COPY " + table + " FROM ' " + file + " ' DELIMITERS ';' WITH NULL AS 'null string' " ;
2012-08-23 14:15:56 +02:00
}
2013-06-24 18:15:24 +02:00
public static String copyFileFromTableStatement ( String file , String table , String delimiter , boolean withheader ) {
String withheaderS = " " ;
2013-06-20 17:51:39 +02:00
if ( withheader )
2013-06-24 18:15:24 +02:00
withheaderS = " CSV HEADER " ;
return " COPY " + table + " TO ' " + file + " ' DELIMITERS ' " + delimiter + " ' WITH NULL AS '' " + withheaderS ;
2013-06-20 17:51:39 +02:00
}
2013-06-24 18:15:24 +02:00
public static String updateTableColumnFromOther ( String tableName , String fieldToUpdate , String otherTable , String otherColumn , String keyColumn , String otherKeyColumn ) {
return " UPDATE " + tableName + " SET " + fieldToUpdate + " = " + otherTable + " . " + otherColumn + " FROM " + otherTable + " WHERE " + tableName + " . " + keyColumn + " = " + otherTable + " . " + otherKeyColumn ;
2012-08-23 14:15:56 +02:00
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
public static String updateColumn ( String table , String keyColumnName , String valueColumnName , List < Object > couplesColumnAndKeys ) {
StringBuffer buffer = new StringBuffer ( ) ;
int ncols = couplesColumnAndKeys . size ( ) ;
for ( int i = 0 ; i < ncols ; i + + ) {
Object [ ] couples = ( Object [ ] ) couplesColumnAndKeys . get ( i ) ;
String key = " " + couples [ 0 ] ;
String value = " " + couples [ 1 ] ;
buffer . append ( " UPDATE " + table + " SET " + valueColumnName + " = ' " + value + " ' " + " WHERE " + keyColumnName + " = ' " + key + " ' " ) ;
if ( i < ncols - 1 )
buffer . append ( " ; \ n " ) ;
}
return buffer . toString ( ) ;
}
public String buildCreateStatement ( String originaltable , String destinationTable ) {
// take the structure of table 1
List < Object > keys = DatabaseFactory . executeSQLQuery ( String . format ( queryForKeys , originaltable ) , referencedbConnection ) ;
getPrimaryKeys ( keys , destinationTable ) ;
columns = DatabaseFactory . executeSQLQuery ( String . format ( queryDesc , originaltable ) , referencedbConnection ) ;
getColumnsDesc ( columns ) ;
String creationStatement = String . format ( genCreationStatement , destinationTable , columnDescrs , primaryK ) ;
return creationStatement ;
}
2013-06-24 18:15:24 +02:00
2012-09-14 18:10:52 +02:00
public static String getColumnsNamesStatement ( String table ) {
String statement = String . format ( queryColumns , table ) ;
return statement ;
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
public static void createBigTable ( boolean createTable , String table , String dbdriver , String dbuser , String dbpassword , String dburl , String creationStatement , SessionFactory dbHibConnection ) throws Exception {
if ( createTable ) {
try {
2017-07-06 18:16:42 +02:00
LOGGER . debug ( " Dropping previous table if exists " ) ;
2012-08-23 14:15:56 +02:00
DatabaseFactory . executeSQLUpdate ( " drop table " + table , dbHibConnection ) ;
} catch ( Exception e ) {
2017-07-06 18:16:42 +02:00
2012-08-23 14:15:56 +02:00
}
2017-07-06 18:16:42 +02:00
LOGGER . debug ( " Creating Big Table " ) ;
2012-08-23 14:15:56 +02:00
DatabaseFactory . executeUpdateNoTransaction ( creationStatement , dbdriver , dbuser , dbpassword , dburl , true ) ;
}
}
2013-06-24 18:15:24 +02:00
public static void createRemoteTableFromFile ( String filePath , String tablename , String delimiter , boolean hasHeader , String username , String password , String databaseurl ) throws Exception {
Connection conn = DatabaseFactory . getDBConnection ( " org.postgresql.Driver " , username , password , databaseurl ) ;
2012-10-19 18:36:47 +02:00
CopyManager copyManager = new CopyManager ( ( BaseConnection ) conn ) ;
FileInputStream fis = new FileInputStream ( filePath ) ;
2013-06-24 18:15:24 +02:00
copyManager . copyIn ( String . format ( " COPY %s FROM STDIN WITH DELIMITER '%s' %s " , tablename , delimiter , ( hasHeader ) ? " CSV HEADER " : " CSV " ) , fis ) ;
2012-10-19 18:36:47 +02:00
conn . close ( ) ;
fis . close ( ) ;
}
2013-06-24 18:15:24 +02:00
public static void createLocalFileFromRemoteTable ( String filePath , String tablename , String delimiter , String username , String password , String databaseurl ) throws Exception {
Connection conn = DatabaseFactory . getDBConnection ( " org.postgresql.Driver " , username , password , databaseurl ) ;
CopyManager copyManager = new CopyManager ( ( BaseConnection ) conn ) ;
FileWriter fw = new FileWriter ( filePath ) ;
copyManager . copyOut ( String . format ( " COPY %s TO STDOUT WITH DELIMITER '%s' NULL AS '' " , tablename , delimiter ) , fw ) ;
conn . close ( ) ;
fw . close ( ) ;
}
2012-08-23 14:15:56 +02:00
public static SessionFactory initDBSession ( AlgorithmConfiguration config ) {
SessionFactory dbHibConnection = null ;
try {
if ( ( config ! = null ) & & ( config . getConfigPath ( ) ! = null ) ) {
String defaultDatabaseFile = config . getConfigPath ( ) + AlgorithmConfiguration . defaultConnectionFile ;
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
config . setDatabaseDriver ( config . getParam ( " DatabaseDriver " ) ) ;
config . setDatabaseUserName ( config . getParam ( " DatabaseUserName " ) ) ;
config . setDatabasePassword ( config . getParam ( " DatabasePassword " ) ) ;
config . setDatabaseURL ( config . getParam ( " DatabaseURL " ) ) ;
2017-07-06 18:16:42 +02:00
LOGGER . trace ( " initializing DB session {} {} " , config . getParam ( " DatabaseUserName " ) , config . getParam ( " DatabaseURL " ) ) ;
2012-08-23 14:15:56 +02:00
dbHibConnection = DatabaseFactory . initDBConnection ( defaultDatabaseFile , config ) ;
}
} catch ( Exception e ) {
2017-07-06 18:16:42 +02:00
LOGGER . error ( " ERROR IN DB INITIALIZATION " , e ) ;
2012-08-23 14:15:56 +02:00
}
return dbHibConnection ;
}
2013-06-24 18:15:24 +02:00
public static void closeDBConnection ( SessionFactory dbHibConnection ) {
try {
2012-08-23 14:15:56 +02:00
dbHibConnection . close ( ) ;
2013-06-24 18:15:24 +02:00
} catch ( Exception e ) {
}
2012-08-23 14:15:56 +02:00
}
2013-06-24 18:15:24 +02:00
2012-08-23 14:15:56 +02:00
}