git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@150851 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
dbaedc1ce7
commit
86460bc7a4
|
@ -4,11 +4,15 @@ package org.gcube.dataanalysis.ecoengine.utils;
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.io.StringWriter;
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.sql.DriverManager;
|
import java.sql.DriverManager;
|
||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
import javax.xml.parsers.DocumentBuilderFactory;
|
import javax.xml.parsers.DocumentBuilderFactory;
|
||||||
import javax.xml.parsers.SAXParser;
|
import javax.xml.parsers.SAXParser;
|
||||||
|
@ -23,14 +27,19 @@ import org.hibernate.Query;
|
||||||
import org.hibernate.Session;
|
import org.hibernate.Session;
|
||||||
import org.hibernate.SessionFactory;
|
import org.hibernate.SessionFactory;
|
||||||
import org.hibernate.cfg.Configuration;
|
import org.hibernate.cfg.Configuration;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.xml.sax.Attributes;
|
import org.xml.sax.Attributes;
|
||||||
import org.xml.sax.SAXException;
|
import org.xml.sax.SAXException;
|
||||||
import org.xml.sax.helpers.DefaultHandler;
|
import org.xml.sax.helpers.DefaultHandler;
|
||||||
|
|
||||||
public class DatabaseFactory{
|
public class DatabaseFactory{
|
||||||
|
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseFactory.class);
|
||||||
|
|
||||||
public static SessionFactory initDBConnection(String configurationFile) throws Exception {
|
public static SessionFactory initDBConnection(String configurationFile) throws Exception {
|
||||||
String xml = FileTools.readXMLDoc(configurationFile);
|
String xml = FileTools.readXMLDoc(configurationFile);
|
||||||
|
LOGGER.debug("initialising DB Connection with conf: {} ",xml);
|
||||||
SessionFactory DBSessionFactory = null;
|
SessionFactory DBSessionFactory = null;
|
||||||
Configuration cfg = new Configuration();
|
Configuration cfg = new Configuration();
|
||||||
cfg = cfg.configure(DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new ByteArrayInputStream(xml.getBytes())));
|
cfg = cfg.configure(DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new ByteArrayInputStream(xml.getBytes())));
|
||||||
|
@ -43,6 +52,8 @@ public class DatabaseFactory{
|
||||||
|
|
||||||
final List<StatisticalType> defaultconfig = new ArrayList<StatisticalType>();
|
final List<StatisticalType> defaultconfig = new ArrayList<StatisticalType>();
|
||||||
|
|
||||||
|
LOGGER.debug("reading default DB configuration from {}",configurationFile);
|
||||||
|
|
||||||
// take the configuration file
|
// take the configuration file
|
||||||
File fl = new File(configurationFile);
|
File fl = new File(configurationFile);
|
||||||
FileInputStream stream = new FileInputStream(fl);
|
FileInputStream stream = new FileInputStream(fl);
|
||||||
|
@ -99,27 +110,70 @@ public class DatabaseFactory{
|
||||||
return defaultconfig;
|
return defaultconfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings({"unchecked"})
|
|
||||||
public static SessionFactory initDBConnection(String configurationFile, LexicalEngineConfiguration config) throws Exception {
|
public static SessionFactory initDBConnection(String configurationFile, LexicalEngineConfiguration config) throws Exception {
|
||||||
|
|
||||||
|
LOGGER.debug("init DB configuration from {}",configurationFile);
|
||||||
|
|
||||||
if (config==null)
|
if (config==null)
|
||||||
return initDBConnection(configurationFile);
|
return initDBConnection(configurationFile);
|
||||||
|
|
||||||
|
|
||||||
// take the configuration file
|
|
||||||
File fl = new File(configurationFile);
|
|
||||||
FileInputStream stream = new FileInputStream(fl);
|
|
||||||
|
|
||||||
Configuration cfg = new Configuration();
|
Configuration cfg = new Configuration();
|
||||||
cfg = cfg.configure(DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(stream));
|
|
||||||
cfg.setProperty("hibernate.hbm2ddl.auto", "create");
|
try(FileInputStream stream = new FileInputStream(new File(configurationFile))){
|
||||||
|
cfg = cfg.configure(DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(stream));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if ((config.getDatabaseDriver() != null)&&(config.getDatabaseDriver().length()>0))
|
||||||
|
cfg.setProperty("hibernate.connection.driver_class",config.getDatabaseDriver());
|
||||||
|
else
|
||||||
|
config.setDatabaseDriver(cfg.getProperty("connection.driver_class"));
|
||||||
|
|
||||||
|
if ((config.getDatabaseURL() != null)&&(config.getDatabaseURL().length()>0))
|
||||||
|
cfg.setProperty("hibernate.connection.url",config.getDatabaseURL());
|
||||||
|
else
|
||||||
|
config.setDatabaseURL(cfg.getProperty("connection.url"));
|
||||||
|
|
||||||
|
if ((config.getDatabaseUserName() != null)&&(config.getDatabaseUserName().length()>0))
|
||||||
|
cfg.setProperty("hibernate.connection.username", config.getDatabaseUserName());
|
||||||
|
else
|
||||||
|
config.setDatabaseUserName(cfg.getProperty("connection.username"));
|
||||||
|
|
||||||
|
if ((config.getDatabasePassword() != null)&&(config.getDatabasePassword().length()>0))
|
||||||
|
cfg.setProperty("hibernate.connection.password", config.getDatabasePassword());
|
||||||
|
else
|
||||||
|
config.setDatabasePassword(cfg.getProperty("connection.password"));
|
||||||
|
|
||||||
|
if ((config.getDatabaseDialect() != null)&&(config.getDatabaseDialect().length()>0))
|
||||||
|
cfg.setProperty("hibernate.dialect", config.getDatabaseDialect());
|
||||||
|
else
|
||||||
|
config.setDatabaseDialect(cfg.getProperty("dialect"));
|
||||||
|
|
||||||
|
if (config.getDatabaseIdleConnectionTestPeriod() != null)
|
||||||
|
cfg.setProperty("hibernate.c3p0.idleConnectionTestPeriod",config.getDatabaseIdleConnectionTestPeriod());
|
||||||
|
else
|
||||||
|
config.setDatabaseIdleConnectionTestPeriod(cfg.getProperty("c3p0.idleConnectionTestPeriod"));
|
||||||
|
|
||||||
|
if (config.getDatabaseAutomaticTestTable() != null)
|
||||||
|
cfg.setProperty("hibernate.c3p0.automaticTestTable",config.getDatabaseAutomaticTestTable());
|
||||||
|
else
|
||||||
|
config.setDatabaseAutomaticTestTable(cfg.getProperty("c3p0.automaticTestTable"));
|
||||||
|
|
||||||
|
|
||||||
|
cfg.setProperty("hibernate.hbm2ddl.auto", "create");
|
||||||
|
|
||||||
|
Properties props = cfg.getProperties();
|
||||||
|
|
||||||
|
/* LOGGER.debug("retrieving property for connection:");
|
||||||
|
|
||||||
|
for (Entry<Object, Object> propEntry : props.entrySet()){
|
||||||
|
LOGGER.debug("----- property {} -> {} ",propEntry.getKey(), propEntry.getValue());
|
||||||
|
}*/
|
||||||
|
|
||||||
SessionFactory DBSessionFactory = null;
|
SessionFactory DBSessionFactory = null;
|
||||||
DBSessionFactory = cfg.buildSessionFactory();
|
DBSessionFactory = cfg.buildSessionFactory();
|
||||||
|
|
||||||
// close stream
|
|
||||||
stream.close();
|
|
||||||
return DBSessionFactory;
|
return DBSessionFactory;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -228,7 +282,7 @@ public class DatabaseFactory{
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ss = DBSessionFactory.getCurrentSession();
|
ss = DBSessionFactory.getCurrentSession();
|
||||||
// System.out.println("executing query");
|
// System.out.println("executing query");
|
||||||
ss.beginTransaction();
|
ss.beginTransaction();
|
||||||
Query qr = null;
|
Query qr = null;
|
||||||
|
|
|
@ -5,14 +5,17 @@ import java.io.FileWriter;
|
||||||
import java.sql.Connection;
|
import java.sql.Connection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
|
||||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||||
import org.hibernate.SessionFactory;
|
import org.hibernate.SessionFactory;
|
||||||
import org.postgresql.copy.CopyManager;
|
import org.postgresql.copy.CopyManager;
|
||||||
import org.postgresql.core.BaseConnection;
|
import org.postgresql.core.BaseConnection;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class DatabaseUtils {
|
public class DatabaseUtils {
|
||||||
|
|
||||||
|
private static Logger LOGGER = LoggerFactory.getLogger(DatabaseUtils.class);
|
||||||
|
|
||||||
static String queryDesc = "SELECT column_name,data_type, character_maximum_length, is_nullable FROM information_schema.COLUMNS WHERE table_name ='%1$s'";
|
static String queryDesc = "SELECT column_name,data_type, character_maximum_length, is_nullable FROM information_schema.COLUMNS WHERE table_name ='%1$s'";
|
||||||
static String queryColumns = "SELECT column_name FROM information_schema.COLUMNS WHERE table_name ='%1$s'";
|
static String queryColumns = "SELECT column_name FROM information_schema.COLUMNS WHERE table_name ='%1$s'";
|
||||||
static String queryForKeys = "SELECT b.column_name as name, a.constraint_type as type FROM information_schema.table_constraints as a join information_schema.key_column_usage as b on a.table_name ='%1$s' and a.constraint_name = b.constraint_name";
|
static String queryForKeys = "SELECT b.column_name as name, a.constraint_type as type FROM information_schema.table_constraints as a join information_schema.key_column_usage as b on a.table_name ='%1$s' and a.constraint_name = b.constraint_name";
|
||||||
|
@ -35,7 +38,7 @@ public class DatabaseUtils {
|
||||||
|
|
||||||
private String primaryK;
|
private String primaryK;
|
||||||
private String primaryKColName;
|
private String primaryKColName;
|
||||||
|
|
||||||
public static long estimateNumberofRows(String table, SessionFactory dbconnection) throws Exception{
|
public static long estimateNumberofRows(String table, SessionFactory dbconnection) throws Exception{
|
||||||
List<Object> explain = DatabaseFactory.executeSQLQuery("EXPLAIN SELECT * FROM "+table, dbconnection);
|
List<Object> explain = DatabaseFactory.executeSQLQuery("EXPLAIN SELECT * FROM "+table, dbconnection);
|
||||||
String explained = "" + explain.get(0);
|
String explained = "" + explain.get(0);
|
||||||
|
@ -43,7 +46,7 @@ public class DatabaseUtils {
|
||||||
explained = explained.substring(explained.indexOf('=') + 1, explained.indexOf(' '));
|
explained = explained.substring(explained.indexOf('=') + 1, explained.indexOf(' '));
|
||||||
return Long.parseLong(explained);
|
return Long.parseLong(explained);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void getPrimaryKeys(List<Object> keys, String table) {
|
private void getPrimaryKeys(List<Object> keys, String table) {
|
||||||
int keynum = 0;
|
int keynum = 0;
|
||||||
if (keys != null)
|
if (keys != null)
|
||||||
|
@ -177,9 +180,9 @@ public class DatabaseUtils {
|
||||||
public static void insertChunksIntoTable(String table, String columnsNames, List<String[]> values, int chunkSize,SessionFactory dbconnection) throws Exception{
|
public static void insertChunksIntoTable(String table, String columnsNames, List<String[]> values, int chunkSize,SessionFactory dbconnection) throws Exception{
|
||||||
insertChunksIntoTable(table, columnsNames, values, chunkSize,dbconnection, true) ;
|
insertChunksIntoTable(table, columnsNames, values, chunkSize,dbconnection, true) ;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void insertChunksIntoTable(String table, String columnsNames, List<String[]> values, int chunkSize,SessionFactory dbconnection, boolean correctApos) throws Exception{
|
public static void insertChunksIntoTable(String table, String columnsNames, List<String[]> values, int chunkSize,SessionFactory dbconnection, boolean correctApos) throws Exception{
|
||||||
|
|
||||||
int valuesize = values.size();
|
int valuesize = values.size();
|
||||||
StringBuffer sb = new StringBuffer();
|
StringBuffer sb = new StringBuffer();
|
||||||
int stopIndex =0;
|
int stopIndex =0;
|
||||||
|
@ -206,23 +209,23 @@ public class DatabaseUtils {
|
||||||
else if (i<valuesize-1)
|
else if (i<valuesize-1)
|
||||||
sb.append(",");
|
sb.append(",");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (stopIndex<valuesize-1){
|
if (stopIndex<valuesize-1){
|
||||||
if (sb.length()>0){
|
if (sb.length()>0){
|
||||||
// System.out.println(sb);
|
// System.out.println(sb);
|
||||||
try{
|
try{
|
||||||
DatabaseFactory.executeSQLUpdate(insertFromBuffer(table, columnsNames, sb), dbconnection);
|
DatabaseFactory.executeSQLUpdate(insertFromBuffer(table, columnsNames, sb), dbconnection);
|
||||||
}catch(Exception e){
|
}catch(Exception e){
|
||||||
System.out.println("Query:"+sb);
|
System.out.println("Query:"+sb);
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public static String insertFromString(String table, String columnsNames, String values) {
|
public static String insertFromString(String table, String columnsNames, String values) {
|
||||||
|
|
||||||
return "insert into " + table + " (" + columnsNames + ") values " + values;
|
return "insert into " + table + " (" + columnsNames + ") values " + values;
|
||||||
|
@ -287,12 +290,12 @@ public class DatabaseUtils {
|
||||||
public static void createBigTable(boolean createTable, String table, String dbdriver, String dbuser, String dbpassword, String dburl, String creationStatement, SessionFactory dbHibConnection) throws Exception {
|
public static void createBigTable(boolean createTable, String table, String dbdriver, String dbuser, String dbpassword, String dburl, String creationStatement, SessionFactory dbHibConnection) throws Exception {
|
||||||
if (createTable) {
|
if (createTable) {
|
||||||
try {
|
try {
|
||||||
AnalysisLogger.getLogger().debug("Dropping previous table if exists");
|
LOGGER.debug("Dropping previous table if exists");
|
||||||
DatabaseFactory.executeSQLUpdate("drop table " + table, dbHibConnection);
|
DatabaseFactory.executeSQLUpdate("drop table " + table, dbHibConnection);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
// e.printStackTrace();
|
|
||||||
}
|
}
|
||||||
AnalysisLogger.getLogger().debug("Creating Big Table");
|
LOGGER.debug("Creating Big Table");
|
||||||
DatabaseFactory.executeUpdateNoTransaction(creationStatement, dbdriver, dbuser, dbpassword, dburl, true);
|
DatabaseFactory.executeUpdateNoTransaction(creationStatement, dbdriver, dbuser, dbpassword, dburl, true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -327,13 +330,13 @@ public class DatabaseUtils {
|
||||||
config.setDatabaseUserName(config.getParam("DatabaseUserName"));
|
config.setDatabaseUserName(config.getParam("DatabaseUserName"));
|
||||||
config.setDatabasePassword(config.getParam("DatabasePassword"));
|
config.setDatabasePassword(config.getParam("DatabasePassword"));
|
||||||
config.setDatabaseURL(config.getParam("DatabaseURL"));
|
config.setDatabaseURL(config.getParam("DatabaseURL"));
|
||||||
|
|
||||||
|
LOGGER.trace("initializing DB session {} {} ",config.getParam("DatabaseUserName"), config.getParam("DatabaseURL"));
|
||||||
|
|
||||||
dbHibConnection = DatabaseFactory.initDBConnection(defaultDatabaseFile, config);
|
dbHibConnection = DatabaseFactory.initDBConnection(defaultDatabaseFile, config);
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
System.out.println("ERROR IN DB INITIALIZATION : " + e.getLocalizedMessage());
|
LOGGER.error("ERROR IN DB INITIALIZATION",e);
|
||||||
e.printStackTrace();
|
|
||||||
// AnalysisLogger.getLogger().trace(e);
|
|
||||||
}
|
}
|
||||||
return dbHibConnection;
|
return dbHibConnection;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue