package eu.openaire.urls_controller.configuration; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.beans.PropertyVetoException; import java.io.File; import java.io.FileReader; import java.sql.*; import java.util.Properties; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; public final class ImpalaConnector { private static final Logger logger = LoggerFactory.getLogger(ImpalaConnector.class); public static String impalaDriver; public static String impalaConnectionUrl; public static String poolName; public static int hikariMaxConnectionPoolSize; public static int hikariMinIdleConnections; public static int hikariConnectionTimeOut; public static int hikariIdleTimeOut; public static int hikariMaxLifetime; public static String oldDatabaseName = "pdfaggregation_i"; public static String databaseName = "pdfAggregationDatabase"; public static final Lock databaseLock = new ReentrantLock(true); // This lock is locking the threads trying to execute queries in the database. public static HikariDataSource hikariDataSource; private static final ImpalaConnector singletonObject = new ImpalaConnector(); public static ImpalaConnector getInstance() { return singletonObject; } public ImpalaConnector() { logger.info("Max available memory to the Controller: " + Runtime.getRuntime().maxMemory() + " bytes."); try { String dbSettingsPropertyFile = System.getProperty("user.dir") + File.separator + "src" + File.separator + "main" + File.separator + "resources" + File.separator + "application.properties"; FileReader fReader = new FileReader(dbSettingsPropertyFile); Properties props = new Properties(); props.load(fReader); // Load jdbc related properties. // Get each property value. impalaDriver = props.getProperty("spring.impala.driver-class-name"); if ( !"".equals(impalaDriver) ) { // If not "null" or empty. Class.forName(impalaDriver); impalaConnectionUrl = props.getProperty("spring.impala.url"); poolName = props.getProperty("spring.datasource.hikari.pool-name"); hikariMaxConnectionPoolSize = Integer.parseInt(props.getProperty("spring.datasource.hikari.maximumPoolSize")); hikariMaxLifetime = Integer.parseInt(props.getProperty("spring.datasource.hikari.maxLifetime")); hikariMinIdleConnections = Integer.parseInt(props.getProperty("spring.datasource.hikari.minimumIdle")); hikariConnectionTimeOut = Integer.parseInt(props.getProperty("spring.datasource.hikari.connectionTimeout")); hikariIdleTimeOut = Integer.parseInt(props.getProperty("spring.datasource.hikari.idleTimeout")); } else throw new RuntimeException("The \"impalaDriver\" was null or empty!"); } catch(Exception e) { String errorMsg = "Error when loading the database properties!\n" + e.getMessage(); logger.error(errorMsg, e); System.err.println(errorMsg); System.exit(11); } try { hikariDataSource = impalaDS(); } catch (SQLException | PropertyVetoException e) { logger.error("Problem when creating the Hikari connection pool!", e); } createDatabase(); } public HikariDataSource impalaDS() throws SQLException, PropertyVetoException { HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setDriverClassName(ImpalaConnector.impalaDriver); hikariConfig.setAutoCommit(true); hikariConfig.setJdbcUrl(ImpalaConnector.impalaConnectionUrl); hikariConfig.setPoolName(poolName); hikariConfig.setMaximumPoolSize(hikariMaxConnectionPoolSize); hikariConfig.setMaxLifetime(hikariMaxLifetime); hikariConfig.setMinimumIdle(hikariMinIdleConnections); hikariConfig.setConnectionTimeout(hikariConnectionTimeOut); hikariConfig.setIdleTimeout(hikariIdleTimeOut); return new HikariDataSource(hikariConfig); } public void createDatabase() { Connection con = getConnection(); if ( con == null ) System.exit(22); try { if ( !con.getMetaData().supportsBatchUpdates() ) logger.warn("The database does not support \"BatchUpdates\"!"); } catch (SQLException e) { logger.error(e.getMessage(), e); } logger.info("Going to create the database and the tables, if they do not exist. Also will fill some tables with data from OpenAIRE."); Statement statement = null; try { statement = con.createStatement(); } catch (SQLException sqle) { logger.error("Problem when creating a connection-statement!\n" + sqle.getMessage()); ImpalaConnector.closeConnection(con); System.exit(33); } try { statement.execute("CREATE DATABASE IF NOT EXISTS " + databaseName); statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication stored as parquet as select * from " + oldDatabaseName + ".publication"); statement.execute("COMPUTE STATS " + databaseName + ".publication"); statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication_pids stored as parquet as select * from " + oldDatabaseName + ".publication_pids"); statement.execute("COMPUTE STATS " + databaseName + ".publication_pids"); statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication_urls stored as parquet as select * from " + oldDatabaseName + ".publication_urls"); statement.execute("COMPUTE STATS " + databaseName + ".publication_urls"); statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".datasource stored as parquet as select * from " + oldDatabaseName + ".datasource"); statement.execute("COMPUTE STATS " + databaseName + ".datasource"); statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".assignment (id string, original_url string, workerid string, `date` timestamp) stored as parquet"); statement.execute("COMPUTE STATS " + databaseName + ".assignment"); statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".attempt (id string, original_url string, `date` timestamp, status string, error_class string, error_message string) stored as parquet"); statement.execute("COMPUTE STATS " + databaseName + ".attempt"); statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".payload (id string, original_url string, actual_url string, `date` timestamp, mimetype string, size string, `hash` string, `location` string, provenance string) stored as parquet"); statement.execute("COMPUTE STATS " + databaseName + ".payload"); } catch (SQLException sqle) { String errorMsg = "Problem when executing the \"create database and create tables queries!\n" + sqle.getMessage() + "\nSQL state: " + sqle.getSQLState() + "\nError code: " + sqle.getErrorCode(); logger.error(errorMsg, sqle); System.err.println(errorMsg); System.exit(44); } finally { try { statement.close(); con.close(); } catch (SQLException sqle2) { logger.error("Could not close the connection with the Impala-database.\n" + sqle2.getMessage()); } } logger.info("The database \"" + databaseName + "\" and its tables were created or validated."); } public Connection getConnection() { try { return hikariDataSource.getConnection(); //return DriverManager.getConnection(impalaConnectionUrl, null, null); // This is for non pooled connections. } catch (SQLException sqle) { logger.error("Problem when connecting with the Impala-database!\n" + sqle.getMessage()); return null; } } public boolean testDatabaseAccess() { logger.info("Going to test Impala access.."); Connection con = getConnection(); if ( con == null ) return false; ResultSet res = null; try { String tableName = "publication"; // show tables String sql = "show tables '" + tableName + "'"; logger.debug("Running: " + sql); res = con.prepareStatement(sql).executeQuery(); if ( res.next() ) { logger.debug(res.getString(1)); } // describe table sql = "describe " + tableName; logger.debug("Running: " + sql); res = con.prepareStatement(sql).executeQuery(); while ( res.next() ) { logger.debug(res.getString(1) + "\t" + res.getString(2)); } // select * query sql = "select * from " + tableName + " limit 3;"; logger.debug("Running: " + sql); res = con.prepareStatement(sql).executeQuery(); while ( res.next() ) { logger.debug(res.getString(1)); } // Get Assignments, only for testing here. //UrlController urlController = new UrlController(); //ResponseEntity responseEntity = urlController.getUrls("worker_1", ControllerConstants.ASSIGNMENTS_LIMIT); //logger.debug(responseEntity.toString()); } catch (SQLException sqle) { logger.error(sqle.getMessage(), sqle); return false; } finally { try { if ( res != null ) res.close(); con.close(); } catch (SQLException sqle) { logger.error("Could not close the connection with the Impala-database.\n" + sqle); } } return true; } public static boolean closeConnection(Connection con) { try { if ( con != null ) con.close(); // It may have already closed and that's fine. return true; } catch (SQLException sqle) { logger.error("Could not close the connection with the Impala-database.\n" + sqle.getMessage()); return false; } } public static String handlePreparedStatementException(String queryName, String query, String preparedStatementName, PreparedStatement preparedStatement, Connection con, Exception e) { String errorMsg = "Problem when creating " + (( ! queryName.startsWith("get")) ? "and executing " : "") + "the prepared statement for \"" + queryName + "\"!\n"; logger.error(errorMsg + "\n\n" + query + "\n\n" + e.getMessage(), e); closeConnection(con); return errorMsg; } }