You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
253 lines
11 KiB
Java
253 lines
11 KiB
Java
package eu.openaire.urls_controller.configuration;
|
|
|
|
import com.zaxxer.hikari.HikariConfig;
|
|
import com.zaxxer.hikari.HikariDataSource;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import java.beans.PropertyVetoException;
|
|
import java.io.File;
|
|
import java.io.FileReader;
|
|
import java.sql.*;
|
|
import java.util.Properties;
|
|
import java.util.concurrent.locks.Lock;
|
|
import java.util.concurrent.locks.ReentrantLock;
|
|
|
|
|
|
public final class ImpalaConnector {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(ImpalaConnector.class);
|
|
|
|
public static String impalaDriver;
|
|
public static String impalaConnectionUrl;
|
|
public static String poolName;
|
|
public static int hikariMaxConnectionPoolSize;
|
|
public static int hikariMinIdleConnections;
|
|
public static int hikariConnectionTimeOut;
|
|
public static int hikariIdleTimeOut;
|
|
public static int hikariMaxLifetime;
|
|
|
|
public static String oldDatabaseName = "pdfaggregation_i";
|
|
public static String databaseName = "pdfAggregationDatabase";
|
|
|
|
public static final Lock databaseLock = new ReentrantLock(true); // This lock is locking the threads trying to execute queries in the database.
|
|
|
|
public static HikariDataSource hikariDataSource;
|
|
|
|
private static final ImpalaConnector singletonObject = new ImpalaConnector();
|
|
|
|
public static ImpalaConnector getInstance()
|
|
{
|
|
return singletonObject;
|
|
}
|
|
|
|
|
|
public ImpalaConnector()
|
|
{
|
|
logger.info("Max available memory to the Controller: " + Runtime.getRuntime().maxMemory() + " bytes.");
|
|
|
|
try {
|
|
String dbSettingsPropertyFile = System.getProperty("user.dir") + File.separator + "src" + File.separator + "main" + File.separator + "resources" + File.separator + "application.properties";
|
|
FileReader fReader = new FileReader(dbSettingsPropertyFile);
|
|
Properties props = new Properties();
|
|
props.load(fReader); // Load jdbc related properties.
|
|
|
|
// Get each property value.
|
|
impalaDriver = props.getProperty("spring.impala.driver-class-name");
|
|
if ( !"".equals(impalaDriver) ) { // If not "null" or empty.
|
|
Class.forName(impalaDriver);
|
|
impalaConnectionUrl = props.getProperty("spring.impala.url");
|
|
poolName = props.getProperty("spring.datasource.hikari.pool-name");
|
|
hikariMaxConnectionPoolSize = Integer.parseInt(props.getProperty("spring.datasource.hikari.maximumPoolSize"));
|
|
hikariMaxLifetime = Integer.parseInt(props.getProperty("spring.datasource.hikari.maxLifetime"));
|
|
hikariMinIdleConnections = Integer.parseInt(props.getProperty("spring.datasource.hikari.minimumIdle"));
|
|
hikariConnectionTimeOut = Integer.parseInt(props.getProperty("spring.datasource.hikari.connectionTimeout"));
|
|
hikariIdleTimeOut = Integer.parseInt(props.getProperty("spring.datasource.hikari.idleTimeout"));
|
|
} else
|
|
throw new RuntimeException("The \"impalaDriver\" was null or empty!");
|
|
} catch(Exception e) {
|
|
String errorMsg = "Error when loading the database properties!\n" + e.getMessage();
|
|
logger.error(errorMsg, e);
|
|
System.err.println(errorMsg);
|
|
System.exit(11);
|
|
}
|
|
|
|
try {
|
|
hikariDataSource = impalaDS();
|
|
} catch (SQLException | PropertyVetoException e) {
|
|
logger.error("Problem when creating the Hikari connection pool!", e);
|
|
}
|
|
|
|
createDatabase();
|
|
}
|
|
|
|
|
|
public HikariDataSource impalaDS() throws SQLException, PropertyVetoException
|
|
{
|
|
HikariConfig hikariConfig = new HikariConfig();
|
|
hikariConfig.setDriverClassName(ImpalaConnector.impalaDriver);
|
|
hikariConfig.setAutoCommit(true);
|
|
hikariConfig.setJdbcUrl(ImpalaConnector.impalaConnectionUrl);
|
|
hikariConfig.setPoolName(poolName);
|
|
hikariConfig.setMaximumPoolSize(hikariMaxConnectionPoolSize);
|
|
hikariConfig.setMaxLifetime(hikariMaxLifetime);
|
|
hikariConfig.setMinimumIdle(hikariMinIdleConnections);
|
|
hikariConfig.setConnectionTimeout(hikariConnectionTimeOut);
|
|
hikariConfig.setIdleTimeout(hikariIdleTimeOut);
|
|
return new HikariDataSource(hikariConfig);
|
|
}
|
|
|
|
|
|
public void createDatabase()
|
|
{
|
|
Connection con = getConnection();
|
|
if ( con == null )
|
|
System.exit(22);
|
|
|
|
try {
|
|
if ( !con.getMetaData().supportsBatchUpdates() )
|
|
logger.warn("The database does not support \"BatchUpdates\"!");
|
|
} catch (SQLException e) {
|
|
logger.error(e.getMessage(), e);
|
|
}
|
|
|
|
logger.info("Going to create the database and the tables, if they do not exist. Also will fill some tables with data from OpenAIRE.");
|
|
Statement statement = null;
|
|
try {
|
|
statement = con.createStatement();
|
|
} catch (SQLException sqle) {
|
|
logger.error("Problem when creating a connection-statement!\n" + sqle.getMessage());
|
|
ImpalaConnector.closeConnection(con);
|
|
System.exit(33);
|
|
}
|
|
|
|
try {
|
|
statement.execute("CREATE DATABASE IF NOT EXISTS " + databaseName);
|
|
|
|
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication stored as parquet as select * from " + oldDatabaseName + ".publication");
|
|
statement.execute("COMPUTE STATS " + databaseName + ".publication");
|
|
|
|
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication_pids stored as parquet as select * from " + oldDatabaseName + ".publication_pids");
|
|
statement.execute("COMPUTE STATS " + databaseName + ".publication_pids");
|
|
|
|
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication_urls stored as parquet as select * from " + oldDatabaseName + ".publication_urls");
|
|
statement.execute("COMPUTE STATS " + databaseName + ".publication_urls");
|
|
|
|
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".datasource stored as parquet as select * from " + oldDatabaseName + ".datasource");
|
|
statement.execute("COMPUTE STATS " + databaseName + ".datasource");
|
|
|
|
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".assignment (id string, original_url string, workerid string, `date` timestamp) stored as parquet");
|
|
statement.execute("COMPUTE STATS " + databaseName + ".assignment");
|
|
|
|
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".attempt (id string, original_url string, `date` timestamp, status string, error_class string, error_message string) stored as parquet");
|
|
statement.execute("COMPUTE STATS " + databaseName + ".attempt");
|
|
|
|
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".payload (id string, original_url string, actual_url string, `date` timestamp, mimetype string, size string, `hash` string, `location` string, provenance string) stored as parquet");
|
|
statement.execute("COMPUTE STATS " + databaseName + ".payload");
|
|
} catch (SQLException sqle) {
|
|
String errorMsg = "Problem when executing the \"create database and create tables queries!\n" + sqle.getMessage() + "\nSQL state: " + sqle.getSQLState() + "\nError code: " + sqle.getErrorCode();
|
|
logger.error(errorMsg, sqle);
|
|
System.err.println(errorMsg);
|
|
System.exit(44);
|
|
} finally {
|
|
try {
|
|
statement.close();
|
|
con.close();
|
|
} catch (SQLException sqle2) {
|
|
logger.error("Could not close the connection with the Impala-database.\n" + sqle2.getMessage());
|
|
}
|
|
}
|
|
|
|
logger.info("The database \"" + databaseName + "\" and its tables were created or validated.");
|
|
}
|
|
|
|
|
|
public Connection getConnection()
|
|
{
|
|
try {
|
|
return hikariDataSource.getConnection();
|
|
//return DriverManager.getConnection(impalaConnectionUrl, null, null); // This is for non pooled connections.
|
|
} catch (SQLException sqle) {
|
|
logger.error("Problem when connecting with the Impala-database!\n" + sqle.getMessage());
|
|
return null;
|
|
}
|
|
}
|
|
|
|
|
|
public boolean testDatabaseAccess()
|
|
{
|
|
logger.info("Going to test Impala access..");
|
|
Connection con = getConnection();
|
|
if ( con == null )
|
|
return false;
|
|
|
|
ResultSet res = null;
|
|
try {
|
|
String tableName = "publication";
|
|
|
|
// show tables
|
|
String sql = "show tables '" + tableName + "'";
|
|
logger.debug("Running: " + sql);
|
|
res = con.prepareStatement(sql).executeQuery();
|
|
if ( res.next() ) {
|
|
logger.debug(res.getString(1));
|
|
}
|
|
|
|
// describe table
|
|
sql = "describe " + tableName;
|
|
logger.debug("Running: " + sql);
|
|
res = con.prepareStatement(sql).executeQuery();
|
|
while ( res.next() ) {
|
|
logger.debug(res.getString(1) + "\t" + res.getString(2));
|
|
}
|
|
|
|
// select * query
|
|
sql = "select * from " + tableName + " limit 3;";
|
|
logger.debug("Running: " + sql);
|
|
res = con.prepareStatement(sql).executeQuery();
|
|
while ( res.next() ) {
|
|
logger.debug(res.getString(1));
|
|
}
|
|
|
|
// Get Assignments, only for testing here.
|
|
//UrlController urlController = new UrlController();
|
|
//ResponseEntity<?> responseEntity = urlController.getUrls("worker_1", ControllerConstants.ASSIGNMENTS_LIMIT);
|
|
//logger.debug(responseEntity.toString());
|
|
|
|
} catch (SQLException sqle) {
|
|
logger.error(sqle.getMessage(), sqle);
|
|
return false;
|
|
} finally {
|
|
try {
|
|
if ( res != null )
|
|
res.close();
|
|
con.close();
|
|
} catch (SQLException sqle) {
|
|
logger.error("Could not close the connection with the Impala-database.\n" + sqle);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
|
|
public static boolean closeConnection(Connection con) {
|
|
try {
|
|
if ( con != null )
|
|
con.close(); // It may have already closed and that's fine.
|
|
return true;
|
|
} catch (SQLException sqle) {
|
|
logger.error("Could not close the connection with the Impala-database.\n" + sqle.getMessage());
|
|
return false;
|
|
}
|
|
}
|
|
|
|
|
|
public static String handlePreparedStatementException(String queryName, String query, String preparedStatementName, PreparedStatement preparedStatement, Connection con, Exception e)
|
|
{
|
|
String errorMsg = "Problem when creating " + (( ! queryName.startsWith("get")) ? "and executing " : "") + "the prepared statement for \"" + queryName + "\"!\n";
|
|
logger.error(errorMsg + "\n\n" + query + "\n\n" + e.getMessage(), e);
|
|
closeConnection(con);
|
|
return errorMsg;
|
|
}
|
|
|
|
}
|