- Implement the "getUrls" and "addWorkerReport" endpoints with full database-handling.
- Add connectivity with an Impala-database and create a dedicated Controller for future statistics-requests. - Optimize the "getTestUrls"-endpoint. - Disable the "reportCurrentTime()" scheduled-task. - Update dependencies and bump project's version to '1.0.0-SNAPSHOT'. - Set the logging-appender to "File". - Code cleanup.springify_project
parent
0d47c33a08
commit
d100af35d0
@ -0,0 +1,239 @@
|
||||
package eu.openaire.urls_controller.configuration;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import java.beans.PropertyVetoException;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.sql.*;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
|
||||
public final class ImpalaConnector {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImpalaConnector.class);
|
||||
|
||||
public static String impalaDriver;
|
||||
public static String impalaConnectionUrl;
|
||||
public static String poolName;
|
||||
public static int hikariMaxConnectionPoolSize;
|
||||
public static int hikariMinIdleConnections;
|
||||
public static int hikariConnectionTimeOut;
|
||||
public static int hikariIdleTimeOut;
|
||||
public static int hikariMaxLifetime;
|
||||
|
||||
public static String oldDatabaseName = "pdfaggregation_i";
|
||||
public static String databaseName = "pdfAggregationDatabase";
|
||||
|
||||
public static final Lock databaseLock = new ReentrantLock(true); // This lock is locking the threads trying to execute queries in the database.
|
||||
|
||||
public static HikariDataSource hikariDataSource;
|
||||
|
||||
private static final ImpalaConnector singletonObject = new ImpalaConnector();
|
||||
|
||||
public static ImpalaConnector getInstance()
|
||||
{
|
||||
return singletonObject;
|
||||
}
|
||||
|
||||
|
||||
public HikariDataSource impalaDS() throws SQLException, PropertyVetoException
|
||||
{
|
||||
HikariConfig hikariConfig = new HikariConfig();
|
||||
hikariConfig.setDriverClassName(ImpalaConnector.impalaDriver);
|
||||
hikariConfig.setAutoCommit(true);
|
||||
hikariConfig.setJdbcUrl(ImpalaConnector.impalaConnectionUrl);
|
||||
hikariConfig.setPoolName(poolName);
|
||||
hikariConfig.setMaximumPoolSize(hikariMaxConnectionPoolSize);
|
||||
hikariConfig.setMaxLifetime(hikariMaxLifetime);
|
||||
hikariConfig.setMinimumIdle(hikariMinIdleConnections);
|
||||
hikariConfig.setConnectionTimeout(hikariConnectionTimeOut);
|
||||
hikariConfig.setIdleTimeout(hikariIdleTimeOut);
|
||||
return new HikariDataSource(hikariConfig);
|
||||
}
|
||||
|
||||
|
||||
public ImpalaConnector()
|
||||
{
|
||||
logger.info("Max available memory to the Controller: " + Runtime.getRuntime().maxMemory() + " bytes.");
|
||||
|
||||
try {
|
||||
String dbSettingsPropertyFile = System.getProperty("user.dir") + File.separator + "src" + File.separator + "main" + File.separator + "resources" + File.separator + "application.properties";
|
||||
FileReader fReader = new FileReader(dbSettingsPropertyFile);
|
||||
Properties props = new Properties();
|
||||
props.load(fReader); // Load jdbc related properties.
|
||||
|
||||
// Get each property value.
|
||||
impalaDriver = props.getProperty("spring.impala.driver-class-name");
|
||||
if ( !"".equals(impalaDriver) ) { // If not "null" or empty.
|
||||
Class.forName(impalaDriver);
|
||||
impalaConnectionUrl = props.getProperty("spring.impala.url");
|
||||
poolName = props.getProperty("spring.datasource.hikari.pool-name");
|
||||
hikariMaxConnectionPoolSize = Integer.parseInt(props.getProperty("spring.datasource.hikari.maximumPoolSize"));
|
||||
hikariMaxLifetime = Integer.parseInt(props.getProperty("spring.datasource.hikari.maxLifetime"));
|
||||
hikariMinIdleConnections = Integer.parseInt(props.getProperty("spring.datasource.hikari.minimumIdle"));
|
||||
hikariConnectionTimeOut = Integer.parseInt(props.getProperty("spring.datasource.hikari.connectionTimeout"));
|
||||
hikariIdleTimeOut = Integer.parseInt(props.getProperty("spring.datasource.hikari.idleTimeout"));
|
||||
} else
|
||||
throw new RuntimeException("The \"impalaDriver\" was null or empty!");
|
||||
} catch(Exception e) {
|
||||
logger.error("Error when loading the database properties!\n" + e.getMessage());
|
||||
e.printStackTrace();
|
||||
System.exit(11);
|
||||
}
|
||||
|
||||
try {
|
||||
hikariDataSource = impalaDS();
|
||||
} catch (SQLException | PropertyVetoException e) {
|
||||
logger.error("Problem when creating the Hikari connection pool!");
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
createDatabase();
|
||||
}
|
||||
|
||||
|
||||
public void createDatabase()
|
||||
{
|
||||
databaseLock.lock(); // Make sure the database and tables are created before the workers can request assignments.
|
||||
Connection con = getConnection();
|
||||
if ( con == null ) {
|
||||
databaseLock.unlock();
|
||||
System.exit(22);
|
||||
}
|
||||
|
||||
try {
|
||||
if ( !con.getMetaData().supportsBatchUpdates() )
|
||||
logger.warn("The database does not support \"BatchUpdates\"!");
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
logger.info("Going to create the database and the tables, if they do not exist. Also will fill some tables with data from OpenAIRE.");
|
||||
Statement statement = null;
|
||||
try {
|
||||
statement = con.createStatement();
|
||||
} catch (SQLException sqle) {
|
||||
logger.error("Problem when creating a connection-statement!\n" + sqle.getMessage());
|
||||
try {
|
||||
con.close();
|
||||
} catch (SQLException sqle2) {
|
||||
logger.error("Could not close the connection with the Impala-database.\n" + sqle2.getMessage());
|
||||
}
|
||||
databaseLock.unlock();
|
||||
System.exit(33);
|
||||
}
|
||||
|
||||
try {
|
||||
statement.execute("CREATE DATABASE IF NOT EXISTS " + databaseName);
|
||||
|
||||
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication stored as parquet as select * from " + oldDatabaseName + ".publication");
|
||||
statement.execute("COMPUTE STATS " + databaseName + ".publication");
|
||||
|
||||
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication_pids stored as parquet as select * from " + oldDatabaseName + ".publication_pids");
|
||||
statement.execute("COMPUTE STATS " + databaseName + ".publication_pids");
|
||||
|
||||
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".publication_urls stored as parquet as select * from " + oldDatabaseName + ".publication_urls");
|
||||
statement.execute("COMPUTE STATS " + databaseName + ".publication_urls");
|
||||
|
||||
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".datasource stored as parquet as select * from " + oldDatabaseName + ".datasource");
|
||||
statement.execute("COMPUTE STATS " + databaseName + ".datasource");
|
||||
|
||||
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".assignment (id string, original_url string, workerid string, `date` timestamp) stored as parquet");
|
||||
statement.execute("COMPUTE STATS " + databaseName + ".assignment");
|
||||
|
||||
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".attempt (id string, original_url string, `date` timestamp, status string, error_class string, error_message string) stored as parquet");
|
||||
statement.execute("COMPUTE STATS " + databaseName + ".attempt");
|
||||
|
||||
statement.execute("CREATE TABLE IF NOT EXISTS " + databaseName + ".payload (id string, original_url string, actual_url string, `date` timestamp, mimetype string, size string, `hash` string, `location` string, provenance string) stored as parquet");
|
||||
statement.execute("COMPUTE STATS " + databaseName + ".payload");
|
||||
} catch (SQLException sqle) {
|
||||
logger.error("Problem when executing the \"create database and create tables queries!\n" + sqle.getMessage() + "\nSQL state: " + sqle.getSQLState() + "\nError code: " + sqle.getErrorCode());
|
||||
sqle.printStackTrace();
|
||||
System.exit(44);
|
||||
} finally {
|
||||
databaseLock.unlock();
|
||||
try {
|
||||
statement.close();
|
||||
con.close();
|
||||
} catch (SQLException sqle2) {
|
||||
logger.error("Could not close the connection with the Impala-database.\n" + sqle2.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("The database \"" + databaseName + "\" and its tables were created or validated.");
|
||||
}
|
||||
|
||||
|
||||
public Connection getConnection()
|
||||
{
|
||||
try {
|
||||
return hikariDataSource.getConnection();
|
||||
//return DriverManager.getConnection(impalaConnectionUrl, null, null); // This is for non pooled connections.
|
||||
} catch (SQLException sqle) {
|
||||
logger.error("Problem when connecting with the Impala-database!\n" + sqle.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean testDatabaseAccess()
|
||||
{
|
||||
logger.info("Going to test Impala access..");
|
||||
Connection con = getConnection();
|
||||
if ( con == null )
|
||||
return false;
|
||||
|
||||
ResultSet res = null;
|
||||
try {
|
||||
String tableName = "publication";
|
||||
|
||||
// show tables
|
||||
String sql = "show tables '" + tableName + "'";
|
||||
logger.debug("Running: " + sql);
|
||||
res = con.prepareStatement(sql).executeQuery();
|
||||
if ( res.next() ) {
|
||||
logger.debug(res.getString(1));
|
||||
}
|
||||
|
||||
// describe table
|
||||
sql = "describe " + tableName;
|
||||
logger.debug("Running: " + sql);
|
||||
res = con.prepareStatement(sql).executeQuery();
|
||||
while ( res.next() ) {
|
||||
logger.debug(res.getString(1) + "\t" + res.getString(2));
|
||||
}
|
||||
|
||||
// select * query
|
||||
sql = "select * from " + tableName + " limit 3;";
|
||||
logger.debug("Running: " + sql);
|
||||
res = con.prepareStatement(sql).executeQuery();
|
||||
while ( res.next() ) {
|
||||
logger.debug(res.getString(1));
|
||||
}
|
||||
|
||||
// Get Assignments, only for testing here.
|
||||
//UrlController urlController = new UrlController();
|
||||
//ResponseEntity<?> responseEntity = urlController.getUrls("worker_1", ControllerConstants.ASSIGNMENTS_LIMIT);
|
||||
//logger.debug(responseEntity.toString());
|
||||
|
||||
} catch (SQLException sqle) {
|
||||
sqle.printStackTrace();
|
||||
return false;
|
||||
} finally {
|
||||
try {
|
||||
if ( res != null )
|
||||
res.close();
|
||||
con.close();
|
||||
} catch (SQLException sqle) {
|
||||
logger.error("Could not close the connection with the Impala-database.\n" + sqle);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
package eu.openaire.urls_controller.controllers;
|
||||
|
||||
import eu.openaire.urls_controller.configuration.ImpalaConnector;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/impala")
|
||||
public class ImpalaController {
|
||||
|
||||
// This controller will test the connectivity with the database and return statistics!
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ImpalaController.class);
|
||||
|
||||
|
||||
@GetMapping("get10PublicationIdsTest")
|
||||
public ResponseEntity<?> get10PublicationIdsTest() {
|
||||
|
||||
Connection con = ImpalaConnector.getInstance().getConnection();
|
||||
if ( con == null )
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Problem when connecting with the Impala-database!");
|
||||
|
||||
String query = "SELECT id FROM publication LIMIT 10;";
|
||||
|
||||
try ( ResultSet res = con.prepareStatement(query).executeQuery()) {
|
||||
if ( !res.first() ) {
|
||||
String errorMsg = "No results retrieved from the \"getAssignmentsQuery\"!";
|
||||
logger.error(errorMsg);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
}
|
||||
|
||||
List<String> publications = new ArrayList<>();
|
||||
do {
|
||||
publications.add(res.getString(0));
|
||||
} while ( res.next() );
|
||||
|
||||
return new ResponseEntity<String>(publications.toString(), HttpStatus.OK);
|
||||
|
||||
} catch (Exception e) {
|
||||
String errorMsg = "Problem when executing \"getAssignmentsQuery\": " + query;
|
||||
logger.error(errorMsg);
|
||||
e.printStackTrace();
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
|
||||
} finally {
|
||||
try {
|
||||
con.close();
|
||||
} catch (SQLException sqle) {
|
||||
logger.error("Could not close the connection with the Impala-database.\n" + sqle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue