- Fix a "databaseLock" bug, which could cause both the payload and attempt inserts and the "mergeParquetFiles" to fail, as the inserts could be executed concurrently with tables-compaction.
- Fix the "null" representation of an "unknown" payload-size in the database.
- Remove the obsolete thread-locking for the "CreateDatabase" operation. This code is guaranteed to run BEFORE any other operation in the database.
- Implement the "handlePreparedStatementException" and "closeConnection" methods.
- Improve error-logs.
- Update dependencies.
- Code cleanup.
logger.error("Problem when executing the \"create database and create tables queries!\n"+sqle.getMessage()+"\nSQL state: "+sqle.getSQLState()+"\nError code: "+sqle.getErrorCode());
sqle.printStackTrace();
StringerrorMsg="Problem when executing the \"create database and create tables queries!\n"+sqle.getMessage()+"\nSQL state: "+sqle.getSQLState()+"\nError code: "+sqle.getErrorCode();
logger.error(errorMsg,sqle);
System.err.println(errorMsg);
System.exit(44);
}finally{
databaseLock.unlock();
try{
statement.close();
con.close();
@ -222,7 +214,7 @@ public final class ImpalaConnector {
//logger.debug(responseEntity.toString());
}catch(SQLExceptionsqle){
sqle.printStackTrace();
logger.error(sqle.getMessage(),sqle);
returnfalse;
}finally{
try{
@ -236,4 +228,25 @@ public final class ImpalaConnector {
logger.info("Sending batch-assignments_" +assignmentsBatchCounter.incrementAndGet()+" with "+assignmentsSize+" assignments to worker with ID: "+workerId+".");
logger.info("Received the WorkerReport for batch_ "+workerReport.getAssignmentRequestCounter()+", from the worker with id: "+workerReport.getWorkerId()+". It contains "+urlReports.size()+" urlReports. Going to insert them into the database.");
// TODO - The Controller will have to request the files from the Worker, in order to upload them to the S3, in the future.
// We will have to UPDATE the "location" of each of those files in the UrlReports and then insert them all into the database.
// The column "size" in the table is of type "String" so we cast the Long to String. The Parquet-format in the database does not work well with integers.
con.close();// It may have already closed and that's fine.
returntrue;
}
catch(SQLExceptionsqle){
}catch(SQLExceptionsqle){
logger.error("Could not close the connection with the Impala-database.\n"+sqle.getMessage());
returnfalse;
}
@ -576,7 +568,7 @@ public class UrlController {
logger.info("Sending batch_"+assignmentsBatchCounter.incrementAndGet()+" with "+assignments.size()+" assignments ("+FileUtils.duplicateIdUrlEntries.get()+" more assignments were discarded as duplicates), to worker with ID: "+workerId);