michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

- Optimize the placement of "DatabaseConnector.databaseLock.unlock()" statements. 

- Rename a maven-repository.
This commit is contained in:
Lampros Smyrnaios 2024-03-20 15:08:01 +02:00
parent 785204419d
commit 724eae1514
3 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -13,7 +13,7 @@ java {
repositories { repositories {
mavenCentral() mavenCentral()
maven { maven {
name "omtd" name "madgik"
url "https://repo.madgik.di.uoa.gr/content/repositories/thirdparty/" url "https://repo.madgik.di.uoa.gr/content/repositories/thirdparty/"
} }
} }

9
src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java
View File

@ -209,14 +209,13 @@ public class BulkImportServiceImpl implements BulkImportService {
// Merge the parquet files inside the table "payload_bulk_import", to improve performance of future operations. // Merge the parquet files inside the table "payload_bulk_import", to improve performance of future operations.
DatabaseConnector.databaseLock.lock(); DatabaseConnector.databaseLock.lock();
String mergeErrorMsg = fileUtils.mergeParquetFiles("payload_bulk_import", "", null); // msg is already logged String mergeErrorMsg = fileUtils.mergeParquetFiles("payload_bulk_import", "", null); // msg is already logged
DatabaseConnector.databaseLock.unlock();
if ( mergeErrorMsg != null ) { // the message in already logged if ( mergeErrorMsg != null ) { // the message in already logged
DatabaseConnector.databaseLock.unlock();
bulkImportReport.addEvent(mergeErrorMsg); bulkImportReport.addEvent(mergeErrorMsg);
fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true); fileUtils.writeToFile(bulkImportReportLocation, bulkImportReport.getJsonReport(), true);
BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName); BulkImportController.bulkImportDirsUnderProcessing.remove(bulkImportDirName);
return false; return false;
} }
DatabaseConnector.databaseLock.unlock();
String successMsg = "Finished the bulk-import procedure for " + provenance + " and bulkImportDir: " + bulkImportDirName; String successMsg = "Finished the bulk-import procedure for " + provenance + " and bulkImportDir: " + bulkImportDirName;
logger.info(successMsg); logger.info(successMsg);
@ -329,8 +328,9 @@ public class BulkImportServiceImpl implements BulkImportService {
logger.trace("Going to load the data of parquet-file: \"" + parquetFileName + "\" to the database-table: \"payload_bulk_import\"." + additionalLoggingMsg); // DEBUG! logger.trace("Going to load the data of parquet-file: \"" + parquetFileName + "\" to the database-table: \"payload_bulk_import\"." + additionalLoggingMsg); // DEBUG!
DatabaseConnector.databaseLock.lock(); DatabaseConnector.databaseLock.lock();
if ( !parquetFileUtils.loadParquetDataIntoTable((currentBulkImportHdfsDir + parquetFileName), "payload_bulk_import") ) { boolean parquetDataLoaded = parquetFileUtils.loadParquetDataIntoTable((currentBulkImportHdfsDir + parquetFileName), "payload_bulk_import");
DatabaseConnector.databaseLock.unlock(); DatabaseConnector.databaseLock.unlock();
if ( !parquetDataLoaded ) {
errorMsg = "Could not load the payload-records to the database, for segment-" + segmentCounter + "!"; errorMsg = "Could not load the payload-records to the database, for segment-" + segmentCounter + "!";
logger.error(errorMsg + additionalLoggingMsg); logger.error(errorMsg + additionalLoggingMsg);
bulkImportReport.addEvent(errorMsg); bulkImportReport.addEvent(errorMsg);
@ -338,7 +338,6 @@ public class BulkImportServiceImpl implements BulkImportService {
// None of the files of this segment will be deleted, in any case. // None of the files of this segment will be deleted, in any case.
return numOfFilesInSegment; // All files of this segment have failed. return numOfFilesInSegment; // All files of this segment have failed.
} }
DatabaseConnector.databaseLock.unlock();
String segmentSuccessMsg = "Finished importing " + numOfPayloadRecords + " files, out of " + numOfFilesInSegment + ", for segment-" + segmentCounter + "."; String segmentSuccessMsg = "Finished importing " + numOfPayloadRecords + " files, out of " + numOfFilesInSegment + ", for segment-" + segmentCounter + ".";
logger.info(segmentSuccessMsg + additionalLoggingMsg); logger.info(segmentSuccessMsg + additionalLoggingMsg);

2
src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java
View File

@ -334,8 +334,8 @@ public class UrlsServiceImpl implements UrlsService {
SumParquetSuccess sumParquetSuccess = parquetFileUtils.checkParquetFilesSuccess(futures); SumParquetSuccess sumParquetSuccess = parquetFileUtils.checkParquetFilesSuccess(futures);
ResponseEntity<?> errorResponseEntity = sumParquetSuccess.getResponseEntity(); ResponseEntity<?> errorResponseEntity = sumParquetSuccess.getResponseEntity();
if ( errorResponseEntity != null ) { // The related log is already shown in this case. if ( errorResponseEntity != null ) { // The related log is already shown in this case.
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, "Error when creating or uploading the parquet files!");
DatabaseConnector.databaseLock.unlock(); DatabaseConnector.databaseLock.unlock();
postReportResultToWorker(curWorkerId, curReportAssignmentsCounter, "Error when creating or uploading the parquet files!");
return false; return false;
} }
hasAttemptParquetFileProblem = sumParquetSuccess.isAttemptParquetFileProblem(); hasAttemptParquetFileProblem = sumParquetSuccess.isAttemptParquetFileProblem();