michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

- Improve some log-messages. 

- Increase app's version.
- Code polishing.
This commit is contained in:
Lampros Smyrnaios 2023-10-06 17:28:54 +03:00
parent 0c79fdea35
commit a354da763d
3 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -6,7 +6,7 @@ plugins {
java { java {
group = 'eu.openaire.urls_controller' group = 'eu.openaire.urls_controller'
version = '2.4.0-SNAPSHOT' version = '2.6.0-SNAPSHOT'
sourceCompatibility = JavaVersion.VERSION_1_8 sourceCompatibility = JavaVersion.VERSION_1_8
} }

8
src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java
View File

@ -350,7 +350,7 @@ public class UrlsServiceImpl implements UrlsService {
else if ( hasPayloadParquetFileProblem ) else if ( hasPayloadParquetFileProblem )
logger.error("The single payload-parquet-file failed to be created or uploaded! Will avoid to execute load-requests into the database-table \"payload_aggregated\", for batch-assignments_" + curReportAssignmentsCounter); logger.error("The single payload-parquet-file failed to be created or uploaded! Will avoid to execute load-requests into the database-table \"payload_aggregated\", for batch-assignments_" + curReportAssignmentsCounter);
else else
logger.debug("Going to execute \"load\"-requests on the database, for the uploaded parquet-files."); logger.debug("Going to execute \"load\"-requests on the database, for the uploaded parquet-files, for batch-assignments_" + curReportAssignmentsCounter);
} }
// Load all the parquet files of each type into its table. // Load all the parquet files of each type into its table.
@ -471,9 +471,9 @@ public class UrlsServiceImpl implements UrlsService {
private boolean mergeWorkerRelatedTables(String curWorkerId, long curReportAssignmentsCounter, boolean hasAttemptParquetFileProblem, boolean hasPayloadParquetFileProblem) private boolean mergeWorkerRelatedTables(String curWorkerId, long curReportAssignmentsCounter, boolean hasAttemptParquetFileProblem, boolean hasPayloadParquetFileProblem)
{ {
logger.debug("Going to merge the parquet files for the tables which were altered."); logger.debug("Going to merge the parquet files for the tables which were altered, for batch-assignments_" + curReportAssignmentsCounter);
// When the uploaded parquet files are "loaded" into the tables, they are actually moved into the directory which contains the data of the table. // When the uploaded parquet files are "loaded" into the tables, they are actually moved into the directory which contains the data of the table.
// This means that over time a table may have thousands of parquet files and the search through them will be very slow. Thus, we merge them every now and then. // This means that over time a table may have thousands of parquet files and the search through them will be very slow. Thus, we merge them regularly.
String mergeErrorMsg; String mergeErrorMsg;
@ -498,7 +498,7 @@ public class UrlsServiceImpl implements UrlsService {
} }
DatabaseConnector.databaseLock.unlock(); DatabaseConnector.databaseLock.unlock();
logger.debug("Finished merging the database tables."); logger.debug("Finished merging the database tables, for batch-assignments_" + curReportAssignmentsCounter);
return true; return true;
} }

6
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -214,9 +214,9 @@ public class FileUtils {
workerIp = workerInfo.getWorkerIP(); // This won't be null. workerIp = workerInfo.getWorkerIP(); // This won't be null.
// Get the file-locations. // Get the file-locations.
final AtomicInteger numPayloadsToBeHandled = new AtomicInteger(); final AtomicInteger numPayloadsToBeHandled = new AtomicInteger(0);
final AtomicInteger numFullTextsFound = new AtomicInteger(); final AtomicInteger numFullTextsFound = new AtomicInteger(0);
final AtomicInteger numFilesFoundFromPreviousAssignmentsBatches = new AtomicInteger(); final AtomicInteger numFilesFoundFromPreviousAssignmentsBatches = new AtomicInteger(0);
SetMultimap<String, Payload> allFileNamesWithPayloads = Multimaps.synchronizedSetMultimap(HashMultimap.create((sizeOfUrlReports / 5), 3)); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it. SetMultimap<String, Payload> allFileNamesWithPayloads = Multimaps.synchronizedSetMultimap(HashMultimap.create((sizeOfUrlReports / 5), 3)); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it.