diff --git a/build.gradle b/build.gradle index 59c3b21..ea54276 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { java { group = 'eu.openaire.urls_controller' - version = '2.4.0-SNAPSHOT' + version = '2.6.0-SNAPSHOT' sourceCompatibility = JavaVersion.VERSION_1_8 } diff --git a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java index 5d2041b..34768ad 100644 --- a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java +++ b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java @@ -350,7 +350,7 @@ public class UrlsServiceImpl implements UrlsService { else if ( hasPayloadParquetFileProblem ) logger.error("The single payload-parquet-file failed to be created or uploaded! Will avoid to execute load-requests into the database-table \"payload_aggregated\", for batch-assignments_" + curReportAssignmentsCounter); else - logger.debug("Going to execute \"load\"-requests on the database, for the uploaded parquet-files."); + logger.debug("Going to execute \"load\"-requests on the database, for the uploaded parquet-files, for batch-assignments_" + curReportAssignmentsCounter); } // Load all the parquet files of each type into its table. @@ -471,9 +471,9 @@ public class UrlsServiceImpl implements UrlsService { private boolean mergeWorkerRelatedTables(String curWorkerId, long curReportAssignmentsCounter, boolean hasAttemptParquetFileProblem, boolean hasPayloadParquetFileProblem) { - logger.debug("Going to merge the parquet files for the tables which were altered."); + logger.debug("Going to merge the parquet files for the tables which were altered, for batch-assignments_" + curReportAssignmentsCounter); // When the uploaded parquet files are "loaded" into the tables, they are actually moved into the directory which contains the data of the table. - // This means that over time a table may have thousands of parquet files and the search through them will be very slow. Thus, we merge them every now and then. + // This means that over time a table may have thousands of parquet files and the search through them will be very slow. Thus, we merge them regularly. String mergeErrorMsg; @@ -498,7 +498,7 @@ public class UrlsServiceImpl implements UrlsService { } DatabaseConnector.databaseLock.unlock(); - logger.debug("Finished merging the database tables."); + logger.debug("Finished merging the database tables, for batch-assignments_" + curReportAssignmentsCounter); return true; } diff --git a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java index df6c984..174df78 100644 --- a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java @@ -214,9 +214,9 @@ public class FileUtils { workerIp = workerInfo.getWorkerIP(); // This won't be null. // Get the file-locations. - final AtomicInteger numPayloadsToBeHandled = new AtomicInteger(); - final AtomicInteger numFullTextsFound = new AtomicInteger(); - final AtomicInteger numFilesFoundFromPreviousAssignmentsBatches = new AtomicInteger(); + final AtomicInteger numPayloadsToBeHandled = new AtomicInteger(0); + final AtomicInteger numFullTextsFound = new AtomicInteger(0); + final AtomicInteger numFilesFoundFromPreviousAssignmentsBatches = new AtomicInteger(0); SetMultimap allFileNamesWithPayloads = Multimaps.synchronizedSetMultimap(HashMultimap.create((sizeOfUrlReports / 5), 3)); // Holds multiple values for any key, if a fileName(key) has many IDs(values) associated with it.