diff --git a/build.gradle b/build.gradle index 2bc11ad..c501025 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { id 'org.springframework.boot' version '2.7.18' - id 'io.spring.dependency-management' version '1.1.4' + id 'io.spring.dependency-management' version '1.1.5' id 'java' } @@ -43,7 +43,7 @@ dependencies { //implementation group: 'jakarta.validation', name: 'jakarta.validation-api', version: '3.0.2' // https://mvnrepository.com/artifact/com.google.guava/guava - implementation group: 'com.google.guava', name: 'guava', version: '33.1.0-jre' + implementation group: 'com.google.guava', name: 'guava', version: '33.2.0-jre' // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0' @@ -54,7 +54,7 @@ dependencies { } implementation 'com.github.luben:zstd-jni:1.5.6-3' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often. - implementation 'io.minio:minio:8.5.9' + implementation 'io.minio:minio:8.5.10' // https://mvnrepository.com/artifact/com.cloudera.impala/jdbc implementation("com.cloudera.impala:jdbc:2.5.31") { @@ -77,7 +77,7 @@ dependencies { } // https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro - implementation('org.apache.parquet:parquet-avro:1.13.1') + implementation('org.apache.parquet:parquet-avro:1.14.0') // https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common implementation("org.apache.hadoop:hadoop-common:$hadoopVersion") { @@ -119,7 +119,7 @@ dependencies { implementation 'com.google.code.gson:gson:2.10.1' // https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus - runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.12.5' + runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.13.0' testImplementation 'org.springframework.security:spring-security-test' testImplementation "org.springframework.boot:spring-boot-starter-test" diff --git a/src/main/java/eu/openaire/urls_controller/controllers/ShutdownController.java b/src/main/java/eu/openaire/urls_controller/controllers/ShutdownController.java index dedcc77..381e33d 100644 --- a/src/main/java/eu/openaire/urls_controller/controllers/ShutdownController.java +++ b/src/main/java/eu/openaire/urls_controller/controllers/ShutdownController.java @@ -153,7 +153,11 @@ public class ShutdownController { logger.info(initMsg); workerInfo.setHasShutdown(true); // This will update the map. - UrlsController.numOfActiveWorkers.decrementAndGet(); + int numActiveWorkers = UrlsController.numOfActiveWorkers.decrementAndGet(); + if ( (numActiveWorkers == 0) && shouldShutdownAllWorkers ) { // If all workers have shutdown and the "shouldShutdownAllWorkers" was set, then reset the indicator to allow for future shutdowns. + logger.info("All workers have shutdown."); + shouldShutdownAllWorkers = false; // Make sure we can request that all the workers will shut-down again, when the user starts a couple of workers afterwards and sometime in the future he wants to shut them down. + } // Return "HTTP-OK" to this worker. If this was part of a shutdown-service request, then wait for the scheduler to check and shutdown the service. return ResponseEntity.ok().build(); diff --git a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java index 68e0493..2b18cc4 100644 --- a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java @@ -722,19 +722,14 @@ public class FileUtils { logger.error("No value was able to be retrieved from one of the columns of row_" + rs.getRow(), sqle); return; // Move to the next payload. } - Set foundPayloads = urlToPayloadsMultimap.get(original_url); - if ( foundPayloads == null ) { + Set foundPayloads = urlToPayloadsMultimap.get(original_url); // It doesn't return null, on error, but an empty set. + if ( foundPayloads.size() == 0 ) { logger.error("No payloads associated with \"original_url\" = \"" + original_url + "\"!"); return; } - // Select a random "foundPayload" to use its data to fill the "prefilledPayload" (in a "Set" the first element is pseudo-random). - Optional optPayload = foundPayloads.stream().findFirst(); - if ( !optPayload.isPresent() ) { - logger.error("Could not retrieve any payload for the \"original_url\": " + original_url); - return; // Move to the next payload. - } - Payload prefilledPayload = optPayload.get().clone(); // We take a clone of the original payload and change just the id and the original_url. + // Since the "foundPayloads" is not-empty at this point, the, we do not have to check if the first element is present. + Payload prefilledPayload = foundPayloads.stream().findFirst().get().clone(); // We take a clone of the original payload and change just the id and the original_url. prefilledPayload.setId(id); prefilledPayload.setOriginal_url(original_url); prefilledPayloads.add(prefilledPayload);