lsmyrnaios
/
UrlsController
1
0
Fork 2
Code Issues Pull Requests Releases 19 Wiki Activity

- Fix not allowing the user to use the "shutdownAllWorkersGracefully" endpoint twice. 

- Code optimization.
- Update dependencies.
This commit is contained in:
Lampros Smyrnaios 2024-05-21 23:43:49 +03:00
parent 39c36f9e66
commit e2e7ca72d5
3 changed files with 14 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build.gradle
View File

@ -1,6 +1,6 @@
plugins {
id 'org.springframework.boot' version '2.7.18'
id 'io.spring.dependency-management' version '1.1.4'
id 'io.spring.dependency-management' version '1.1.5'
id 'java'
}
@ -43,7 +43,7 @@ dependencies {
//implementation group: 'jakarta.validation', name: 'jakarta.validation-api', version: '3.0.2'
// https://mvnrepository.com/artifact/com.google.guava/guava
implementation group: 'com.google.guava', name: 'guava', version: '33.1.0-jre'
implementation group: 'com.google.guava', name: 'guava', version: '33.2.0-jre'
// https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
@ -54,7 +54,7 @@ dependencies {
}
implementation 'com.github.luben:zstd-jni:1.5.6-3' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
implementation 'io.minio:minio:8.5.9'
implementation 'io.minio:minio:8.5.10'
// https://mvnrepository.com/artifact/com.cloudera.impala/jdbc
implementation("com.cloudera.impala:jdbc:2.5.31") {
@ -77,7 +77,7 @@ dependencies {
}
// https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro
implementation('org.apache.parquet:parquet-avro:1.13.1')
implementation('org.apache.parquet:parquet-avro:1.14.0')
// https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common
implementation("org.apache.hadoop:hadoop-common:$hadoopVersion") {
@ -119,7 +119,7 @@ dependencies {
implementation 'com.google.code.gson:gson:2.10.1'
// https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.12.5'
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.13.0'
testImplementation 'org.springframework.security:spring-security-test'
testImplementation "org.springframework.boot:spring-boot-starter-test"

6
src/main/java/eu/openaire/urls_controller/controllers/ShutdownController.java
View File

@ -153,7 +153,11 @@ public class ShutdownController {
logger.info(initMsg);
workerInfo.setHasShutdown(true); // This will update the map.
UrlsController.numOfActiveWorkers.decrementAndGet();
int numActiveWorkers = UrlsController.numOfActiveWorkers.decrementAndGet();
if ( (numActiveWorkers == 0) && shouldShutdownAllWorkers ) { // If all workers have shutdown and the "shouldShutdownAllWorkers" was set, then reset the indicator to allow for future shutdowns.
logger.info("All workers have shutdown.");
shouldShutdownAllWorkers = false; // Make sure we can request that all the workers will shut-down again, when the user starts a couple of workers afterwards and sometime in the future he wants to shut them down.
}
// Return "HTTP-OK" to this worker. If this was part of a shutdown-service request, then wait for the scheduler to check and shutdown the service.
return ResponseEntity.ok().build();

13
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -722,19 +722,14 @@ public class FileUtils {
logger.error("No value was able to be retrieved from one of the columns of row_" + rs.getRow(), sqle);
return; // Move to the next payload.
}
Set<Payload> foundPayloads = urlToPayloadsMultimap.get(original_url);
if ( foundPayloads == null ) {
Set<Payload> foundPayloads = urlToPayloadsMultimap.get(original_url); // It doesn't return null, on error, but an empty set.
if ( foundPayloads.size() == 0 ) {
logger.error("No payloads associated with \"original_url\" = \"" + original_url + "\"!");
return;
}
// Select a random "foundPayload" to use its data to fill the "prefilledPayload" (in a "Set" the first element is pseudo-random).
Optional<Payload> optPayload = foundPayloads.stream().findFirst();
if ( !optPayload.isPresent() ) {
logger.error("Could not retrieve any payload for the \"original_url\": " + original_url);
return; // Move to the next payload.
}
Payload prefilledPayload = optPayload.get().clone(); // We take a clone of the original payload and change just the id and the original_url.
// Since the "foundPayloads" is not-empty at this point, the, we do not have to check if the first element is present.
Payload prefilledPayload = foundPayloads.stream().findFirst().get().clone(); // We take a clone of the original payload and change just the id and the original_url.
prefilledPayload.setId(id);
prefilledPayload.setOriginal_url(original_url);
prefilledPayloads.add(prefilledPayload);