From 380137fbff6104597346728fcf7521e293245f95 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Mon, 11 Oct 2021 13:27:40 +0300 Subject: [PATCH] - Add an HTTP-error-handler in "AssignmentHandler.requestAssignments()". - Increase the "requestConnectTimeoutDuration" and the "requestReadTimeoutDuration". - Increase project's version to "1.0.0-SNAPSHOT". - Update dependencies. - Code cleanup. --- build.gradle | 12 ++--- installAndRun.sh | 4 +- .../urls_worker/util/AssignmentHandler.java | 45 ++++++++++++++++--- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/build.gradle b/build.gradle index c18ae8d..89e1fb4 100644 --- a/build.gradle +++ b/build.gradle @@ -1,18 +1,18 @@ buildscript { ext { - springBootVersion = "2.5.4" + springBootVersion = "2.5.5" springSecurityVersion = "5.5.2" } } plugins { - id 'org.springframework.boot' version '2.5.4' + id 'org.springframework.boot' version '2.5.5' id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'java' } group = 'eu.openaire.urls_worker' -version = '0.0.1-SNAPSHOT' +version = '1.0.0-SNAPSHOT' sourceCompatibility = '1.8' @@ -35,8 +35,10 @@ dependencies { implementation("org.springframework.security:spring-security-config:${springSecurityVersion}") //implementation("io.jsonwebtoken:jjwt:0.9.1") // Use this in case we use auth-tokens later on. - implementation "org.projectlombok:lombok:1.18.20" - implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' + implementation "org.projectlombok:lombok:1.18.22" + + // Enable the validation annotations. + //implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' implementation ("eu.openaire:publications_retriever:1.0-SNAPSHOT") { exclude group: 'ch.qos.logback', module: 'logback-core' diff --git a/installAndRun.sh b/installAndRun.sh index 0117074..02cbd52 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -36,8 +36,6 @@ if [[ justInstall -eq 0 ]]; then cd libs || exit 3 git clone https://github.com/LSmyrnaios/PublicationsRetriever.git # We assume there is no previously source-code here, if so, it will be overwritten. - # Do not need to perform a string-replace in "build.gradle", since it automatically gets all ".jar" files. - # Keep a backup of the existing JAR file. mv ./publications_retriever-1.0-SNAPSHOT.jar ./publications_retriever-1.0-SNAPSHOT_BACKUP.jar @@ -51,10 +49,10 @@ if [[ justInstall -eq 0 ]]; then # Clean and (re)build and run the project. cd ../ - echo -e "\nAsking for sudo, in order to verify the installation of 'gradle'..\n" if [[ ! -d /opt/gradle/gradle-${gradleVersion} ]]; then wget https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip + echo -e "\nAsking for sudo, in order to install 'gradle'..\n" sudo mkdir /opt/gradle sudo apt install -y unzip && sudo unzip -d /opt/gradle gradle-${gradleVersion}-bin.zip #ls /opt/gradle/gradle-${gradleVersion} # For debugging installation diff --git a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java b/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java index fe4bbb0..0ff61ae 100644 --- a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java +++ b/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java @@ -15,9 +15,12 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.web.client.ResponseErrorHandler; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; +import java.io.IOException; import java.time.Duration; import java.util.*; @@ -32,10 +35,11 @@ public class AssignmentHandler { private static final boolean askForTest = false; // Enable this only for testing. - private static final Duration requestConnectTimeoutDuration = Duration.ofSeconds(20); // 20 seconds. - private static final Duration requestReadTimeoutDuration = Duration.ofMinutes(30); // 30 minutes. + private static final Duration requestConnectTimeoutDuration = Duration.ofMinutes(1); // 1 minute. + private static final Duration requestReadTimeoutDuration = Duration.ofMinutes(60); // 60 minutes. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work. // The controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker. + private static boolean encounteredHTTPRequestError = false; public static AssignmentRequest requestAssignments() { @@ -43,14 +47,45 @@ public class AssignmentHandler { logger.info("Going to request assignments from the controller-server: " + requestUrl); RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(requestReadTimeoutDuration).build(); + + ResponseErrorHandler responseErrorHandler = new ResponseErrorHandler() { + @Override + public boolean hasError(ClientHttpResponse response) throws IOException { + if ( response.getRawStatusCode() != 200 ) { + encounteredHTTPRequestError = true; + return true; + } else { + encounteredHTTPRequestError = false; // Make sure the value is reset here, to avoid non-present errors from previous requests. + return false; + } + } + + @Override + public void handleError(ClientHttpResponse response) throws IOException { + int responseCode = response.getRawStatusCode(); + String statusText = response.getStatusText(); + String additionalErrorMessage = ((!"".equals(statusText)) ? statusText : "The HTTP-response-code was: " + responseCode); + if ( (responseCode >= 500) && responseCode <= 599 ) + logger.error("The Controller encountered a problem! " + additionalErrorMessage); + else if ( (responseCode >= 400) && (responseCode <= 499) ) + logger.error("There was a bad request to the Controller! " + additionalErrorMessage); + else if ( responseCode != 200 ) + logger.error("There was an HTTP-error when requesting the assignments from the Controller! " + additionalErrorMessage); + } + }; + restTemplate.setErrorHandler(responseErrorHandler); + String json = null; - try { + try { // Here, the HTTP-request is executed. json = restTemplate.getForObject(requestUrl, String.class); } catch (RestClientException e) { logger.error("Could not retrieve the assignments!\n" + e.getMessage()); return null; } + if ( encounteredHTTPRequestError ) + return null; + AssignmentRequest assignmentRequest = null; try { assignmentRequest = new ObjectMapper().readValue(json, AssignmentRequest.class); @@ -124,7 +159,7 @@ public class AssignmentHandler { else postWorkerReport(assignmentRequestCounter); - // Note: Cannot call this method here retrospectively, as if it runs 100s of times, the memory may break.. + // Note: Cannot call this method here retrospectively, as if it runs 100s of times, the memory-stack may break.. // The scheduler will handle calling it every half an hour, in case the Worker is available for work.. } @@ -136,7 +171,7 @@ public class AssignmentHandler { try { ResponseEntity responseEntity = new RestTemplateBuilder().build().postForEntity(postUrl, new WorkerReport(UrlsWorkerApplication.workerId, assignmentRequestCounter, urlReports), String.class); - int responseCode = responseEntity.getStatusCode().value(); + int responseCode = responseEntity.getStatusCodeValue(); if ( responseCode != HttpStatus.OK.value() ) { logger.error("Connection problem with the submission of the WorkerReport of assignment_" + assignmentRequestCounter + " to the Controller. Error-code was: " + responseCode); return false;