- Add an HTTP-error-handler in "AssignmentHandler.requestAssignments()".

- Increase the "requestConnectTimeoutDuration" and the "requestReadTimeoutDuration".
- Increase project's version to "1.0.0-SNAPSHOT".
- Update dependencies.
- Code cleanup.
This commit is contained in:
Lampros Smyrnaios 2021-10-11 13:27:40 +03:00
parent 42f8cb769d
commit 380137fbff
3 changed files with 48 additions and 13 deletions

View File

@ -1,18 +1,18 @@
buildscript { buildscript {
ext { ext {
springBootVersion = "2.5.4" springBootVersion = "2.5.5"
springSecurityVersion = "5.5.2" springSecurityVersion = "5.5.2"
} }
} }
plugins { plugins {
id 'org.springframework.boot' version '2.5.4' id 'org.springframework.boot' version '2.5.5'
id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java' id 'java'
} }
group = 'eu.openaire.urls_worker' group = 'eu.openaire.urls_worker'
version = '0.0.1-SNAPSHOT' version = '1.0.0-SNAPSHOT'
sourceCompatibility = '1.8' sourceCompatibility = '1.8'
@ -35,8 +35,10 @@ dependencies {
implementation("org.springframework.security:spring-security-config:${springSecurityVersion}") implementation("org.springframework.security:spring-security-config:${springSecurityVersion}")
//implementation("io.jsonwebtoken:jjwt:0.9.1") // Use this in case we use auth-tokens later on. //implementation("io.jsonwebtoken:jjwt:0.9.1") // Use this in case we use auth-tokens later on.
implementation "org.projectlombok:lombok:1.18.20" implementation "org.projectlombok:lombok:1.18.22"
implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
// Enable the validation annotations.
//implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final'
implementation ("eu.openaire:publications_retriever:1.0-SNAPSHOT") { implementation ("eu.openaire:publications_retriever:1.0-SNAPSHOT") {
exclude group: 'ch.qos.logback', module: 'logback-core' exclude group: 'ch.qos.logback', module: 'logback-core'

View File

@ -36,8 +36,6 @@ if [[ justInstall -eq 0 ]]; then
cd libs || exit 3 cd libs || exit 3
git clone https://github.com/LSmyrnaios/PublicationsRetriever.git # We assume there is no previously source-code here, if so, it will be overwritten. git clone https://github.com/LSmyrnaios/PublicationsRetriever.git # We assume there is no previously source-code here, if so, it will be overwritten.
# Do not need to perform a string-replace in "build.gradle", since it automatically gets all ".jar" files.
# Keep a backup of the existing JAR file. # Keep a backup of the existing JAR file.
mv ./publications_retriever-1.0-SNAPSHOT.jar ./publications_retriever-1.0-SNAPSHOT_BACKUP.jar mv ./publications_retriever-1.0-SNAPSHOT.jar ./publications_retriever-1.0-SNAPSHOT_BACKUP.jar
@ -51,10 +49,10 @@ if [[ justInstall -eq 0 ]]; then
# Clean and (re)build and run the project. # Clean and (re)build and run the project.
cd ../ cd ../
echo -e "\nAsking for sudo, in order to verify the installation of 'gradle'..\n"
if [[ ! -d /opt/gradle/gradle-${gradleVersion} ]]; then if [[ ! -d /opt/gradle/gradle-${gradleVersion} ]]; then
wget https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip wget https://services.gradle.org/distributions/gradle-${gradleVersion}-bin.zip
echo -e "\nAsking for sudo, in order to install 'gradle'..\n"
sudo mkdir /opt/gradle sudo mkdir /opt/gradle
sudo apt install -y unzip && sudo unzip -d /opt/gradle gradle-${gradleVersion}-bin.zip sudo apt install -y unzip && sudo unzip -d /opt/gradle gradle-${gradleVersion}-bin.zip
#ls /opt/gradle/gradle-${gradleVersion} # For debugging installation #ls /opt/gradle/gradle-${gradleVersion} # For debugging installation

View File

@ -15,9 +15,12 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.web.client.ResponseErrorHandler;
import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate; import org.springframework.web.client.RestTemplate;
import java.io.IOException;
import java.time.Duration; import java.time.Duration;
import java.util.*; import java.util.*;
@ -32,10 +35,11 @@ public class AssignmentHandler {
private static final boolean askForTest = false; // Enable this only for testing. private static final boolean askForTest = false; // Enable this only for testing.
private static final Duration requestConnectTimeoutDuration = Duration.ofSeconds(20); // 20 seconds. private static final Duration requestConnectTimeoutDuration = Duration.ofMinutes(1); // 1 minute.
private static final Duration requestReadTimeoutDuration = Duration.ofMinutes(30); // 30 minutes. private static final Duration requestReadTimeoutDuration = Duration.ofMinutes(60); // 60 minutes. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work.
// The controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker. // The controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker.
private static boolean encounteredHTTPRequestError = false;
public static AssignmentRequest requestAssignments() public static AssignmentRequest requestAssignments()
{ {
@ -43,14 +47,45 @@ public class AssignmentHandler {
logger.info("Going to request assignments from the controller-server: " + requestUrl); logger.info("Going to request assignments from the controller-server: " + requestUrl);
RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(requestReadTimeoutDuration).build(); RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(requestReadTimeoutDuration).build();
ResponseErrorHandler responseErrorHandler = new ResponseErrorHandler() {
@Override
public boolean hasError(ClientHttpResponse response) throws IOException {
if ( response.getRawStatusCode() != 200 ) {
encounteredHTTPRequestError = true;
return true;
} else {
encounteredHTTPRequestError = false; // Make sure the value is reset here, to avoid non-present errors from previous requests.
return false;
}
}
@Override
public void handleError(ClientHttpResponse response) throws IOException {
int responseCode = response.getRawStatusCode();
String statusText = response.getStatusText();
String additionalErrorMessage = ((!"".equals(statusText)) ? statusText : "The HTTP-response-code was: " + responseCode);
if ( (responseCode >= 500) && responseCode <= 599 )
logger.error("The Controller encountered a problem! " + additionalErrorMessage);
else if ( (responseCode >= 400) && (responseCode <= 499) )
logger.error("There was a bad request to the Controller! " + additionalErrorMessage);
else if ( responseCode != 200 )
logger.error("There was an HTTP-error when requesting the assignments from the Controller! " + additionalErrorMessage);
}
};
restTemplate.setErrorHandler(responseErrorHandler);
String json = null; String json = null;
try { try { // Here, the HTTP-request is executed.
json = restTemplate.getForObject(requestUrl, String.class); json = restTemplate.getForObject(requestUrl, String.class);
} catch (RestClientException e) { } catch (RestClientException e) {
logger.error("Could not retrieve the assignments!\n" + e.getMessage()); logger.error("Could not retrieve the assignments!\n" + e.getMessage());
return null; return null;
} }
if ( encounteredHTTPRequestError )
return null;
AssignmentRequest assignmentRequest = null; AssignmentRequest assignmentRequest = null;
try { try {
assignmentRequest = new ObjectMapper().readValue(json, AssignmentRequest.class); assignmentRequest = new ObjectMapper().readValue(json, AssignmentRequest.class);
@ -124,7 +159,7 @@ public class AssignmentHandler {
else else
postWorkerReport(assignmentRequestCounter); postWorkerReport(assignmentRequestCounter);
// Note: Cannot call this method here retrospectively, as if it runs 100s of times, the memory may break.. // Note: Cannot call this method here retrospectively, as if it runs 100s of times, the memory-stack may break..
// The scheduler will handle calling it every half an hour, in case the Worker is available for work.. // The scheduler will handle calling it every half an hour, in case the Worker is available for work..
} }
@ -136,7 +171,7 @@ public class AssignmentHandler {
try { try {
ResponseEntity<String> responseEntity = new RestTemplateBuilder().build().postForEntity(postUrl, new WorkerReport(UrlsWorkerApplication.workerId, assignmentRequestCounter, urlReports), String.class); ResponseEntity<String> responseEntity = new RestTemplateBuilder().build().postForEntity(postUrl, new WorkerReport(UrlsWorkerApplication.workerId, assignmentRequestCounter, urlReports), String.class);
int responseCode = responseEntity.getStatusCode().value(); int responseCode = responseEntity.getStatusCodeValue();
if ( responseCode != HttpStatus.OK.value() ) { if ( responseCode != HttpStatus.OK.value() ) {
logger.error("Connection problem with the submission of the WorkerReport of assignment_" + assignmentRequestCounter + " to the Controller. Error-code was: " + responseCode); logger.error("Connection problem with the submission of the WorkerReport of assignment_" + assignmentRequestCounter + " to the Controller. Error-code was: " + responseCode);
return false; return false;