lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 16 Wiki Activity

Use a single "restTemplate" object, with the same timeouts (a bit increased from the old requestRestTemplate, to account for a possible overloaded Controller), since we no longer need to wait for hours until the workerReport is processed by the Controller.

This commit is contained in:
Lampros Smyrnaios 2023-05-29 14:15:55 +03:00
parent 2b69733912
commit 0908dcab8a
1 changed files with 3 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java
View File

@ -54,11 +54,7 @@ public class AssignmentsHandler {
private static final boolean askForTest = false; // Enable this only for testing. private static final boolean askForTest = false; // Enable this only for testing.
private static String requestUrl; private static String requestUrl;
private static final Duration requestConnectTimeoutDuration = Duration.ofMinutes(1); // 1 minute. public static final RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(Duration.ofMinutes(2)).setReadTimeout(Duration.ofHours(1)).build();
public static final RestTemplate restTemplateForRequest = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(Duration.ofMinutes(30)).build();
public static RestTemplate restTemplateForReport = null;
public static boolean hadConnectionErrorOnRequest = false; public static boolean hadConnectionErrorOnRequest = false;
@ -92,10 +88,6 @@ public class AssignmentsHandler {
long durationInHours = (long) Math.ceil((double) this.maxAssignmentsLimitPerBatch / 1000); // For example, for 10_000 assignments we wait at most 10 hours. long durationInHours = (long) Math.ceil((double) this.maxAssignmentsLimitPerBatch / 1000); // For example, for 10_000 assignments we wait at most 10 hours.
logger.debug("Setting the max-connection duration for the \"post-worker-report\" to " + durationInHours + " hours."); logger.debug("Setting the max-connection duration for the \"post-worker-report\" to " + durationInHours + " hours.");
restTemplateForReport = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(Duration.ofHours(durationInHours)).build();
// X hours. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work.
// When giving the assignments, the Controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker.
// When receiving the Worker-Report, the Controller has to check for existing fulltext files (by previous runs), request and get thousands of file from the Worker (in batches), upload them to S3, prepare and import the payload and the attempt records in the database and return to the Worker.
if ( !workerReportsDirPath.endsWith("/") ) if ( !workerReportsDirPath.endsWith("/") )
workerReportsDirPath += "/"; workerReportsDirPath += "/";
@ -116,7 +108,7 @@ public class AssignmentsHandler {
logger.info("Going to request " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl); logger.info("Going to request " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl);
AssignmentsRequest assignmentRequest = null; AssignmentsRequest assignmentRequest = null;
try { // Here, the HTTP-request is executed. try { // Here, the HTTP-request is executed.
assignmentRequest = restTemplateForRequest.getForObject(requestUrl, AssignmentsRequest.class); assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class);
} catch (RestClientException rce) { } catch (RestClientException rce) {
logger.error("Could not retrieve the assignments!\n" + rce.getMessage()); // It shows the response body (from Spring v.2.5.6 onwards). logger.error("Could not retrieve the assignments!\n" + rce.getMessage()); // It shows the response body (from Spring v.2.5.6 onwards).
hadConnectionErrorOnRequest = true; hadConnectionErrorOnRequest = true;
@ -257,7 +249,7 @@ public class AssignmentsHandler {
// The worker sends this "WorkerReport" to the Controller, which after some checks, it adds a job to a background thread and responds to the Worker with HTTP-200-OK. // The worker sends this "WorkerReport" to the Controller, which after some checks, it adds a job to a background thread and responds to the Worker with HTTP-200-OK.
try { try {
ResponseEntity<String> responseEntity = restTemplateForReport.postForEntity(postUrl, workerReport, String.class); ResponseEntity<String> responseEntity = restTemplate.postForEntity(postUrl, workerReport, String.class);
int responseCode = responseEntity.getStatusCodeValue(); int responseCode = responseEntity.getStatusCodeValue();
if ( responseCode == HttpStatus.OK.value() ) { if ( responseCode == HttpStatus.OK.value() ) {
logger.info("The submission of the WorkerReport of assignments_" + assignmentRequestCounter + " to the Controller, was successful."); logger.info("The submission of the WorkerReport of assignments_" + assignmentRequestCounter + " to the Controller, was successful.");