lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

Use restTemplates with different read timeouts depending on the operation. For the assignments-request we need a shorter read timeout, than the one we need for the worker-report. This guarantees that the connection does not hungs for so long, when the Controller crashes before sending the assignments.

This commit is contained in:
Lampros Smyrnaios 2023-04-29 17:24:16 +03:00
parent 53ab51922a
commit d5a997ad3d
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java
View File

@ -47,12 +47,15 @@ public class AssignmentsHandler {
private static String requestUrl;
private static final Duration requestConnectTimeoutDuration = Duration.ofMinutes(1); // 1 minute.
private static final Duration requestReadTimeoutDuration = Duration.ofHours(10); // 10 hours. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work.
public static final RestTemplate restTemplateForRequest = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(Duration.ofMinutes(30)).build();
public static final RestTemplate restTemplateForReport = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(Duration.ofHours(10)).build();
// 10 hours. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work.
// When giving the assignments, the Controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker.
// TODO - Create a separate "restTemplate" object for that with a shorter read-duration, as that process happens within 3 minutes.
// When receiving the Worker-Report, the Controller has to check for existing fulltext files (by previous runs), request and get thousands of file from the Worker (in batches), upload them to S3, prepare and import the payload and the attempt records in the database and return to the Worker.
public static final RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(requestReadTimeoutDuration).build();
public static boolean hadConnectionErrorOnRequest = false;
public static long numHandledAssignmentsBatches = 0; // No need to be synchronized.
@ -88,7 +91,7 @@ public class AssignmentsHandler {
AssignmentsRequest assignmentRequest = null;
try { // Here, the HTTP-request is executed.
assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class);
assignmentRequest = restTemplateForRequest.getForObject(requestUrl, AssignmentsRequest.class);
} catch (RestClientException rce) {
logger.error("Could not retrieve the assignments!\n" + rce.getMessage()); // It shows the response body (from Spring v.2.5.6 onwards).
hadConnectionErrorOnRequest = true;
@ -219,7 +222,7 @@ public class AssignmentsHandler {
String postUrl = this.controllerBaseUrl + "urls/addWorkerReport";
logger.info("Going to post the WorkerReport of assignments_" + assignmentRequestCounter + " to the controller-server: " + postUrl);
try {
ResponseEntity<String> responseEntity = restTemplate.postForEntity(postUrl, new WorkerReport(this.workerId, assignmentRequestCounter, urlReports), String.class);
ResponseEntity<String> responseEntity = restTemplateForReport.postForEntity(postUrl, new WorkerReport(this.workerId, assignmentRequestCounter, urlReports), String.class);
// The worker sends the "WorkerReport" and before this "POST"-request returns here, the Controller, after analyzing the report, opens new request to the Worker in order to receive the full-texts.
// After the report and the full-texts are received and uploaded to the database/S3, the Controller returns a response to the Worker.