From bd0ead816d996f5904a5c88aab85fe9e90221480 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Tue, 16 May 2023 19:08:59 +0300 Subject: [PATCH] Make the value of time-out for "restTemplateForReport", to scale along the "maxAssignmentsLimitPerBatch". --- .../urls_worker/components/AssignmentsHandler.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java index 4420c99..095f0bd 100644 --- a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java @@ -50,11 +50,7 @@ public class AssignmentsHandler { public static final RestTemplate restTemplateForRequest = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(Duration.ofMinutes(30)).build(); - public static final RestTemplate restTemplateForReport = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(Duration.ofHours(10)).build(); - // 10 hours. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work. - // When giving the assignments, the Controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker. - // When receiving the Worker-Report, the Controller has to check for existing fulltext files (by previous runs), request and get thousands of file from the Worker (in batches), upload them to S3, prepare and import the payload and the attempt records in the database and return to the Worker. - + public static RestTemplate restTemplateForReport = null; public static boolean hadConnectionErrorOnRequest = false; @@ -82,6 +78,13 @@ public class AssignmentsHandler { int expectedAssignmentsPerDatasource = (this.maxAssignmentsLimitPerBatch / expectedDatasourcesPerRequest); assignmentsForPlugins = HashMultimap.create(expectedDatasourcesPerRequest, expectedAssignmentsPerDatasource); requestUrl = this.controllerBaseUrl + (askForTest ? "test/" : "") + "urls?workerId=" + this.workerId + "&workerAssignmentsLimit=" + this.maxAssignmentsLimitPerBatch; + + long durationInHours = (long) Math.ceil((double) this.maxAssignmentsLimitPerBatch / 1000); // For example, for 10_000 assignments we wait at most 10 hours. + logger.debug("Setting the max-connection duration for the \"post-worker-report\" to " + durationInHours + " hours."); + restTemplateForReport = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(Duration.ofHours(durationInHours)).build(); + // X hours. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work. + // When giving the assignments, the Controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker. + // When receiving the Worker-Report, the Controller has to check for existing fulltext files (by previous runs), request and get thousands of file from the Worker (in batches), upload them to S3, prepare and import the payload and the attempt records in the database and return to the Worker. }