diff --git a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java index 9245e94..553b143 100644 --- a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java @@ -47,8 +47,10 @@ public class AssignmentsHandler { private static String requestUrl; private static final Duration requestConnectTimeoutDuration = Duration.ofMinutes(1); // 1 minute. - private static final Duration requestReadTimeoutDuration = Duration.ofHours(5); // 5 hours. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work. - // The controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker. + private static final Duration requestReadTimeoutDuration = Duration.ofHours(10); // 10 hours. Time to wait for the data to get transferred over the network. Many workers may try to get assignments from the Worker, so each worker might have to wait some 10s of minutes for work. + // When giving the assignments, the Controller has to retrieve the data from the database, then prepare them in memory, insert them in the "assignment"-table and, finally, return them to the worker. + // TODO - Create a separate "restTemplate" object for that with a shorter read-duration, as that process happens within 3 minutes. + // When receiving the Worker-Report, the Controller has to check for existing fulltext files (by previous runs), request and get thousands of file from the Worker (in batches), upload them to S3, prepare and import the payload and the attempt records in the database and return to the Worker. public static final RestTemplate restTemplate = new RestTemplateBuilder().setConnectTimeout(requestConnectTimeoutDuration).setReadTimeout(requestReadTimeoutDuration).build(); public static boolean hadConnectionErrorOnRequest = false;