From 2aedae23673ae9cb6a7a65a37bee5490f1bea7c7 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Wed, 31 May 2023 15:25:36 +0300 Subject: [PATCH] - In case a serious error happened while processing the assignments, instead of shutting down immediately, now the Worker shuts down the executor service, registers that it will shut down soon and waits for the Controller to retrieve the already downloaded full-text files. - In case the full-texts' subdirectory could not be created, then terminate the "handleAssignment" method immediately. No posting of a faulty workerReport to the Controller should happen. - Code polishing. --- .../urls_worker/components/AssignmentsHandler.java | 5 +++-- .../openaire/urls_worker/components/ScheduledTasks.java | 4 ++-- .../components/plugins/PublicationsRetrieverPlugin.java | 9 +++++---- .../urls_worker/controllers/GeneralController.java | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java index 8afd06e..4a74f44 100644 --- a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java @@ -179,7 +179,8 @@ public class AssignmentsHandler { publicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values()); } catch (Exception e) { logger.error("Exception when processing the assignments_" + assignmentRequestCounter, e); - } // In this case, we will either have an empty WorkerReport or a half-filled one. Either way, we want to report back to the Controller. + return; + } // In this case, no assignments were processed. PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " finished after: "); @@ -214,7 +215,7 @@ public class AssignmentsHandler { } if ( GeneralController.shouldShutdownWorker - || (AssignmentsHandler.numHandledAssignmentsBatches == this.maxAssignmentsBatchesToHandleBeforeShutdown) ) + || (numHandledAssignmentsBatches == this.maxAssignmentsBatchesToHandleBeforeShutdown) ) { logger.info("The worker will shutdown, after the full-texts are delivered to the Controller, as " + (GeneralController.shouldShutdownWorker ? "it received a \"shutdownWorker\" request!" diff --git a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java index bef39fa..213e241 100644 --- a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java @@ -55,7 +55,7 @@ public class ScheduledTasks { @Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes. public void handleNewAssignments() { - if ( AssignmentsHandler.shouldNotRequestMore ) { + if ( GeneralController.shouldShutdownWorker || AssignmentsHandler.shouldNotRequestMore ) { // Here we will be right after the Worker has posted its last report. It is guaranteed that the Controller will not have processed it and have not requested the full-text files. // We do not want to shut down the controller. return; @@ -172,7 +172,7 @@ public class ScheduledTasks { long currentTime = System.currentTimeMillis(); - // Loop through the array and print only the directories + // Loop through the array, check the "lastModified" time and if it is too old delete the full-texts subDir and the related workerReport. for ( File subDir : fulltextSubDirs ) { long lastModified = subDir.lastModified(); diff --git a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java index e7b2986..5ca3886 100644 --- a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java @@ -11,6 +11,7 @@ import eu.openaire.publications_retriever.util.url.UrlUtils; import eu.openaire.urls_worker.UrlsWorkerApplication; import eu.openaire.urls_worker.components.AssignmentsHandler; import eu.openaire.urls_worker.components.ConnWithController; +import eu.openaire.urls_worker.controllers.GeneralController; import eu.openaire.urls_worker.models.Assignment; import eu.openaire.urls_worker.models.Error; import eu.openaire.urls_worker.models.Payload; @@ -153,10 +154,10 @@ public class PublicationsRetrieverPlugin { int numFailedTasks = LoaderAndChecker.invokeAllTasksAndWait(callableTasks); if ( numFailedTasks == -1 ) { // The unknown exception is logged inside the above method. - System.err.println("Invoking and/or executing the callableTasks failed with the exception (which is written in the log files)!"); - connWithController.postShutdownReportToController(workerId); - UrlsWorkerApplication.gentleAppShutdown(); - return; // Not relly needed, but have it for code-readability. + GeneralController.shouldShutdownWorker = true; + AssignmentsHandler.shouldNotRequestMore = true; + PublicationsRetriever.executor.shutdownNow(); // Close the thread-pool immediately. It will not be used again while the Worker is still running. + throw new RuntimeException("Invoking and/or executing the callableTasks failed with the exception (which is written in the log files)!"); } if ( numFailedTasks > 0 ) diff --git a/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java b/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java index 2a043e9..708f978 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java @@ -104,7 +104,7 @@ public class GeneralController { String directoryPath = PublicationsRetrieverPlugin.assignmentsBasePath + "assignments_" + assignmentsCounter + "_fullTexts"; File dir = new File(directoryPath); if ( ! dir.isDirectory() ) { - logger.error("The \"addRReportResultToWorker\"-endpoint was called for an unknown \"assignmentsCounter\": " + assignmentsCounter); + logger.error("The \"addReportResultToWorker\"-endpoint was called for an unknown \"assignmentsCounter\": " + assignmentsCounter); return ResponseEntity.notFound().build(); }