- Add the "shutdownWorker"-endpoint to accept requests for shutting-down the Worker, gracefully, after it completes its current work (including sending the publications-files to the Controller). A user-defined "auth-code" is required.

- Add the "cancelShutdownWorker"-endpoint to cancel a previous "shutdownWorker"-request. A user-defined "auth-code" is required.

src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java

@@ -41,7 +41,7 @@ public class UrlsWorkerApplication {
     private static final String inputDataFilePath = FileUtils.workingDir + "inputData.txt";
     public static String workerId = null;
     public static int maxAssignmentsLimitPerBatch = 0;
-    public static int maxAssignmentsBatchesToHandleBeforeRestart = -1;  // Default value: -1 = argument-absent, 0 = infinite-batches
+    public static int maxAssignmentsBatchesToHandleBeforeShutdown = -1;  // Default value: -1 = argument-absent, 0 = infinite-batches
     public static String controllerBaseUrl = null;  // BaseUrl template: "http://IP:PORT/api/"
 
     private static ConfigurableApplicationContext context;
@@ -64,7 +64,7 @@ public class UrlsWorkerApplication {
 
     public static void gentleAppShutdown()
     {
-        int exitCode = SpringApplication.exit(context, () -> 0);    // The "PreDestroy" method will be called.
+        int exitCode = SpringApplication.exit(context, () -> 0);    // The "PreDestroy" method will be called. (the "context" will be closed automatically (I checked it))
         System.exit(exitCode);
     }
 
@@ -145,10 +145,10 @@ public class UrlsWorkerApplication {
                 }
                 String maxAssignmentsBatchesStr = data[2].trim();
                 try {
-                    maxAssignmentsBatchesToHandleBeforeRestart = Integer.parseInt(maxAssignmentsBatchesStr);
+                    maxAssignmentsBatchesToHandleBeforeShutdown = Integer.parseInt(maxAssignmentsBatchesStr);
                 } catch (NumberFormatException nfe) {
                     logger.warn("The given \"maxAssignmentsBatchesToHandleBeforeRestart\" (" + maxAssignmentsBatchesStr + ") was not a number! Will handle an infinite number of batches!");
-                    maxAssignmentsBatchesToHandleBeforeRestart = 0;
+                    maxAssignmentsBatchesToHandleBeforeShutdown = 0;
                 }
                 controllerBaseUrl = data[3].trim();
                 try {
@@ -163,14 +163,14 @@ public class UrlsWorkerApplication {
                     controllerBaseUrl += "/";   // Make sure the other urls will not break later.
             }
 
-            if ( (workerId == null) || (maxAssignmentsLimitPerBatch == 0) || (maxAssignmentsBatchesToHandleBeforeRestart == -1) || (controllerBaseUrl == null) ) {
+            if ( (workerId == null) || (maxAssignmentsLimitPerBatch == 0) || (maxAssignmentsBatchesToHandleBeforeShutdown == -1) || (controllerBaseUrl == null) ) {
                 String errorMsg = "No \"workerId\" or/and \"maxAssignmentsLimitPerBatch\" or/and \"maxAssignmentsBatchesToHandleBeforeRestart\" or/and \"controllerBaseUrl\" could be retrieved from the file: " + inputDataFilePath;
                 logger.error(errorMsg);
                 System.err.println(errorMsg);
                 System.exit(63);
             }
 
-            logger.info("workerId: " + workerId + ", maxAssignmentsLimitPerBatch: " + maxAssignmentsLimitPerBatch + ", maxAssignmentsBatchesToHandleBeforeRestart: " + maxAssignmentsBatchesToHandleBeforeRestart + ", controllerBaseUrl: " + controllerBaseUrl);    // It's safe and helpful to show them in the logs.
+            logger.info("workerId: " + workerId + ", maxAssignmentsLimitPerBatch: " + maxAssignmentsLimitPerBatch + ", maxAssignmentsBatchesToHandleBeforeRestart: " + maxAssignmentsBatchesToHandleBeforeShutdown + ", controllerBaseUrl: " + controllerBaseUrl);    // It's safe and helpful to show them in the logs.
 
         } catch (Exception e) {
             String errorMsg = "An error prevented the retrieval of the workerId and the controllerBaseUrl from the file: " + inputDataFilePath + "\n" + e.getMessage();

src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java

@@ -2,8 +2,11 @@ package eu.openaire.urls_worker.controllers;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
@@ -18,18 +21,54 @@ public class GeneralController {
 
     private static final Logger logger = LoggerFactory.getLogger(GeneralController.class);
 
+    @Value("${security.shutdownOrCancelCode}")
+    private String shutdownOrCancelCode;
+
 
     public GeneralController() {}
 
+
     @GetMapping("isAlive")
     public ResponseEntity<?> isWorkerAlive() {
-
         logger.info("Received an \"isAlive\" request.");
-
         return ResponseEntity.ok().build();
     }
 
 
+    public static boolean shouldShutdownWorker = false;
+
+    @GetMapping("shutdownWorker/{shutdownCode}")
+    public ResponseEntity<?> shutdownWorkerGracefully(@PathVariable String shutdownCode)
+    {
+        String initMsg = "Received a \"shutdownWorker\" request.";
+        if ( shutdownCode.equals(shutdownOrCancelCode) ) {
+            shouldShutdownWorker = true;
+            logger.info(initMsg + " The worker will shutdown, after finishing current work.");
+            return ResponseEntity.ok().build();
+        } else {
+            String errorMsg = initMsg + " But, it has an invalid \"shutdownCode\": " + shutdownCode;
+            logger.error(errorMsg);
+            return ResponseEntity.status(HttpStatus.FORBIDDEN).body(errorMsg);
+        }
+    }
+
+
+    @GetMapping("cancelShutdownWorker/{cancelCode}")
+    public ResponseEntity<?> cancelShutdownWorkerGracefully(@PathVariable String cancelCode)
+    {
+        String initMsg = "Received a \"cancelShutdownWorker\" request.";
+        if ( cancelCode.equals(shutdownOrCancelCode) ) {
+            shouldShutdownWorker = false;
+            logger.info(initMsg + " Any previous \"shutdownWorker\"-request is canceled. The \"maxAssignmentsBatchesToHandleBeforeShutdown\" will still be honored (if it's set).");
+            return ResponseEntity.ok().build();
+        } else {
+            String errorMsg = initMsg + " But, it has an invalid \"cancelCode\": " + cancelCode;
+            logger.error(errorMsg);
+            return ResponseEntity.status(HttpStatus.FORBIDDEN).body(errorMsg);
+        }
+    }
+
+
     @GetMapping("getHandledAssignmentsCounts")
     public ResponseEntity<?> getHandledAssignmentsCounts()
     {

src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java

@@ -5,6 +5,7 @@ import com.google.common.collect.Multimap;
 import eu.openaire.publications_retriever.util.url.GenericUtils;
 import eu.openaire.publications_retriever.util.url.UrlUtils;
 import eu.openaire.urls_worker.UrlsWorkerApplication;
+import eu.openaire.urls_worker.controllers.GeneralController;
 import eu.openaire.urls_worker.models.Assignment;
 import eu.openaire.urls_worker.models.UrlReport;
 import eu.openaire.urls_worker.payloads.requests.AssignmentsRequest;
@@ -129,7 +130,7 @@ public class AssignmentsHandler {
         else
             postWorkerReport(assignmentRequestCounter);
 
-        numHandledAssignmentsBatches ++;    // This is used later to stop this app, when a user-defined upper limit is reached.
+        numHandledAssignmentsBatches ++;    // This is used later to stop this app, if a user-defined upper limit is set and reached.
 
         // Every time we reach a "limit" of handled id-url clear some data-structures of the underlying "PublicationsRetriever" program.
         // This helps with reducing the memory consumption over the period of weeks or months, and also give a 2nd chance to some domains which may be blocked due to a connectivity issues, but after a month they may be fine.
@@ -140,9 +141,12 @@ public class AssignmentsHandler {
         if ( idUrlPairsHandled >= idUrlsToHandleBeforeClearingDomainAndPathTrackingData )
             GenericUtils.clearDomainAndPathTrackingData();
 
-        if ( AssignmentsHandler.numHandledAssignmentsBatches == UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart )
+        if ( GeneralController.shouldShutdownWorker
+                || (AssignmentsHandler.numHandledAssignmentsBatches == UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeShutdown) )
         {
-            logger.info("The maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart + ") to be handled was reached! Shut down, in order for the external Linux-service to restart on its own..");
+            logger.info("The worker will now shutdown, as " + (GeneralController.shouldShutdownWorker
+                    ? "it received a \"shutdownWorker\" request!"
+                    : "the maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeShutdown + ") to be handled was reached!"));
             UrlsWorkerApplication.gentleAppShutdown();
             // The "gentleAppShutdown()" will exit the app, so the "isAvailableForWork" will not be set below.
         }
@@ -152,6 +156,10 @@ public class AssignmentsHandler {
     }
 
 
+    /**
+     * Post the worker report and wait for the Controller to request the publication-files.
+     * Once the Controller finishes with uploading the files to the S3-ObjectStore, it returns an "HTTP-200-OK" response to the Worker.
+     * */
     public static boolean postWorkerReport(Long assignmentRequestCounter)
     {
         String postUrl = UrlsWorkerApplication.controllerBaseUrl + "urls/addWorkerReport";

src/main/resources/application.properties

@@ -15,6 +15,10 @@ server.port = 1881
 # Server api path
 server.servlet.context-path=/api
 
+# Here set the code to be checked, when receiving "shutdown" and "cancel-shutdown" requests.
+security.shutdownOrCancelCode=
+
+
 # LOGGING LEVELS
 logging.config=classpath:logback-spring.xml
 logging.level.root=INFO