diff --git a/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java b/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java index f609153..41ceb1b 100644 --- a/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java +++ b/src/main/java/eu/openaire/urls_worker/UrlsWorkerApplication.java @@ -41,7 +41,7 @@ public class UrlsWorkerApplication { private static final String inputDataFilePath = FileUtils.workingDir + "inputData.txt"; public static String workerId = null; public static int maxAssignmentsLimitPerBatch = 0; - public static int maxAssignmentsBatchesToHandleBeforeRestart = -1; // Default value: -1 = argument-absent, 0 = infinite-batches + public static int maxAssignmentsBatchesToHandleBeforeShutdown = -1; // Default value: -1 = argument-absent, 0 = infinite-batches public static String controllerBaseUrl = null; // BaseUrl template: "http://IP:PORT/api/" private static ConfigurableApplicationContext context; @@ -64,7 +64,7 @@ public class UrlsWorkerApplication { public static void gentleAppShutdown() { - int exitCode = SpringApplication.exit(context, () -> 0); // The "PreDestroy" method will be called. + int exitCode = SpringApplication.exit(context, () -> 0); // The "PreDestroy" method will be called. (the "context" will be closed automatically (I checked it)) System.exit(exitCode); } @@ -145,10 +145,10 @@ public class UrlsWorkerApplication { } String maxAssignmentsBatchesStr = data[2].trim(); try { - maxAssignmentsBatchesToHandleBeforeRestart = Integer.parseInt(maxAssignmentsBatchesStr); + maxAssignmentsBatchesToHandleBeforeShutdown = Integer.parseInt(maxAssignmentsBatchesStr); } catch (NumberFormatException nfe) { logger.warn("The given \"maxAssignmentsBatchesToHandleBeforeRestart\" (" + maxAssignmentsBatchesStr + ") was not a number! Will handle an infinite number of batches!"); - maxAssignmentsBatchesToHandleBeforeRestart = 0; + maxAssignmentsBatchesToHandleBeforeShutdown = 0; } controllerBaseUrl = data[3].trim(); try { @@ -163,14 +163,14 @@ public class UrlsWorkerApplication { controllerBaseUrl += "/"; // Make sure the other urls will not break later. } - if ( (workerId == null) || (maxAssignmentsLimitPerBatch == 0) || (maxAssignmentsBatchesToHandleBeforeRestart == -1) || (controllerBaseUrl == null) ) { + if ( (workerId == null) || (maxAssignmentsLimitPerBatch == 0) || (maxAssignmentsBatchesToHandleBeforeShutdown == -1) || (controllerBaseUrl == null) ) { String errorMsg = "No \"workerId\" or/and \"maxAssignmentsLimitPerBatch\" or/and \"maxAssignmentsBatchesToHandleBeforeRestart\" or/and \"controllerBaseUrl\" could be retrieved from the file: " + inputDataFilePath; logger.error(errorMsg); System.err.println(errorMsg); System.exit(63); } - logger.info("workerId: " + workerId + ", maxAssignmentsLimitPerBatch: " + maxAssignmentsLimitPerBatch + ", maxAssignmentsBatchesToHandleBeforeRestart: " + maxAssignmentsBatchesToHandleBeforeRestart + ", controllerBaseUrl: " + controllerBaseUrl); // It's safe and helpful to show them in the logs. + logger.info("workerId: " + workerId + ", maxAssignmentsLimitPerBatch: " + maxAssignmentsLimitPerBatch + ", maxAssignmentsBatchesToHandleBeforeRestart: " + maxAssignmentsBatchesToHandleBeforeShutdown + ", controllerBaseUrl: " + controllerBaseUrl); // It's safe and helpful to show them in the logs. } catch (Exception e) { String errorMsg = "An error prevented the retrieval of the workerId and the controllerBaseUrl from the file: " + inputDataFilePath + "\n" + e.getMessage(); diff --git a/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java b/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java index f16ca5d..25a993e 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java @@ -2,8 +2,11 @@ package eu.openaire.urls_worker.controllers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -18,18 +21,54 @@ public class GeneralController { private static final Logger logger = LoggerFactory.getLogger(GeneralController.class); + @Value("${security.shutdownOrCancelCode}") + private String shutdownOrCancelCode; + public GeneralController() {} + @GetMapping("isAlive") public ResponseEntity isWorkerAlive() { - logger.info("Received an \"isAlive\" request."); - return ResponseEntity.ok().build(); } + public static boolean shouldShutdownWorker = false; + + @GetMapping("shutdownWorker/{shutdownCode}") + public ResponseEntity shutdownWorkerGracefully(@PathVariable String shutdownCode) + { + String initMsg = "Received a \"shutdownWorker\" request."; + if ( shutdownCode.equals(shutdownOrCancelCode) ) { + shouldShutdownWorker = true; + logger.info(initMsg + " The worker will shutdown, after finishing current work."); + return ResponseEntity.ok().build(); + } else { + String errorMsg = initMsg + " But, it has an invalid \"shutdownCode\": " + shutdownCode; + logger.error(errorMsg); + return ResponseEntity.status(HttpStatus.FORBIDDEN).body(errorMsg); + } + } + + + @GetMapping("cancelShutdownWorker/{cancelCode}") + public ResponseEntity cancelShutdownWorkerGracefully(@PathVariable String cancelCode) + { + String initMsg = "Received a \"cancelShutdownWorker\" request."; + if ( cancelCode.equals(shutdownOrCancelCode) ) { + shouldShutdownWorker = false; + logger.info(initMsg + " Any previous \"shutdownWorker\"-request is canceled. The \"maxAssignmentsBatchesToHandleBeforeShutdown\" will still be honored (if it's set)."); + return ResponseEntity.ok().build(); + } else { + String errorMsg = initMsg + " But, it has an invalid \"cancelCode\": " + cancelCode; + logger.error(errorMsg); + return ResponseEntity.status(HttpStatus.FORBIDDEN).body(errorMsg); + } + } + + @GetMapping("getHandledAssignmentsCounts") public ResponseEntity getHandledAssignmentsCounts() { diff --git a/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java index 8c9ed0e..283cd98 100644 --- a/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/util/AssignmentsHandler.java @@ -5,6 +5,7 @@ import com.google.common.collect.Multimap; import eu.openaire.publications_retriever.util.url.GenericUtils; import eu.openaire.publications_retriever.util.url.UrlUtils; import eu.openaire.urls_worker.UrlsWorkerApplication; +import eu.openaire.urls_worker.controllers.GeneralController; import eu.openaire.urls_worker.models.Assignment; import eu.openaire.urls_worker.models.UrlReport; import eu.openaire.urls_worker.payloads.requests.AssignmentsRequest; @@ -129,7 +130,7 @@ public class AssignmentsHandler { else postWorkerReport(assignmentRequestCounter); - numHandledAssignmentsBatches ++; // This is used later to stop this app, when a user-defined upper limit is reached. + numHandledAssignmentsBatches ++; // This is used later to stop this app, if a user-defined upper limit is set and reached. // Every time we reach a "limit" of handled id-url clear some data-structures of the underlying "PublicationsRetriever" program. // This helps with reducing the memory consumption over the period of weeks or months, and also give a 2nd chance to some domains which may be blocked due to a connectivity issues, but after a month they may be fine. @@ -140,9 +141,12 @@ public class AssignmentsHandler { if ( idUrlPairsHandled >= idUrlsToHandleBeforeClearingDomainAndPathTrackingData ) GenericUtils.clearDomainAndPathTrackingData(); - if ( AssignmentsHandler.numHandledAssignmentsBatches == UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart ) + if ( GeneralController.shouldShutdownWorker + || (AssignmentsHandler.numHandledAssignmentsBatches == UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeShutdown) ) { - logger.info("The maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeRestart + ") to be handled was reached! Shut down, in order for the external Linux-service to restart on its own.."); + logger.info("The worker will now shutdown, as " + (GeneralController.shouldShutdownWorker + ? "it received a \"shutdownWorker\" request!" + : "the maximum assignments-batches (" + UrlsWorkerApplication.maxAssignmentsBatchesToHandleBeforeShutdown + ") to be handled was reached!")); UrlsWorkerApplication.gentleAppShutdown(); // The "gentleAppShutdown()" will exit the app, so the "isAvailableForWork" will not be set below. } @@ -152,6 +156,10 @@ public class AssignmentsHandler { } + /** + * Post the worker report and wait for the Controller to request the publication-files. + * Once the Controller finishes with uploading the files to the S3-ObjectStore, it returns an "HTTP-200-OK" response to the Worker. + * */ public static boolean postWorkerReport(Long assignmentRequestCounter) { String postUrl = UrlsWorkerApplication.controllerBaseUrl + "urls/addWorkerReport"; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 49316d2..cf37dbd 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -15,6 +15,10 @@ server.port = 1881 # Server api path server.servlet.context-path=/api +# Here set the code to be checked, when receiving "shutdown" and "cancel-shutdown" requests. +security.shutdownOrCancelCode= + + # LOGGING LEVELS logging.config=classpath:logback-spring.xml logging.level.root=INFO