lsmyrnaios
/
UrlsController
1
0
Fork 1
Code Issues Pull Requests Releases 13 Wiki Activity

- Add checks to verify that there are active workers in the Service, before proceeding to try posting "(cancel)Shutdown" requests to all known workers. 

- Add documentation in README.
This commit is contained in:
Lampros Smyrnaios 2024-06-14 13:32:44 +03:00
parent 3417e5c68c
commit 0d63165b6d
2 changed files with 30 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -60,6 +60,12 @@ For managing and generating data, we use [**Impala**](https://impala.apache.org/
Once the Workers are about to shut down, they send a "shutdownReport" to the Controller. A scheduling task runs in the Controller, every 2 hours, and if the user has specified that the Controller must shut down and all the Workers participating in the Service have shutdown, then it gently shuts down the Controller.
- "**cancelShutdownService**" endpoint: **http://localhost:\<PORT\>/api/cancelShutdownService** <br>
This endpoint specifies that the Controller will not shut down, and sends "cancelShutdownWorker" requests to all the Workers which are actively participating in the Service (have not shut down yet), so that they can continue to request assignments.
- "**shutdownAllWorkers**" endpoint: **http://localhost:\<PORT\>/api/shutdownAllWorkers** <br>
This endpoint sends "shutdownWorker" requests to all the Workers which are actively participating in the Service. The Workers will shut down after finishing their work-in-progress and all full-texts have been either transferred to the Controller or deleted, in case an error has appeared.<br>
Once the Workers are about to shut down, they send a "shutdownReport" to the Controller. <br>
This endpoint is helpful when we want to update only the Workers, while keeping the Service running for Bulk-import procedures.
- "**cancelShutdownAllWorkers**" endpoint: **http://localhost:\<PORT\>/api/cancelShutdownAllWorkers** <br>
This endpoint specifies that the Workers will not shut down, and sends "cancelShutdownWorker" requests to all the Workers which are actively participating in the Service (have not shut down yet), so that they can continue to request assignments.
<br>
Note: The Shutdown Service API is accessible by the Controller's host machine.

41
src/main/java/eu/openaire/urls_controller/controllers/ShutdownController.java
View File

@ -46,10 +46,13 @@ public class ShutdownController {
} else {
shouldShutdownService = true;
endingMsg = "The service will shutdown, after finishing current work.";
logger.info(initMsg + endingMsg);
shutdownService.postShutdownOrCancelRequestsToAllWorkers(false);
if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
endingMsg += " None of the workers is active in order to post shutdown requests to.";
logger.info(initMsg + endingMsg);
} else {
logger.info(initMsg + endingMsg); // Show this message before the post-operation, as the following code may take some time to finish.
shutdownService.postShutdownOrCancelRequestsToAllWorkers(false);
}
// That's it for now. The workers may take some hours to finish their work (including delivering the full-text files).
// A scheduler monitors the shutdown of the workers. Once all worker have shutdown, the Controller shuts down as well.
}
@ -70,11 +73,13 @@ public class ShutdownController {
shouldShutdownService = false;
String endingMsg = "Any previous \"shutdownService\"-request is canceled.";
logger.info(initMsg + endingMsg);
// Cancel the shutdown of the workers, if we are able to catch up with them before they have already shutdown..
shutdownService.postShutdownOrCancelRequestsToAllWorkers(true);
if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
endingMsg += " None of the workers is active in order to post cancel-shutdown requests to.";
logger.info(initMsg + endingMsg);
} else {
logger.info(initMsg + endingMsg); // Show this message before the post-operation, as the following code may take some time to finish.
shutdownService.postShutdownOrCancelRequestsToAllWorkers(true); // Cancel the shutdown of the workers, if we are able to catch up with them before they have already shutdown..
}
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
}
@ -96,18 +101,18 @@ public class ShutdownController {
if ( shouldShutdownAllWorkers ) {
endingMsg = "The controller has already received a \"shutdownAllWorkers\" request (which was not canceled afterwards).";
logger.info(initMsg + endingMsg);
} else if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
endingMsg = "None of the workers is active in order to post shutdown requests to.";
logger.info(initMsg + endingMsg);
} else {
shouldShutdownAllWorkers = true;
endingMsg = "All workers will shutdown, after finishing current work.";
logger.info(initMsg + endingMsg);
shutdownService.postShutdownOrCancelRequestsToAllWorkers(false);
// That's it for now. The workers may take some hours to finish their work (including delivering the full-text files).
// The service will continue to run and handle bulk-import requests.
// Once the workers are ready to work again, they can be started without any additional configuration.
}
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
}
@ -124,11 +129,13 @@ public class ShutdownController {
shouldShutdownAllWorkers = false;
String endingMsg = "Any previous \"shutdownAllWorkers\"-request is canceled.";
logger.info(initMsg + endingMsg);
// Cancel the shutdown of the workers, if we are able to catch up with them before they have already shutdown..
shutdownService.postShutdownOrCancelRequestsToAllWorkers(true);
if ( UrlsController.numOfActiveWorkers.get() == 0 ) {
endingMsg += " None of the workers is active in order to post cancel-shutdown requests to.";
logger.info(initMsg + endingMsg);
} else {
logger.info(initMsg + endingMsg); // Show this message before the post-operation, as the following code may take some time to finish.
shutdownService.postShutdownOrCancelRequestsToAllWorkers(true); // Cancel the shutdown of the workers, if we are able to catch up with them before they have already shutdown..
}
return ResponseEntity.ok().body(endingMsg + GenericUtils.endOfLine);
}