- Fix the process of shutting down the worker, in case the user sends the relevant request, while the worker is stuck in a data-request error-loop.

- Upload the updated gradle-wrapper.
This commit is contained in:
Lampros Smyrnaios 2024-04-29 17:08:40 +03:00
parent 34407179fc
commit b40c72f78f
4 changed files with 11 additions and 8 deletions

Binary file not shown.

View File

@ -47,7 +47,7 @@ public class AssignmentsHandler {
private final String workerId; private final String workerId;
private final String controllerBaseUrl; private final String controllerBaseUrl;
private final int maxAssignmentsLimitPerBatch; private final int maxAssignmentsLimitPerBatch;
private final int maxAssignmentsBatchesToHandleBeforeShutdown; public final int maxAssignmentsBatchesToHandleBeforeShutdown;
public static List<UrlReport> urlReports = null; public static List<UrlReport> urlReports = null;
private static final int expectedDatasourcesPerRequest = 1400; // Per 10_000 assignments. private static final int expectedDatasourcesPerRequest = 1400; // Per 10_000 assignments.

View File

@ -65,12 +65,6 @@ public class ScheduledTasks {
return; return;
} }
// The user might have just requested the Worker to shut-down.
if ( GeneralController.shouldShutdownWorker ) { // Make sure the worker shuts-down, in case the user sends the relevant request, while the worker is stuck in a data-request error-loop.
AssignmentsHandler.shouldNotRequestMore = true;
return;
}
if ( rootPath.getFreeSpace() < requiredFreeSpace ) { if ( rootPath.getFreeSpace() < requiredFreeSpace ) {
// It's not safe to proceed with downloading more files and risk of "noSpaceLeft" error. // It's not safe to proceed with downloading more files and risk of "noSpaceLeft" error.
// Wait for the Controller to take the full-texts and any remaining files to be deleted, so that more free-space becomes available. // Wait for the Controller to take the full-texts and any remaining files to be deleted, so that more free-space becomes available.

View File

@ -5,6 +5,7 @@ import eu.openaire.urls_worker.components.plugins.PublicationsRetrieverPlugin;
import eu.openaire.urls_worker.util.UriBuilder; import eu.openaire.urls_worker.util.UriBuilder;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
@ -22,6 +23,9 @@ public class GeneralController {
private static final Logger logger = LoggerFactory.getLogger(GeneralController.class); private static final Logger logger = LoggerFactory.getLogger(GeneralController.class);
@Autowired
AssignmentsHandler assignmentsHandler;
private final String controllerIp; private final String controllerIp;
private final String workerReportsDirPath; private final String workerReportsDirPath;
private final String workerId; private final String workerId;
@ -67,8 +71,10 @@ public class GeneralController {
String finalMsg = ""; String finalMsg = "";
if ( shouldShutdownWorker ) if ( shouldShutdownWorker )
finalMsg = "The worker has already received a \"shutdownWorker\" request (which was not canceled afterwards). "; finalMsg = "The worker has already received a \"shutdownWorker\" request (which was not canceled afterwards). ";
else else {
shouldShutdownWorker = true; shouldShutdownWorker = true;
AssignmentsHandler.shouldNotRequestMore = true;
}
finalMsg += "The worker will shutdown, after finishing current work."; finalMsg += "The worker will shutdown, after finishing current work.";
logger.info(initMsg + finalMsg); logger.info(initMsg + finalMsg);
@ -85,6 +91,9 @@ public class GeneralController {
return responseEntity; return responseEntity;
shouldShutdownWorker = false; shouldShutdownWorker = false;
if ( AssignmentsHandler.numHandledAssignmentsBatches < assignmentsHandler.maxAssignmentsBatchesToHandleBeforeShutdown )
AssignmentsHandler.shouldNotRequestMore = false; // Make sure the worker shuts-down, in case the user sends the relevant request, while the worker is stuck in a data-request error-loop.
String finalMsg = "Any previous \"shutdownWorker\"-request is canceled. The \"maxAssignmentsBatchesToHandleBeforeShutdown\" will still be honored (if it's set)."; String finalMsg = "Any previous \"shutdownWorker\"-request is canceled. The \"maxAssignmentsBatchesToHandleBeforeShutdown\" will still be honored (if it's set).";
logger.info(initMsg + finalMsg); logger.info(initMsg + finalMsg);
return ResponseEntity.ok().body(finalMsg + "\n"); return ResponseEntity.ok().body(finalMsg + "\n");