- Prevent running out of space, by checking the available free space and stalling the acquisition of new assignments until more free space becomes available.

- Fix missing change.
This commit is contained in:
Lampros Smyrnaios 2023-05-30 17:58:29 +03:00
parent 84f29ea7e0
commit a9b1b20a51
2 changed files with 32 additions and 1 deletions

View File

@ -29,7 +29,7 @@ shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.
if [[ justRun -eq 0 ]]; then if [[ justRun -eq 0 ]]; then
if [[ avoidReInstallingPublicationsRetriever -eq 1 ]]; then if [[ avoidReInstallingPublicationsRetriever -eq 1 ]]; then
if [[ ! -f ./libs/publications_retriever-1.0-SNAPSHOT.jar ]]; then if [[ ! -f ./libs/publications_retriever-1.1-SNAPSHOT.jar ]]; then
echo -e "\n\nThe required \"PublicationsRetriever\" software has not been installed yet, thus the script will override the user-defined value of \"avoidReInstallingPublicationsRetriever\" to FALSE..\n\n" echo -e "\n\nThe required \"PublicationsRetriever\" software has not been installed yet, thus the script will override the user-defined value of \"avoidReInstallingPublicationsRetriever\" to FALSE..\n\n"
avoidReInstallingPublicationsRetriever=0; # In case the jar-file does not exists, then make sure we follow the normal procedure, independently from what the user requested. avoidReInstallingPublicationsRetriever=0; # In case the jar-file does not exists, then make sure we follow the normal procedure, independently from what the user requested.
fi fi

View File

@ -38,6 +38,20 @@ public class ScheduledTasks {
@Value("${workerReportsDirPath}") @Value("${workerReportsDirPath}")
private String workerReportsDirPath; private String workerReportsDirPath;
private static final File rootPath = new File("/");
private static final long oneAndHalfGB = 1_610_612_736L; // 1.5 GB free space per 1.000-assignments-batch.
private static long requiredFreeSpace;
public ScheduledTasks(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch)
{
if ( maxAssignmentsLimitPerBatch < 1_000 )
requiredFreeSpace = oneAndHalfGB;
else
requiredFreeSpace = oneAndHalfGB * (maxAssignmentsLimitPerBatch / 1_000);
logger.info("The \"requiredFreeSpace\" for the app to request new assignments, having \"maxAssignmentsLimitPerBatch\" equal to " + maxAssignmentsLimitPerBatch + " , is: " + (requiredFreeSpace / (1024 * 1024)) + " Mb");
}
@Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes. @Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes.
public void handleNewAssignments() { public void handleNewAssignments() {
@ -47,6 +61,23 @@ public class ScheduledTasks {
return; return;
} }
if ( rootPath.getFreeSpace() < requiredFreeSpace ) {
// It's not safe to proceed with downloading more files and risk of "noSpaceLeft" error.
// Wait for the Controller to take the full-texts and any remaining files to be deleted, so that more free-space becomes available.
// We need to have some buffer zone for the ".tar" files which will be created from the already downloaded full-texts, when the Controller starts requesting them.
if ( GeneralController.shouldShutdownWorker ) { // Make sure the worker shuts-down, in case the user sends the relevant request, while the worker is stuck in a free-space check loop.
AssignmentsHandler.shouldNotRequestMore = true;
return;
}
logger.warn("The free space is running out (less than " + (requiredFreeSpace / (1024 * 1024)) + " Mb). Will avoid to get new assignments for the next 15 minutes.");
try {
Thread.sleep(900_000); // Sleep for 15 mins to stall the scheduler from retrying right away, thus giving time to the disk-space to be freed.
} catch (InterruptedException ie) {
logger.warn("Sleeping was interrupted!");
}
return; // Cause this method to be called again, so that the Free-space can be checked again before proceeding with new assignments.
}
if ( AssignmentsHandler.hadConnectionErrorOnRequest ) { if ( AssignmentsHandler.hadConnectionErrorOnRequest ) {
if ( GeneralController.shouldShutdownWorker ) { // Make sure the worker shuts-down, in case the user sends the relevant request, while the worker is stuck in a data-request error-loop. if ( GeneralController.shouldShutdownWorker ) { // Make sure the worker shuts-down, in case the user sends the relevant request, while the worker is stuck in a data-request error-loop.
AssignmentsHandler.shouldNotRequestMore = true; AssignmentsHandler.shouldNotRequestMore = true;