diff --git a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java index d255fa2..947d9ac 100644 --- a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java @@ -66,6 +66,7 @@ public class ScheduledTasks { @Scheduled(initialDelay = 900_000, fixedDelay = 1_800_000) // InitialDelay = 15 mins, FixedDelay = 30 mins. + //@Scheduled(initialDelay = 20_000, fixedDelay = 20_000) // Just for testing (every 20 secs). public void checkIfShouldShutdown() { if ( !GeneralController.shouldShutdownWorker && !AssignmentsHandler.shouldNotRequestMore ) @@ -79,15 +80,16 @@ public class ScheduledTasks { File[] fulltextSubDirs = fullTextsBaseDir.listFiles(File::isDirectory); if ( fulltextSubDirs == null ) { logger.error("There was an error when getting the subDirs of \"fullTextsBaseDir\": " + fullTextsBaseDir); - return; + return; // It's NOT safe to shut down. } if ( fulltextSubDirs.length > 0 ) { - logger.warn("The base full-texts directory still has sub-directories with full-texts, wait for the Controller to take all the files or some time to past before they are deleted. Then the Worker will shut down."); + logger.warn("The base full-texts directory still has sub-directories with full-texts, wait for the Controller to take all the files, or wait some time to past before they are deleted. Then the Worker will shut down."); + // Some subDirs may be left behind due to some error when processing the WorkerReport. In that case, return; } else logger.debug("The \"fullTextsBaseDir\" is empty. Shutting down.."); } else - logger.warn("The base full-texts directory was not found Shutting down.."); + logger.warn("The base full-texts directory was not found! Shutting down.."); // This base-directory should exist during run-time, but we can proceed with shutting down the Service. connWithController.postShutdownReportToController(workerId); UrlsWorkerApplication.gentleAppShutdown(); @@ -97,9 +99,10 @@ public class ScheduledTasks { private static final Pattern ASSIGNMENTS_COUNTER = Pattern.compile(".*assignments_([\\d]+).*"); - private static final int hoursToWaitBeforeDeletion = 36; + private static final double hoursToWaitBeforeDeletion = 36.0; @Scheduled(initialDelay = 21_600_000, fixedDelay = 21_600_000) // InitialDelay & FixedDelay = 36 hours. + //@Scheduled(initialDelay = 20_000, fixedDelay = 20_000) // Just for testing (every 20 secs). public void checkAndDeleteOldFiles() { // For any reason the Worker-report connection with the Controller may fail, but the Controller will continue requesting the full-text batches. @@ -146,7 +149,7 @@ public class ScheduledTasks { logger.trace("The subDir \"" + subDir.getName() + "\" was last accessed in: " + new Date(lastModified)); // Get the difference in hours. /1000 to get seconds, /60 to get minutes and /60 to get hours. - long elapsedHours = (currentTime - lastModified) / (1000 * 60 * 60); + double elapsedHours = (double) (currentTime - lastModified) / (1000 * 60 * 60); if ( elapsedHours > hoursToWaitBeforeDeletion ) { // Enough time has passed, the directory should be deleted immediately. String subDirName = subDir.getName(); diff --git a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java index 19acc7b..2b21bab 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java @@ -56,7 +56,7 @@ public class FullTextsController { return ResponseEntity.badRequest().body(errorMsg); } - logger.info("Received a \"getFullTexts\" request for returning a tar-file containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ")."); + logger.info("Received a \"getFullTexts\" request for returning a \".tar.zstd\" file, containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ")."); String currentAssignmentsBaseFullTextsPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator;