From 164245cb539293678e68ebc2f3e01a5fdfb32186 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Wed, 24 May 2023 16:59:42 +0300 Subject: [PATCH] - Automatically delete the unsuccessful WorkerReports, which are more than 7 days old. - Optimize the Service's startup speed, by setting "initialDelays" to the scheduled tasks. - Optimize documentation. --- .../components/ScheduledTasks.java | 80 ++++++++++++++++++- .../controllers/UrlsController.java | 3 +- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java index 65bf762..d20a476 100644 --- a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java @@ -4,13 +4,18 @@ import eu.openaire.urls_controller.Application; import eu.openaire.urls_controller.controllers.ShutdownController; import eu.openaire.urls_controller.controllers.UrlsController; import eu.openaire.urls_controller.services.FullTextsServiceImpl; +import eu.openaire.urls_controller.util.FileUtils; import eu.openaire.urls_controller.util.GenericUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import java.io.File; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; @@ -23,9 +28,24 @@ public class ScheduledTasks { private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class); + @Autowired + FileUtils fileUtils; - @Scheduled(fixedDelay = 3_600_000) // Execute this method 1 hour after the last execution, in order for some tasks to have been gathered. - //@Scheduled(fixedDelay = 20_000) // Just for testing (every 20 secs). + + private final String workerReportsDirPath; + + + public ScheduledTasks(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath) + { + if ( !workerReportsDirPath.endsWith("/") ) + workerReportsDirPath += "/"; + + this.workerReportsDirPath = workerReportsDirPath; // This dir will be created later. + } + + + @Scheduled(initialDelay = 3_600_000, fixedDelay = 3_600_000) // Execute this method 1 hour from the start and 1 hour after each last execution, in order for some tasks to have been gathered. + //@Scheduled(initialDelay = 3_600_000, fixedDelay = 20_000) // Just for testing (every 20 secs). public void executeBackgroundTasks() { List> tempList = new ArrayList<>(FullTextsServiceImpl.backgroundCallableTasks); // Copy the list in order to know what was executed and delete only that data later. @@ -60,8 +80,8 @@ public class ScheduledTasks { } - @Scheduled(fixedDelay = 7_200_000) // Check every 2 hours. - //@Scheduled(fixedDelay = 20_000) // Just for testing (every 20 secs). + @Scheduled(initialDelay = 60_000, fixedDelay = 7_200_000) // Check every 2 hours. + //@Scheduled(initialDelay = 60_000, fixedDelay = 20_000) // Just for testing (every 20 secs). public void checkIfServiceIsReadyForShutdown() { if ( ! ShutdownController.shouldShutdownService ) @@ -76,4 +96,56 @@ public class ScheduledTasks { Application.gentleAppShutdown(); } + + private static final int daysToWaitBeforeDeletion = 7; + + + @Scheduled(initialDelay = 120_000, fixedDelay = 604_800_000) // Run every 7 days. + //@Scheduled(initialDelay = 120_000, fixedDelay = 20_000) // Just for testing (every 20 secs). + public void checkAndDeleteUnsuccessfulWorkerReports() + { + logger.debug("Going to check and remove any unsuccessful workerReports, which are more than 7 days old."); + int usableDirsNum = 0; + try { + File workerReportsDir = new File(workerReportsDirPath); + if ( !workerReportsDir.isDirectory() ) { + logger.error("The \"workerReportsDir\" (" + workerReportsDirPath + ") does not exist!"); // This base dir should always exist! + return; + } + + File[] workerReports = workerReportsDir.listFiles(File::isFile); + if ( workerReports == null ) { + logger.error("There was an error when getting the subDirs of \"workerReportsDir\": " + workerReportsDir); + return; + } + + usableDirsNum = workerReports.length; + if ( usableDirsNum == 0 ) { + logger.debug("The \"workerReportsDir\" is empty, so there is nothing to delete."); + return; + } + + long currentTime = System.currentTimeMillis(); + + // Loop through the array and print only the directories + for ( File workerReport : workerReports ) { + long lastModified = workerReport.lastModified(); + + if ( logger.isTraceEnabled() ) + logger.trace("The workerReport \"" + workerReport.getName() + "\" was last accessed in: " + new Date(lastModified)); + + // Get the difference in hours. /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days. + long elapsedWeeks = (currentTime - lastModified) / (1000 * 60 * 60 * 24); + if ( elapsedWeeks > daysToWaitBeforeDeletion ) { + // Enough time has passed, the directory should be deleted immediately. + String workerReportName = workerReport.getName(); + logger.warn("The workerReport \"" + workerReportName + "\" was accessed " + elapsedWeeks + " days ago (passed the " + daysToWaitBeforeDeletion + " days limit) and will be deleted."); + fileUtils.deleteFile(workerReport.getAbsolutePath()); + } + } + } catch (Exception e) { + logger.error("", e); + } + } + } diff --git a/src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java b/src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java index f7eb76b..1f42d31 100644 --- a/src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java +++ b/src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java @@ -46,7 +46,7 @@ public class UrlsController { public static final ConcurrentHashMap workersInfoMap = new ConcurrentHashMap<>(6); - private String workerReportsDirPath; + private final String workerReportsDirPath; public UrlsController(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath) @@ -174,6 +174,7 @@ public class UrlsController { // Create the report file and write the worker-report to it. String workerReportFile = currentWorkerReportLocationDir + "/" + curWorkerId + "_assignments_" + curReportAssignmentsCounter + "_report.json"; + // In case a file with the same name already exists (e.g. from a previous run of the Service), then it will be overwritten. logger.debug("Going to write the worker report to json-file: \"" + workerReportFile + "\"."); fileUtils.writeToFile(workerReportFile, workerReport.getJsonReport(), false); // Only one thread is writing to this specific file. // The above method will overwrite a possibly existing file. So in case of a crash, it's better to back up the reports before starting the Controller again (as the assignments-counter will start over, from 0).