- Automatically delete the unsuccessful WorkerReports, which are more than 7 days old.

- Optimize the Service's startup speed, by setting "initialDelays" to the scheduled tasks.
- Optimize documentation.

src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java

@@ -4,13 +4,18 @@ import eu.openaire.urls_controller.Application;
 import eu.openaire.urls_controller.controllers.ShutdownController;
 import eu.openaire.urls_controller.controllers.UrlsController;
 import eu.openaire.urls_controller.services.FullTextsServiceImpl;
+import eu.openaire.urls_controller.util.FileUtils;
 import eu.openaire.urls_controller.util.GenericUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
+import java.io.File;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CancellationException;
@@ -23,9 +28,24 @@ public class ScheduledTasks {
 
     private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
 
+    @Autowired
+    FileUtils fileUtils;
 
-    @Scheduled(fixedDelay = 3_600_000)  // Execute this method 1 hour after the last execution, in order for some tasks to have been gathered.
-    //@Scheduled(fixedDelay = 20_000)    // Just for testing (every 20 secs).
+
+    private final String workerReportsDirPath;
+
+
+    public ScheduledTasks(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath)
+    {
+        if ( !workerReportsDirPath.endsWith("/") )
+            workerReportsDirPath += "/";
+
+        this.workerReportsDirPath = workerReportsDirPath;   // This dir will be created later.
+    }
+
+
+    @Scheduled(initialDelay = 3_600_000, fixedDelay = 3_600_000)  // Execute this method 1 hour from the start and 1 hour after each last execution, in order for some tasks to have been gathered.
+    //@Scheduled(initialDelay = 3_600_000, fixedDelay = 20_000)    // Just for testing (every 20 secs).
     public void executeBackgroundTasks()
     {
         List<Callable<Boolean>> tempList = new ArrayList<>(FullTextsServiceImpl.backgroundCallableTasks);   // Copy the list in order to know what was executed and delete only that data later.
@@ -60,8 +80,8 @@ public class ScheduledTasks {
     }
 
 
-    @Scheduled(fixedDelay = 7_200_000)  // Check every 2 hours.
-    //@Scheduled(fixedDelay = 20_000)    // Just for testing (every 20 secs).
+    @Scheduled(initialDelay = 60_000, fixedDelay = 7_200_000)  // Check every 2 hours.
+    //@Scheduled(initialDelay = 60_000, fixedDelay = 20_000)    // Just for testing (every 20 secs).
     public void checkIfServiceIsReadyForShutdown()
     {
         if ( ! ShutdownController.shouldShutdownService )
@@ -76,4 +96,56 @@ public class ScheduledTasks {
         Application.gentleAppShutdown();
     }
 
+
+    private static final int daysToWaitBeforeDeletion = 7;
+
+
+    @Scheduled(initialDelay = 120_000, fixedDelay = 604_800_000)  // Run every 7 days.
+    //@Scheduled(initialDelay = 120_000, fixedDelay = 20_000)    // Just for testing (every 20 secs).
+    public void checkAndDeleteUnsuccessfulWorkerReports()
+    {
+        logger.debug("Going to check and remove any unsuccessful workerReports, which are more than 7 days old.");
+        int usableDirsNum = 0;
+        try {
+            File workerReportsDir = new File(workerReportsDirPath);
+            if ( !workerReportsDir.isDirectory() ) {
+                logger.error("The \"workerReportsDir\" (" + workerReportsDirPath + ") does not exist!");    // This base dir should always exist!
+                return;
+            }
+
+            File[] workerReports = workerReportsDir.listFiles(File::isFile);
+            if ( workerReports == null ) {
+                logger.error("There was an error when getting the subDirs of \"workerReportsDir\": " + workerReportsDir);
+                return;
+            }
+
+            usableDirsNum = workerReports.length;
+            if ( usableDirsNum == 0 ) {
+                logger.debug("The \"workerReportsDir\" is empty, so there is nothing to delete.");
+                return;
+            }
+
+            long currentTime = System.currentTimeMillis();
+
+            // Loop through the array and print only the directories
+            for ( File workerReport : workerReports ) {
+                long lastModified = workerReport.lastModified();
+
+                if ( logger.isTraceEnabled() )
+                    logger.trace("The workerReport \"" + workerReport.getName() + "\" was last accessed in: " + new Date(lastModified));
+
+                // Get the difference in hours. /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days.
+                long elapsedWeeks = (currentTime - lastModified) / (1000 * 60 * 60 * 24);
+                if ( elapsedWeeks > daysToWaitBeforeDeletion ) {
+                    // Enough time has passed, the directory should be deleted immediately.
+                    String workerReportName = workerReport.getName();
+                    logger.warn("The workerReport \"" + workerReportName + "\" was accessed " + elapsedWeeks + " days ago (passed the " + daysToWaitBeforeDeletion + " days limit) and will be deleted.");
+                    fileUtils.deleteFile(workerReport.getAbsolutePath());
+                }
+            }
+        } catch (Exception e) {
+            logger.error("", e);
+        }
+    }
+
 }

src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java

@@ -46,7 +46,7 @@ public class UrlsController {
 
     public static final ConcurrentHashMap<String, WorkerInfo> workersInfoMap = new ConcurrentHashMap<>(6);
 
-    private String workerReportsDirPath;
+    private final String workerReportsDirPath;
 
 
     public UrlsController(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath)
@@ -174,6 +174,7 @@ public class UrlsController {
 
         // Create the report file and write the worker-report to it.
         String workerReportFile = currentWorkerReportLocationDir + "/" + curWorkerId + "_assignments_" + curReportAssignmentsCounter + "_report.json";
+        // In case a file with the same name already exists (e.g. from a previous run of the Service), then it will be overwritten.
         logger.debug("Going to write the worker report to json-file: \"" + workerReportFile + "\".");
         fileUtils.writeToFile(workerReportFile, workerReport.getJsonReport(), false);  // Only one thread is writing to this specific file.
         // The above method will overwrite a possibly existing file. So in case of a crash, it's better to back up the reports before starting the Controller again (as the assignments-counter will start over, from 0).