- Code optimization and polishing.

- Update dependencies.

build.gradle

@@ -1,6 +1,6 @@
 plugins {
-    id 'org.springframework.boot' version '2.7.14'
+    id 'org.springframework.boot' version '2.7.15'
-    id 'io.spring.dependency-management' version '1.1.2'
+    id 'io.spring.dependency-management' version '1.1.3'
     id 'java'
 }
 
@@ -39,7 +39,7 @@ dependencies {
         exclude group: 'io.minio'   // This is not used in the Worker, since it's the Controller which uploads the full-texts to S3. It also includes an older "commons-compress" version which causes problems.
     }
 
-    implementation group: 'com.google.guava', name: 'guava', version: '32.1.1-jre'
+    implementation group: 'com.google.guava', name: 'guava', version: '32.1.2-jre'
 
     // https://mvnrepository.com/artifact/com.google.code.gson/gson
     implementation 'com.google.code.gson:gson:2.10.1'

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
 validateDistributionUrl=true
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME

installAndRun.sh

@@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then
 	echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>";	exit 2
 fi
 
-gradleVersion="8.2.1"
+gradleVersion="8.3"
 
 shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.
 

src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java

@@ -178,7 +178,7 @@ public class AssignmentsHandler {
             return;
         }   // In this case, no assignments were processed.
 
-        PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " finished after: ");
+        PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " (containing " + assignmentsSize + " assignments) finished after: ");
 
         if ( askForTest ) {
             logger.debug("UrlReports:");  // DEBUG!

src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java

@@ -39,13 +39,15 @@ public class ScheduledTasks {
     private String workerReportsDirPath;
 
     private static final File rootPath = new File("/");
-    private static final long oneAndHalfGB = 1_610_612_736L;    // 1.5 GB free space per 1.000-assignments-batch.
+    private static final long oneAndHalfGB = 1_610_612_736L;    // We need 1.5 GB free space per 1.000-assignments-batch.
     private static long requiredFreeSpace;
 
+    private static final int oneMb = (1024 * 1024);
+
 
     public ScheduledTasks(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch)
     {
-        if ( maxAssignmentsLimitPerBatch < 1_000 )
+        if ( maxAssignmentsLimitPerBatch <= 1_000 )
             requiredFreeSpace = oneAndHalfGB;
         else
             requiredFreeSpace = oneAndHalfGB * (maxAssignmentsLimitPerBatch / 1_000);
@@ -54,14 +56,15 @@ public class ScheduledTasks {
     }
 
     @Scheduled(fixedDelay = 1)  // Request the next batch immediately after the last one finishes.
-    public void handleNewAssignments() {
+    public void handleNewAssignments()
+    {
         if ( GeneralController.shouldShutdownWorker || AssignmentsHandler.shouldNotRequestMore ) {
             // Here we will be right after the Worker has posted its last report. It is guaranteed that the Controller will not have processed it and have not requested the full-text files.
             // We do not want to shut down the controller.
             return;
         }
 
-        if ( rootPath.getFreeSpace() < requiredFreeSpace ) {
+        if ( rootPath.getFreeSpace() <= requiredFreeSpace ) {
             // It's not safe to proceed with downloading more files and risk of "noSpaceLeft" error.
             // Wait for the Controller to take the full-texts and any remaining files to be deleted, so that more free-space becomes available.
             // We need to have some buffer zone for the ".tar" files which will be created from the already downloaded full-texts, when the Controller starts requesting them.
@@ -69,7 +72,7 @@ public class ScheduledTasks {
                 AssignmentsHandler.shouldNotRequestMore = true;
                 return;
             }
-            logger.warn("The free space is running out (less than " + (requiredFreeSpace / (1024 * 1024)) + " Mb). The Worker will avoid getting new assignments for the next 15 minutes.");
+            logger.warn("The free space is running out (less than " + (requiredFreeSpace / oneMb) + " Mb). The Worker will avoid getting new assignments for the next 15 minutes.");
             try {
                 Thread.sleep(900_000);  // Sleep for 15 mins to stall the scheduler from retrying right away, thus giving time to the disk-space to be freed.
             } catch (InterruptedException ie) {
@@ -132,6 +135,8 @@ public class ScheduledTasks {
 
     private static final double hoursToWaitBeforeDeletion = 36.0;
 
+    private static final int hoursDivisor = (1000 * 60 * 60);    // In order to get the time-difference in hours. We divide with: /1000 to get seconds, /60 to get minutes and /60 to get hours.
+
     @Scheduled(initialDelay = 21_600_000, fixedDelay = 21_600_000)  // InitialDelay & FixedDelay = 36 hours.
     //@Scheduled(initialDelay = 120_000, fixedDelay = 120_000)  // Just for testing (every 2 mins).
     public void checkAndDeleteOldFiles() {
@@ -149,7 +154,6 @@ public class ScheduledTasks {
         // Also, the ".tar.zstd" file of last batch will be deleted here, as well as the whole directory itself.
 
         logger.debug("Going to check if any leftover full-texts exist and delete them.");
-
         int usableDirsNum = 0;
         try {
             File fullTextsBaseDir = new File(fileStorageService.assignmentsBaseLocation);
@@ -180,7 +184,7 @@ public class ScheduledTasks {
                     logger.trace("The subDir \"" + subDir.getName() + "\" was last accessed in: " + new Date(lastModified));
 
                 // Get the difference in hours. /1000 to get seconds, /60 to get minutes and /60 to get hours.
-                double elapsedHours = (double) (currentTime - lastModified) / (1000 * 60 * 60);
+                double elapsedHours = (double) (currentTime - lastModified) / hoursDivisor;
                 if ( elapsedHours > hoursToWaitBeforeDeletion ) {
                     // Enough time has passed, the directory should be deleted immediately.
                     String subDirName = subDir.getName();
@@ -193,7 +197,7 @@ public class ScheduledTasks {
                         String assingmentsCounterString = matcher.group(1);
                         if ( (assingmentsCounterString != null) && !assingmentsCounterString.isEmpty()) {
                             if ( FullTextsController.deleteFile(this.workerReportsDirPath + this.workerId + "_assignments_" + assingmentsCounterString + "_report.json") )
-                                logger.warn("The subDir \"" + subDirName + "\" probably contains some failed file, since the workerReport for assignments_" + assingmentsCounterString + " was deleted only now, which means the Controller failed to successfully process the results of those assignments.");
+                                logger.warn("The subDir \"" + subDirName + "\" probably contains some failed files, since the workerReport for assignments_" + assingmentsCounterString + " was deleted only now, which means the Controller failed to successfully process the results of those assignments.");
                         }
                         else
                             logger.error("The subDir \"" + subDirName + "\" has an invalid name! It does not contains the assignmentsCounter!");