- Update dependencies.

- Code optimization and polishing.

build.gradle

@@ -1,13 +1,13 @@
 plugins {
-    id 'org.springframework.boot' version '2.7.13'
-    id 'io.spring.dependency-management' version '1.1.0'
+    id 'org.springframework.boot' version '2.7.14'
+    id 'io.spring.dependency-management' version '1.1.2'
     id 'java'
 }
 
 java {
     group = 'eu.openaire.urls_worker'
     version = '2.1.0-SNAPSHOT'
-    sourceCompatibility = '1.8'
+    sourceCompatibility = JavaVersion.VERSION_1_8
 }
 
 repositories {
@@ -44,8 +44,11 @@ dependencies {
     // https://mvnrepository.com/artifact/com.google.code.gson/gson
     implementation 'com.google.code.gson:gson:2.10.1'
 
-    implementation 'org.apache.commons:commons-compress:1.23.0'
-    implementation 'com.github.luben:zstd-jni:1.5.5-4'  // Even though this is part of the above dependency, it is needed separately as well, specifically here, in the Worker.
+    implementation("org.apache.commons:commons-compress:1.23.0") {
+        exclude group: 'com.github.luben', module: 'zstd-jni'
+    }
+    implementation 'com.github.luben:zstd-jni:1.5.5-5'  // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
+    // Also, for compressing, we strangely need it, otherwise it does not work.
 
     testImplementation 'org.springframework.security:spring-security-test'
     testImplementation "org.springframework.boot:spring-boot-starter-test"

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
-networkTimeout=10000
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
 validateDistributionUrl=true
+networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

installAndRun.sh

@@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then
 	echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>";	exit 2
 fi
 
-gradleVersion="8.2"
+gradleVersion="8.2.1"
 
 shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.
 

src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java

@@ -102,7 +102,7 @@ public class AssignmentsHandler {
 
     public AssignmentsRequest requestAssignments()
     {
-        logger.info("Going to request " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl);
+        logger.info("Going to request up to " + this.maxAssignmentsLimitPerBatch + " assignments from the Controller: " + requestUrl);
         AssignmentsRequest assignmentRequest = null;
         try {   // Here, the HTTP-request is executed.
             assignmentRequest = restTemplate.getForObject(requestUrl, AssignmentsRequest.class);
@@ -145,7 +145,7 @@ public class AssignmentsHandler {
             return;
         }
 
-        logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " tasks.");
+        logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " assignments.");
 
         Instant startTime = Instant.now();
 
@@ -154,9 +154,8 @@ public class AssignmentsHandler {
 
         assignments = getAssignmentsSpacedOutByDomain(assignments, assignmentsSize, false);
 
-        // Iterate over the tasks and add each task in its own list depending on the DATASOURCE in order to decide which plugin to use later.
+        // Iterate over the assignments and add each assignment in its own list depending on the DATASOURCE in order to decide which plugin to use later.
         for ( Assignment assignment : assignments ) {
-            // Add each task in its own HashSet.
             try {
                 assignmentsForPlugins.put(assignment.getDatasource().getId(), assignment);
             } catch (NullPointerException npe) {
@@ -166,12 +165,12 @@ public class AssignmentsHandler {
 
         //countDatasourcesAndRecords(assignmentsSize);    // Only for DEBUG! Keep it commented in normal run.
 
-        // TODO - Decide which tasks run with what plugin (depending on their datasource).
-        // First run -in parallel- the tasks which require some specific plugin.
-        // Then, after the above plugins are finished, run the remaining tasks in the generic plugin (which handles parallelism itself).
+        // TODO - Decide which assignments should run with what plugin (depending on their datasource).
+        // First run -in parallel- the assignments which require some specific plugin.
+        // Then, after the above plugins are finished, run the remaining assignments in the generic plugin (which handles parallelism itself).
         // TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe.
 
-        // For now, let's just run all tasks in the generic plugin.
+        // For now, let's just run all assignments in the generic plugin.
         try {
             publicationsRetrieverPlugin.processAssignments(assignmentRequestCounter, assignmentsForPlugins.values());
         } catch (Exception e) {

src/main/java/eu/openaire/urls_worker/payloads/responces/WorkerReport.java

@@ -59,7 +59,7 @@ public class WorkerReport {
     }
 
     public String getJsonReport() {
-        return gson.toJson(this);
+        return gson.toJson(this, WorkerReport.class);
     }