- Decrease the time interval for the scheduled task "handleNewAssignments". This helps to reduce the "dead-time" between reporting the current assignments and requesting the new ones.

- Avoid a potential NPE when giving information about the received AssignmentRequest.
- Log and return, when the received assignments-list is empty.
- Improve some logging-messages.
- Update the logs' fileName and change the preferred appender to "File".
- Code cleanup.

src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java

@@ -23,8 +23,8 @@ public class ScheduledTasks {
         logger.info("Server is live! Time is now {}", dateFormat.format(new Date()));
     }
 
-    @Scheduled(fixedRate = 1800_000)   // Every 30 mins: 1800_000
-    public void handleAssignment() {
+    @Scheduled(fixedRate = 900_000)   // Every 15 mins: 900_000
+    public void handleNewAssignments() {
         if ( AssignmentHandler.isAvailableForWork )
             AssignmentHandler.handleAssignments();
         else

src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java

@@ -49,12 +49,11 @@ public class PublicationsRetrieverPlugin {
         LoaderAndChecker.retrieveDocuments = true;
         LoaderAndChecker.retrieveDatasets = false;
         FileUtils.shouldDownloadDocFiles = true;
-        FileUtils.shouldUploadFilesToS3 = true;
         FileUtils.docFileNameType = FileUtils.DocFileNameType.idName;
         PublicationsRetriever.targetUrlType = "docUrl";
 
-        if ( FileUtils.shouldUploadFilesToS3 )
-            new S3ObjectStoreMinIO();   // Check here on how to create the credentials-file: https://github.com/LSmyrnaios/PublicationsRetriever/blob/master/README.md
+        FileUtils.shouldUploadFilesToS3 = true;
+        new S3ObjectStoreMinIO();   // Check here on how to create the credentials-file: https://github.com/LSmyrnaios/PublicationsRetriever/blob/master/README.md
 
         int workerThreadsCount = Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier;
         logger.info("Use " + workerThreadsCount + " worker-threads.");
@@ -65,9 +64,7 @@ public class PublicationsRetrieverPlugin {
     public static void processAssignments(Long assignmentRequestCounter, Collection<Assignment> assignments) throws RuntimeException, FileNotFoundException
     {
         ConnSupportUtils.setKnownMimeTypes();
-
         FileUtils.storeDocFilesDir = assignmentsBaseFullTextsPath + "assignment_" + assignmentRequestCounter + "_fullTexts" + File.separator;  // It needs the last separator, because of how the docFiles are named and stored.
-
         FileUtils.setOutput(new FileOutputStream(assignmentsBasePath + "assignment_" + assignmentRequestCounter + "_generic_results.json"));
 
         int tasksNumber = assignments.size();

src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java

@@ -30,9 +30,7 @@ public class AssignmentHandler {
     private static final Logger logger = LoggerFactory.getLogger(AssignmentHandler.class);
 
     public static boolean isAvailableForWork = true;
-
     public static List<UrlReport> urlReports = null;
-
     private static final boolean askForTest = false; // Enable this only for testing.
 
     private static final Duration requestConnectTimeoutDuration = Duration.ofMinutes(1);   // 1 minute.
@@ -60,11 +58,15 @@ public class AssignmentHandler {
                 }
             }
 
+            /**
+             * This is the HTTP-error-handler.
+             * */
             @Override
             public void handleError(ClientHttpResponse response) throws IOException {
                 int responseCode = response.getRawStatusCode();
                 String statusText = response.getStatusText();
-                String additionalErrorMessage = ((!"".equals(statusText)) ? statusText : "The HTTP-response-code was: " + responseCode);
+                String extraBodyMessage = "\n" + response.getBody();
+                String additionalErrorMessage = ((!"".equals(statusText)) ? statusText + extraBodyMessage : "The HTTP-response-code was: " + responseCode + extraBodyMessage);
                 if ( (responseCode >= 500) && responseCode <= 599 )
                     logger.error("The Controller encountered a problem! " + additionalErrorMessage);
                 else if ( (responseCode >= 400) && (responseCode <= 499) )
@@ -98,9 +100,6 @@ public class AssignmentHandler {
         }
 
         //logger.debug(assignmentRequest.toString());   // DEBUG!
-
-        logger.info("AssignmentRequest < " + assignmentRequest.getAssignmentCounter() + " > was received and it's ready to be processed. It contains " + assignmentRequest.getAssignments().size() + " tasks.");
-
         return assignmentRequest;
     }
 
@@ -114,13 +113,20 @@ public class AssignmentHandler {
         }
 
         Long assignmentRequestCounter = assignmentRequest.getAssignmentCounter();
-
         List<Assignment> assignments = assignmentRequest.getAssignments();
         if ( assignments == null ) {
             logger.warn("The assignments were found to be null for assignmentRequestCounter = " + assignmentRequestCounter);
             return;
         }
 
+        int assignmentsSize = assignments.size();
+        if ( assignmentsSize == 0 ) {
+            logger.warn("The assignmentsSize was < 0 > for assignmentRequestCounter = " + assignmentRequestCounter);
+            return;
+        }
+
+        logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " tasks.");
+
         // Start handling the assignments, the worker is busy.
         isAvailableForWork = false;
 
@@ -167,7 +173,7 @@ public class AssignmentHandler {
     public static boolean postWorkerReport(Long assignmentRequestCounter)
     {
         String postUrl = UrlsWorkerApplication.controllerBaseUrl + "urls/addWorkerReport";
-        logger.info("Going to post the WorkerReport to the controller-server: " + postUrl);
+        logger.info("Going to post the WorkerReport of assignment_" + assignmentRequestCounter + " to the controller-server: " + postUrl);
 
         try {
             ResponseEntity<String> responseEntity = new RestTemplateBuilder().build().postForEntity(postUrl, new WorkerReport(UrlsWorkerApplication.workerId, assignmentRequestCounter, urlReports), String.class);

src/main/resources/logback-spring.xml

@@ -1,10 +1,10 @@
 <configuration debug="false">
 
     <appender name="File" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>logs/urls_worker.log</file>
+        <file>logs/UrlsWorker.log</file>
 
         <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-            <fileNamePattern>logs/urls_worker.%i.log.zip</fileNamePattern>
+            <fileNamePattern>logs/UrlsWorker.%i.log.zip</fileNamePattern>
         </rollingPolicy>
 
         <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
@@ -24,7 +24,7 @@
     </appender>
 
     <root level="debug">
-        <appender-ref ref="Console" />
+        <appender-ref ref="File" />
     </root>
 
 </configuration>