From ce49bff50ee3816ea2336c97edb12073747d9028 Mon Sep 17 00:00:00 2001
From: LSmyrnaios <lsmyrnaios@gmail.com>
Date: Fri, 3 Dec 2021 16:29:16 +0200
Subject: [PATCH] - Reduce memory consumption when loading a zipFile. - Check
 whether the "zipBatchCounter" is larger than the "totalZipBatches". - Improve
 the "failed tasks" log-message.

---
 .../urls_worker/controllers/FullTextsController.java        | 6 ++++++
 .../urls_worker/plugins/PublicationsRetrieverPlugin.java    | 2 +-
 .../openaire/urls_worker/services/FileStorageService.java   | 4 ++--
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
index 8a97663..b7930b9 100644
--- a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
+++ b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
@@ -60,6 +60,12 @@ public class FullTextsController {
             return ResponseEntity.badRequest().body(errorMsg);
         }
 
+        if ( zipBatchCounter > totalZipBatches ) {
+            String errorMsg = "The given \"zipBatchCounter\" (" + zipBatchCounter + ") is greater than the \"totalZipBatches\" (" + totalZipBatches + ")!";
+            logger.error(errorMsg);
+            return ResponseEntity.badRequest().body(errorMsg);
+        }
+
         File zipFile = FilesZipper.zipMultipleFilesAndGetZip(assignmentsCounter, zipBatchCounter, fileNamesWithExtensions, currentAssignmentsBaseFullTextsPath);
         if ( zipFile == null ) {
             String errorMsg = "Failed to create the zip file for \"zipBatchCounter\"-" + zipBatchCounter;
diff --git a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
index 8065a2e..0a881a6 100644
--- a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
+++ b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
@@ -127,7 +127,7 @@ public class PublicationsRetrieverPlugin {
 
         int numFailedTasks = LoaderAndChecker.invokeAllTasksAndWait(callableTasks);
         if ( numFailedTasks > 0 )
-            logger.warn(numFailedTasks + " tasks failed!");
+            logger.warn(numFailedTasks + " tasks failed, from assignments_" + assignmentRequestCounter);
         addUrlReportsToWorkerReport();
         callableTasks.clear(); // Reset the thread-tasks-list for the next batch.
     }
diff --git a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java
index ffe422a..29a538f 100644
--- a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java
+++ b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java
@@ -49,10 +49,10 @@ public class FileStorageService {
     }
 
 
-    private static final int bufferSize = 20971520;
+    private static final int bufferSize = 10485760;  // 10 MB
     public ByteArrayOutputStream loadFileAsAStream(String fullFileName)
     {
-        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(bufferSize);  // 20 MB
+        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(bufferSize);
         try {
             FileInputStream fileInputStream = new FileInputStream(fullFileName);
             int bytesRead;