- Improve Zip-file delivery and significantly decrease memory consumption, by streaming it, instead of loading the whole file in memory before sending it to the Controller.

- Fix not closing the inputStream of the zip-file.
- Count and show the number of files which were zipped in each batch.

src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java

@@ -13,9 +13,9 @@ import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
 import javax.servlet.http.HttpServletRequest;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.util.HashMap;
 import java.util.List;
@@ -75,9 +75,9 @@ public class FullTextsController {
 
         String zipName = zipFile.getName();
         String zipFileFullPath = currentAssignmentsBaseFullTextsPath + zipName;
-        ByteArrayOutputStream byteArrayOutputStream = this.fileStorageService.loadFileAsAStream(zipFileFullPath);
+        StreamingResponseBody streamingResponseBody = this.fileStorageService.loadFileAsAStream(zipFileFullPath);
-        if ( byteArrayOutputStream == null )
+        if ( streamingResponseBody == null )
-            return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Could not load file: " + zipFileFullPath);
+            return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Could not load zip-file: " + zipFileFullPath);
 
         // If this is the last batch for this assignments-count, then make sure it is deleted in the next scheduled delete-operation.
         if ( zipBatchCounter == totalZipBatches ) {
@@ -85,15 +85,11 @@ public class FullTextsController {
             logger.debug("Will return the last batch (" + zipBatchCounter + ") of Assignments_" + assignmentsCounter + " to the Controller and these assignments will be deleted later.");
         }
 
-        String contentType = request.getServletContext().getMimeType(zipFileFullPath);
-        if ( contentType == null )
-            contentType = "application/octet-stream";
-
         logger.info("Sending the zip file \"" + zipFileFullPath + "\".");
         return ResponseEntity.ok()
-                .contentType(MediaType.parseMediaType(contentType))
+                // Do not set any contentType here, Spring will handle it itself (otherwise it fails with "application/zip" and "application/octet-stream").
                 .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zipName + "\"")
-                .body(byteArrayOutputStream.toByteArray());
+                .body(streamingResponseBody);
     }
 
 
@@ -101,19 +97,19 @@ public class FullTextsController {
     public ResponseEntity<?> getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension, HttpServletRequest request) {
 
         logger.info("Received a \"getFullText\" request.");
-        String fullTextFile = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
+        String fullTextFileFullPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
-        File file = new File(fullTextFile);
+        File file = new File(fullTextFileFullPath);
         if ( !file.isFile() ) {
-            logger.error("The file \"" + fullTextFile + "\" does not exist!");
+            logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!");
             return ResponseEntity.notFound().build();
         }
 
-        Resource resource = this.fileStorageService.loadFileAsResource(fullTextFile);
+        Resource resource = this.fileStorageService.loadFileAsResource(fullTextFileFullPath);
         if ( resource == null )
-            return ResponseEntity.internalServerError().body("Could not load file: " + fullTextFile);
+            return ResponseEntity.internalServerError().body("Could not load file: " + fullTextFileFullPath);
 
         String contentType = null;
-        contentType = request.getServletContext().getMimeType(file.getAbsolutePath());
+        contentType = request.getServletContext().getMimeType(fullTextFileFullPath);
         if ( contentType == null ) {
             contentType = "application/octet-stream";
         }

src/main/java/eu/openaire/urls_worker/services/FileStorageService.java

@@ -7,10 +7,12 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.UrlResource;
 import org.springframework.stereotype.Service;
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
 import java.io.*;
 import java.nio.file.*;
 import java.util.Properties;
+import java.util.concurrent.atomic.AtomicReference;
 
 
 @Service
@@ -50,23 +52,35 @@ public class FileStorageService {
 
 
     private static final int bufferSize = 10485760;  // 10 MB
-    public ByteArrayOutputStream loadFileAsAStream(String fullFileName)
+    public StreamingResponseBody loadFileAsAStream(String fullFileName)
     {
-        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(bufferSize);
+        StreamingResponseBody streamingResponseBody = null;
+        AtomicReference<FileInputStream> fileInputStream = new AtomicReference<>();
         try {
-            FileInputStream fileInputStream = new FileInputStream(fullFileName);
+            streamingResponseBody = response -> {   // This does not need to be explicitly closed.
-            int bytesRead;
+                fileInputStream.set(new FileInputStream(fullFileName));
-            byte[] byteBuffer = new byte[bufferSize];
+                int bytesRead;
-            while ( (bytesRead = fileInputStream.read(byteBuffer, 0, bufferSize)) > 0 )
+                byte[] byteBuffer = new byte[bufferSize];
-                byteArrayOutputStream.write(byteBuffer, 0, bytesRead);
+                FileInputStream fInS = fileInputStream.get();
-
+                while ( (bytesRead = fInS.read(byteBuffer, 0, bufferSize)) > 0 )
-            return byteArrayOutputStream;
+                    response.write(byteBuffer, 0, bytesRead);
+            };
+            return streamingResponseBody;
         } catch (Exception e) {
             if ( e instanceof FileNotFoundException )
                 logger.error("File \"" + fullFileName + "\" is not a file!");
             else
                 logger.error(e.getMessage(), e);
             return null;
+        } finally {
+            FileInputStream fInS = fileInputStream.get();
+            if ( fInS != null ) {
+                try {
+                    fInS.close();
+                } catch (IOException ex) {
+                    logger.error("", ex);
+                }
+            }
         }
     }
 

src/main/java/eu/openaire/urls_worker/util/FilesZipper.java

@@ -20,16 +20,19 @@ public class FilesZipper
         String zipFilename = baseDirectory + "assignments_" + assignmentsCounter + "_full-texts_" + zipBatchCounter + ".zip";
         // For example: assignments_2_full-texts_4.zip    | where < 4 > is referred to the 4th batch of files requested by the controller.
 
+        int numZippedFiles = 0;
         File zipFile = new File(zipFilename);
         try ( ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile), StandardCharsets.UTF_8) )
         {
             for ( String file : filesToZip ) {
-                zipAFile(file, zos, baseDirectory);
+                if ( zipAFile(file, zos, baseDirectory) )
+                    numZippedFiles ++;
             }
         } catch (Exception e) {
             logger.error("Exception when creating the zip-file: " + zipFilename, e);
             return null;
         }
+        logger.debug("Zipped " + numZippedFiles + " files for assignments_" + assignmentsCounter + ", batch_" + zipBatchCounter);
         return zipFile;
     }