diff --git a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java index 03ea547..1fcf4fd 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java @@ -13,9 +13,9 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; import javax.servlet.http.HttpServletRequest; -import java.io.ByteArrayOutputStream; import java.io.File; import java.util.HashMap; import java.util.List; @@ -75,9 +75,9 @@ public class FullTextsController { String zipName = zipFile.getName(); String zipFileFullPath = currentAssignmentsBaseFullTextsPath + zipName; - ByteArrayOutputStream byteArrayOutputStream = this.fileStorageService.loadFileAsAStream(zipFileFullPath); - if ( byteArrayOutputStream == null ) - return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Could not load file: " + zipFileFullPath); + StreamingResponseBody streamingResponseBody = this.fileStorageService.loadFileAsAStream(zipFileFullPath); + if ( streamingResponseBody == null ) + return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Could not load zip-file: " + zipFileFullPath); // If this is the last batch for this assignments-count, then make sure it is deleted in the next scheduled delete-operation. if ( zipBatchCounter == totalZipBatches ) { @@ -85,15 +85,11 @@ public class FullTextsController { logger.debug("Will return the last batch (" + zipBatchCounter + ") of Assignments_" + assignmentsCounter + " to the Controller and these assignments will be deleted later."); } - String contentType = request.getServletContext().getMimeType(zipFileFullPath); - if ( contentType == null ) - contentType = "application/octet-stream"; - logger.info("Sending the zip file \"" + zipFileFullPath + "\"."); return ResponseEntity.ok() - .contentType(MediaType.parseMediaType(contentType)) + // Do not set any contentType here, Spring will handle it itself (otherwise it fails with "application/zip" and "application/octet-stream"). .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zipName + "\"") - .body(byteArrayOutputStream.toByteArray()); + .body(streamingResponseBody); } @@ -101,19 +97,19 @@ public class FullTextsController { public ResponseEntity getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension, HttpServletRequest request) { logger.info("Received a \"getFullText\" request."); - String fullTextFile = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension; - File file = new File(fullTextFile); + String fullTextFileFullPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension; + File file = new File(fullTextFileFullPath); if ( !file.isFile() ) { - logger.error("The file \"" + fullTextFile + "\" does not exist!"); + logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!"); return ResponseEntity.notFound().build(); } - Resource resource = this.fileStorageService.loadFileAsResource(fullTextFile); + Resource resource = this.fileStorageService.loadFileAsResource(fullTextFileFullPath); if ( resource == null ) - return ResponseEntity.internalServerError().body("Could not load file: " + fullTextFile); + return ResponseEntity.internalServerError().body("Could not load file: " + fullTextFileFullPath); String contentType = null; - contentType = request.getServletContext().getMimeType(file.getAbsolutePath()); + contentType = request.getServletContext().getMimeType(fullTextFileFullPath); if ( contentType == null ) { contentType = "application/octet-stream"; } diff --git a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java index 29a538f..410daf8 100644 --- a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java +++ b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java @@ -7,10 +7,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.core.io.UrlResource; import org.springframework.stereotype.Service; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; import java.io.*; import java.nio.file.*; import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; @Service @@ -50,23 +52,35 @@ public class FileStorageService { private static final int bufferSize = 10485760; // 10 MB - public ByteArrayOutputStream loadFileAsAStream(String fullFileName) + public StreamingResponseBody loadFileAsAStream(String fullFileName) { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(bufferSize); + StreamingResponseBody streamingResponseBody = null; + AtomicReference fileInputStream = new AtomicReference<>(); try { - FileInputStream fileInputStream = new FileInputStream(fullFileName); - int bytesRead; - byte[] byteBuffer = new byte[bufferSize]; - while ( (bytesRead = fileInputStream.read(byteBuffer, 0, bufferSize)) > 0 ) - byteArrayOutputStream.write(byteBuffer, 0, bytesRead); - - return byteArrayOutputStream; + streamingResponseBody = response -> { // This does not need to be explicitly closed. + fileInputStream.set(new FileInputStream(fullFileName)); + int bytesRead; + byte[] byteBuffer = new byte[bufferSize]; + FileInputStream fInS = fileInputStream.get(); + while ( (bytesRead = fInS.read(byteBuffer, 0, bufferSize)) > 0 ) + response.write(byteBuffer, 0, bytesRead); + }; + return streamingResponseBody; } catch (Exception e) { if ( e instanceof FileNotFoundException ) logger.error("File \"" + fullFileName + "\" is not a file!"); else logger.error(e.getMessage(), e); return null; + } finally { + FileInputStream fInS = fileInputStream.get(); + if ( fInS != null ) { + try { + fInS.close(); + } catch (IOException ex) { + logger.error("", ex); + } + } } } diff --git a/src/main/java/eu/openaire/urls_worker/util/FilesZipper.java b/src/main/java/eu/openaire/urls_worker/util/FilesZipper.java index a1c6975..25a113d 100644 --- a/src/main/java/eu/openaire/urls_worker/util/FilesZipper.java +++ b/src/main/java/eu/openaire/urls_worker/util/FilesZipper.java @@ -20,16 +20,19 @@ public class FilesZipper String zipFilename = baseDirectory + "assignments_" + assignmentsCounter + "_full-texts_" + zipBatchCounter + ".zip"; // For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the controller. + int numZippedFiles = 0; File zipFile = new File(zipFilename); try ( ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile), StandardCharsets.UTF_8) ) { for ( String file : filesToZip ) { - zipAFile(file, zos, baseDirectory); + if ( zipAFile(file, zos, baseDirectory) ) + numZippedFiles ++; } } catch (Exception e) { logger.error("Exception when creating the zip-file: " + zipFilename, e); return null; } + logger.debug("Zipped " + numZippedFiles + " files for assignments_" + assignmentsCounter + ", batch_" + zipBatchCounter); return zipFile; }