lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 16 Wiki Activity

- Improve Zip-file delivery and significantly decrease memory consumption, by streaming it, instead of loading the whole file in memory before sending it to the Controller. 

- Fix not closing the inputStream of the zip-file.
- Count and show the number of files which were zipped in each batch.
This commit is contained in:
Lampros Smyrnaios 2021-12-13 15:29:03 +02:00
parent ab5e04698c
commit 859f850f56
3 changed files with 39 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
View File

@ -13,9 +13,9 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import javax.servlet.http.HttpServletRequest;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.util.HashMap;
import java.util.List;
@ -75,9 +75,9 @@ public class FullTextsController {
String zipName = zipFile.getName();
String zipFileFullPath = currentAssignmentsBaseFullTextsPath + zipName;
ByteArrayOutputStream byteArrayOutputStream = this.fileStorageService.loadFileAsAStream(zipFileFullPath);
if ( byteArrayOutputStream == null )
return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Could not load file: " + zipFileFullPath);
StreamingResponseBody streamingResponseBody = this.fileStorageService.loadFileAsAStream(zipFileFullPath);
if ( streamingResponseBody == null )
return ResponseEntity.status(HttpStatus.NOT_FOUND).body("Could not load zip-file: " + zipFileFullPath);
// If this is the last batch for this assignments-count, then make sure it is deleted in the next scheduled delete-operation.
if ( zipBatchCounter == totalZipBatches ) {
@ -85,15 +85,11 @@ public class FullTextsController {
logger.debug("Will return the last batch (" + zipBatchCounter + ") of Assignments_" + assignmentsCounter + " to the Controller and these assignments will be deleted later.");
}
String contentType = request.getServletContext().getMimeType(zipFileFullPath);
if ( contentType == null )
contentType = "application/octet-stream";
logger.info("Sending the zip file \"" + zipFileFullPath + "\".");
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(contentType))
// Do not set any contentType here, Spring will handle it itself (otherwise it fails with "application/zip" and "application/octet-stream").
.header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zipName + "\"")
.body(byteArrayOutputStream.toByteArray());
.body(streamingResponseBody);
}
@ -101,19 +97,19 @@ public class FullTextsController {
public ResponseEntity<?> getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension, HttpServletRequest request) {
logger.info("Received a \"getFullText\" request.");
String fullTextFile = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
File file = new File(fullTextFile);
String fullTextFileFullPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
File file = new File(fullTextFileFullPath);
if ( !file.isFile() ) {
logger.error("The file \"" + fullTextFile + "\" does not exist!");
logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!");
return ResponseEntity.notFound().build();
}
Resource resource = this.fileStorageService.loadFileAsResource(fullTextFile);
Resource resource = this.fileStorageService.loadFileAsResource(fullTextFileFullPath);
if ( resource == null )
return ResponseEntity.internalServerError().body("Could not load file: " + fullTextFile);
return ResponseEntity.internalServerError().body("Could not load file: " + fullTextFileFullPath);
String contentType = null;
contentType = request.getServletContext().getMimeType(file.getAbsolutePath());
contentType = request.getServletContext().getMimeType(fullTextFileFullPath);
if ( contentType == null ) {
contentType = "application/octet-stream";
}

28
src/main/java/eu/openaire/urls_worker/services/FileStorageService.java
View File

@ -7,10 +7,12 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.stereotype.Service;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.io.*;
import java.nio.file.*;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
@Service
@ -50,23 +52,35 @@ public class FileStorageService {
private static final int bufferSize = 10485760; // 10 MB
public ByteArrayOutputStream loadFileAsAStream(String fullFileName)
public StreamingResponseBody loadFileAsAStream(String fullFileName)
{
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(bufferSize);
StreamingResponseBody streamingResponseBody = null;
AtomicReference<FileInputStream> fileInputStream = new AtomicReference<>();
try {
FileInputStream fileInputStream = new FileInputStream(fullFileName);
streamingResponseBody = response -> { // This does not need to be explicitly closed.
fileInputStream.set(new FileInputStream(fullFileName));
int bytesRead;
byte[] byteBuffer = new byte[bufferSize];
while ( (bytesRead = fileInputStream.read(byteBuffer, 0, bufferSize)) > 0 )
byteArrayOutputStream.write(byteBuffer, 0, bytesRead);
return byteArrayOutputStream;
FileInputStream fInS = fileInputStream.get();
while ( (bytesRead = fInS.read(byteBuffer, 0, bufferSize)) > 0 )
response.write(byteBuffer, 0, bytesRead);
};
return streamingResponseBody;
} catch (Exception e) {
if ( e instanceof FileNotFoundException )
logger.error("File \"" + fullFileName + "\" is not a file!");
else
logger.error(e.getMessage(), e);
return null;
} finally {
FileInputStream fInS = fileInputStream.get();
if ( fInS != null ) {
try {
fInS.close();
} catch (IOException ex) {
logger.error("", ex);
}
}
}
}

5
src/main/java/eu/openaire/urls_worker/util/FilesZipper.java
View File

@ -20,16 +20,19 @@ public class FilesZipper
String zipFilename = baseDirectory + "assignments_" + assignmentsCounter + "_full-texts_" + zipBatchCounter + ".zip";
// For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the controller.
int numZippedFiles = 0;
File zipFile = new File(zipFilename);
try ( ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile), StandardCharsets.UTF_8) )
{
for ( String file : filesToZip ) {
zipAFile(file, zos, baseDirectory);
if ( zipAFile(file, zos, baseDirectory) )
numZippedFiles ++;
}
} catch (Exception e) {
logger.error("Exception when creating the zip-file: " + zipFilename, e);
return null;
}
logger.debug("Zipped " + numZippedFiles + " files for assignments_" + assignmentsCounter + ", batch_" + zipBatchCounter);
return zipFile;
}