diff --git a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java index 67d5b57..117c230 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java @@ -83,7 +83,7 @@ public class FullTextsController { return ResponseEntity.ok() .contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zstdName + "\"") - .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(zstdTarFileFullPath)), FilesCompressor.tenMb))); + .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(zstdTarFileFullPath)), FilesCompressor.bufferSize))); } catch (Exception e) { String errorMsg = "Could not load the FileInputStream of the zstd-tar-file \"" + zstdTarFileFullPath + "\"!"; logger.error(errorMsg, e); @@ -115,7 +115,7 @@ public class FullTextsController { return ResponseEntity.ok() .contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + file.getName() + "\"") - .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(fullTextFileFullPath)), FilesCompressor.tenMb))); + .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(fullTextFileFullPath)), FilesCompressor.bufferSize))); } catch (Exception e) { String errorMsg = "Could not load the FileInputStream of the full-text-file \"" + fullTextFileFullPath + "\"!"; logger.error(errorMsg, e); diff --git a/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java b/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java index f4a4b92..791ed88 100644 --- a/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java +++ b/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java @@ -19,7 +19,7 @@ public class FilesCompressor { private static final Logger logger = LoggerFactory.getLogger(FilesCompressor.class); - public static final int tenMb = (10 * 1_048_576); + public static final int bufferSize = (5 * 1_048_576); // 5 Mb public static File compressMultipleFilesIntoOne(long assignmentsCounter, int tarBatchCounter, List filesToCompress, String baseDirectory) @@ -37,8 +37,8 @@ public class FilesCompressor { String zStandardFileFullPath = tarFilePath + ".zstd"; File zStandardFile = new File(zStandardFileFullPath); - try ( BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Paths.get(tarFilePath)), tenMb); - ZstdCompressorOutputStream zOut = new ZstdCompressorOutputStream(new BufferedOutputStream(Files.newOutputStream(zStandardFile.toPath())), tenMb) ) + try ( BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Paths.get(tarFilePath)), bufferSize); + ZstdCompressorOutputStream zOut = new ZstdCompressorOutputStream(new BufferedOutputStream(Files.newOutputStream(zStandardFile.toPath())), bufferSize) ) { int readByte; while ( (readByte = in.read()) != -1 ) { @@ -67,7 +67,7 @@ public class FilesCompressor { int numTarredFiles = 0; File tarFile = new File(tarFileFullPath); - try ( TarArchiveOutputStream taos = new TarArchiveOutputStream(new BufferedOutputStream(Files.newOutputStream(tarFile.toPath()), tenMb)) ) + try ( TarArchiveOutputStream taos = new TarArchiveOutputStream(new BufferedOutputStream(Files.newOutputStream(tarFile.toPath()), bufferSize)) ) { for ( String fileName : filesToTar ) { if ( addTarEntry(taos, fileName, baseDir) ) @@ -88,10 +88,10 @@ public class FilesCompressor { private static boolean addTarEntry(TarArchiveOutputStream taos, String fileName, String baseDir) { - boolean shouldCloseEntry = false; // Useful in order to know if we should close the entry (an Exception may appear, and so we should not try to close it). + boolean shouldCloseEntry = false; // Useful in order to know if we should close the entry (an Exception may appear when initializing the stream, and so we should not try to close it). Path fullFileNamePath = Paths.get(baseDir + fileName); - try ( BufferedInputStream fis = new BufferedInputStream(Files.newInputStream(fullFileNamePath), tenMb) ) + try ( BufferedInputStream fis = new BufferedInputStream(Files.newInputStream(fullFileNamePath), bufferSize) ) { TarArchiveEntry entry = new TarArchiveEntry(fileName); entry.setSize(Files.size(fullFileNamePath)); // Yes, tar requires that we set the size beforehand..