lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

Reduce memory usage.

This commit is contained in:
Lampros Smyrnaios 2023-04-28 17:59:36 +03:00
parent 0ba15dd31a
commit ec4d084972
2 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
View File

@ -83,7 +83,7 @@ public class FullTextsController {
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zstdName + "\"")
.body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(zstdTarFileFullPath)), FilesCompressor.tenMb)));
.body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(zstdTarFileFullPath)), FilesCompressor.bufferSize)));
} catch (Exception e) {
String errorMsg = "Could not load the FileInputStream of the zstd-tar-file \"" + zstdTarFileFullPath + "\"!";
logger.error(errorMsg, e);
@ -115,7 +115,7 @@ public class FullTextsController {
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + file.getName() + "\"")
.body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(fullTextFileFullPath)), FilesCompressor.tenMb)));
.body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(fullTextFileFullPath)), FilesCompressor.bufferSize)));
} catch (Exception e) {
String errorMsg = "Could not load the FileInputStream of the full-text-file \"" + fullTextFileFullPath + "\"!";
logger.error(errorMsg, e);

12
src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java
View File

@ -19,7 +19,7 @@ public class FilesCompressor {
private static final Logger logger = LoggerFactory.getLogger(FilesCompressor.class);
public static final int tenMb = (10 * 1_048_576);
public static final int bufferSize = (5 * 1_048_576); // 5 Mb
public static File compressMultipleFilesIntoOne(long assignmentsCounter, int tarBatchCounter, List<String> filesToCompress, String baseDirectory)
@ -37,8 +37,8 @@ public class FilesCompressor {
String zStandardFileFullPath = tarFilePath + ".zstd";
File zStandardFile = new File(zStandardFileFullPath);
try ( BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Paths.get(tarFilePath)), tenMb);
ZstdCompressorOutputStream zOut = new ZstdCompressorOutputStream(new BufferedOutputStream(Files.newOutputStream(zStandardFile.toPath())), tenMb) )
try ( BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Paths.get(tarFilePath)), bufferSize);
ZstdCompressorOutputStream zOut = new ZstdCompressorOutputStream(new BufferedOutputStream(Files.newOutputStream(zStandardFile.toPath())), bufferSize) )
{
int readByte;
while ( (readByte = in.read()) != -1 ) {
@ -67,7 +67,7 @@ public class FilesCompressor {
int numTarredFiles = 0;
File tarFile = new File(tarFileFullPath);
try ( TarArchiveOutputStream taos = new TarArchiveOutputStream(new BufferedOutputStream(Files.newOutputStream(tarFile.toPath()), tenMb)) )
try ( TarArchiveOutputStream taos = new TarArchiveOutputStream(new BufferedOutputStream(Files.newOutputStream(tarFile.toPath()), bufferSize)) )
{
for ( String fileName : filesToTar ) {
if ( addTarEntry(taos, fileName, baseDir) )
@ -88,10 +88,10 @@ public class FilesCompressor {
private static boolean addTarEntry(TarArchiveOutputStream taos, String fileName, String baseDir)
{
boolean shouldCloseEntry = false; // Useful in order to know if we should close the entry (an Exception may appear, and so we should not try to close it).
boolean shouldCloseEntry = false; // Useful in order to know if we should close the entry (an Exception may appear when initializing the stream, and so we should not try to close it).
Path fullFileNamePath = Paths.get(baseDir + fileName);
try ( BufferedInputStream fis = new BufferedInputStream(Files.newInputStream(fullFileNamePath), tenMb) )
try ( BufferedInputStream fis = new BufferedInputStream(Files.newInputStream(fullFileNamePath), bufferSize) )
{
TarArchiveEntry entry = new TarArchiveEntry(fileName);
entry.setSize(Files.size(fullFileNamePath)); // Yes, tar requires that we set the size beforehand..