117 lines
4.8 KiB
Java
117 lines
4.8 KiB
Java
package eu.openaire.urls_worker.util;
|
|
|
|
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
|
|
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
|
|
import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import java.io.BufferedInputStream;
|
|
import java.io.BufferedOutputStream;
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.nio.file.Paths;
|
|
import java.util.List;
|
|
|
|
public class FilesCompressor {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(FilesCompressor.class);
|
|
|
|
static final int tenMb = 10 * 1_048_576;
|
|
|
|
|
|
public static File compressMultipleFilesIntoOne(long assignmentsCounter, int zipBatchCounter, List<String> filesToCompress, String baseDirectory)
|
|
{
|
|
// For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the Controller.
|
|
|
|
File tarFile = getTarArchiveWithFullTexts(filesToCompress, baseDirectory, assignmentsCounter, zipBatchCounter);
|
|
if ( tarFile == null )
|
|
return null; // The error-cause is already logged.
|
|
|
|
// The "TAR" archive is not compressed, but it helps deliver multiple full-texts with a single Stream.
|
|
// Then, we compress the archive, using Facebook's "ZStandard" algorithm, which delivers both high compression-rate and compression and decompression efficiency.
|
|
|
|
String tarFilePath = tarFile.getPath();
|
|
String zStandardFileFullPath = tarFilePath + ".zstd";
|
|
File zStandardFile = new File(zStandardFileFullPath);
|
|
|
|
try ( BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Paths.get(tarFilePath)), tenMb);
|
|
ZstdCompressorOutputStream zOut = new ZstdCompressorOutputStream(new BufferedOutputStream(Files.newOutputStream(zStandardFile.toPath())), tenMb) )
|
|
{
|
|
int readByte;
|
|
while ( (readByte = in.read()) != -1 ) {
|
|
zOut.write(readByte);
|
|
}
|
|
} catch (Exception e) {
|
|
logger.error("Exception when compressing the tar-archive: " + tarFilePath, e);
|
|
return null;
|
|
}
|
|
|
|
return zStandardFile;
|
|
}
|
|
|
|
|
|
/**
|
|
* This method adds the requested full-text file into a TAR archive, which later will be compressed.
|
|
* */
|
|
private static File getTarArchiveWithFullTexts(List<String> filesToTar, String baseDir, long assignmentsCounter, int tarBatchCounter) {
|
|
|
|
String tarFileFullPath = baseDir + "assignments_" + assignmentsCounter + "_full-texts_" + tarBatchCounter + ".tar";
|
|
// For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the Controller.
|
|
|
|
// https://commons.apache.org/proper/commons-compress/examples.html
|
|
|
|
int numTarredFiles = 0;
|
|
File tarFile = new File(tarFileFullPath);
|
|
|
|
try ( TarArchiveOutputStream taos = new TarArchiveOutputStream(new BufferedOutputStream(Files.newOutputStream(tarFile.toPath()), tenMb)) )
|
|
{
|
|
for ( String fileName : filesToTar ) {
|
|
if ( addTarEntry(taos, fileName, baseDir) )
|
|
numTarredFiles ++;
|
|
}
|
|
} catch (Exception e) {
|
|
logger.error("Exception when creating the tar-file: " + tarFileFullPath, e);
|
|
return null;
|
|
}
|
|
logger.debug("Tarred " + numTarredFiles + " (out of " + filesToTar.size() + ") files for assignments_" + assignmentsCounter + ", batch_" + tarBatchCounter);
|
|
return tarFile;
|
|
}
|
|
|
|
|
|
private static boolean addTarEntry(TarArchiveOutputStream taos, String fileName, String baseDir)
|
|
{
|
|
boolean shouldCloseEntry = false; // Useful in order to know if we should close the entry (an Exception may appear, and so we should not try to close it).
|
|
|
|
Path fullFileNamePath = Paths.get(baseDir + fileName);
|
|
try ( BufferedInputStream fis = new BufferedInputStream(Files.newInputStream(fullFileNamePath), tenMb) )
|
|
{
|
|
TarArchiveEntry entry = new TarArchiveEntry(fileName);
|
|
entry.setSize(Files.size(fullFileNamePath)); // Yes, tar requires that we set the size beforehand..
|
|
taos.putArchiveEntry(entry);
|
|
shouldCloseEntry = true;
|
|
|
|
int readByte;
|
|
while ( (readByte = fis.read()) != -1 ) {
|
|
taos.write(readByte);
|
|
}
|
|
} catch (Exception e) {
|
|
logger.error("", e);
|
|
return false;
|
|
} finally {
|
|
if ( shouldCloseEntry ) {
|
|
try {
|
|
taos.closeArchiveEntry(); // close just the ZipEntry here (not the ZipOutputStream)
|
|
} catch (IOException e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
}
|