From 4da54e7a7ddebb81124b4b2b5154387d23581b58 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Tue, 7 Mar 2023 16:25:10 +0200 Subject: [PATCH] - Show a warning, in case the number of archived files is different from the number of requested files. - Code polishing. - Update Gradle. --- gradle/wrapper/gradle-wrapper.properties | 2 +- installAndRun.sh | 2 +- .../components/AssignmentsHandler.java | 2 +- .../urls_worker/util/FilesCompressor.java | 18 +++++++++++------- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index fc10b60..bdc9a83 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/installAndRun.sh b/installAndRun.sh index 85c14bd..263804f 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh "; exit 2 fi -gradleVersion="8.0.1" +gradleVersion="8.0.2" if [[ justRun -eq 0 ]]; then diff --git a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java index 6454c97..e555449 100644 --- a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java @@ -210,7 +210,7 @@ public class AssignmentsHandler { /** * Post the worker report and wait for the Controller to request the publication-files. * Once the Controller finishes with uploading the files to the S3-ObjectStore, it returns an "HTTP-200-OK" response to the Worker. - * Afterwards, the Worker, even in case of an error, deletes the fulltext and zip files. + * Afterwards, the Worker, even in case of an error, deletes the full-texts and the ".tar" and ".tar.zstd" files. * */ public boolean postWorkerReport(Long assignmentRequestCounter) { diff --git a/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java b/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java index 9ca1130..45daeee 100644 --- a/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java +++ b/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java @@ -19,14 +19,14 @@ public class FilesCompressor { private static final Logger logger = LoggerFactory.getLogger(FilesCompressor.class); - static final int tenMb = 10 * 1_048_576; + static final int tenMb = (10 * 1_048_576); - public static File compressMultipleFilesIntoOne(long assignmentsCounter, int zipBatchCounter, List filesToCompress, String baseDirectory) + public static File compressMultipleFilesIntoOne(long assignmentsCounter, int tarBatchCounter, List filesToCompress, String baseDirectory) { - // For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the Controller. + // For example: assignments_2_full-texts_4.tar.zstd | where < 4 > is referred to the 4th batch of files requested by the Controller. - File tarFile = getTarArchiveWithFullTexts(filesToCompress, baseDirectory, assignmentsCounter, zipBatchCounter); + File tarFile = getTarArchiveWithFullTexts(filesToCompress, baseDirectory, assignmentsCounter, tarBatchCounter); if ( tarFile == null ) return null; // The error-cause is already logged. @@ -49,6 +49,7 @@ public class FilesCompressor { return null; } + logger.debug("Finished archiving and compressing the full-texts of assignments_" + assignmentsCounter + ", batch_" + tarBatchCounter); return zStandardFile; } @@ -59,7 +60,7 @@ public class FilesCompressor { private static File getTarArchiveWithFullTexts(List filesToTar, String baseDir, long assignmentsCounter, int tarBatchCounter) { String tarFileFullPath = baseDir + "assignments_" + assignmentsCounter + "_full-texts_" + tarBatchCounter + ".tar"; - // For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the Controller. + // For example: assignments_2_full-texts_4.tar.zstd | where < 4 > is referred to the 4th batch of files requested by the Controller. // https://commons.apache.org/proper/commons-compress/examples.html @@ -76,7 +77,10 @@ public class FilesCompressor { logger.error("Exception when creating the tar-file: " + tarFileFullPath, e); return null; } - logger.debug("Tarred " + numTarredFiles + " (out of " + filesToTar.size() + ") files for assignments_" + assignmentsCounter + ", batch_" + tarBatchCounter); + + if ( numTarredFiles != filesToTar.size() ) + logger.warn("The number of \"numTarredFiles\" (" + numTarredFiles + ") is different from the number of files requested to be tarred (" + filesToTar.size() + "), for assignments_" + assignmentsCounter + ", batch_" + tarBatchCounter); + return tarFile; } @@ -103,7 +107,7 @@ public class FilesCompressor { } finally { if ( shouldCloseEntry ) { try { - taos.closeArchiveEntry(); // close just the ZipEntry here (not the ZipOutputStream) + taos.closeArchiveEntry(); // close just the TarEntry here (not the TarArchiveOutputStream) } catch (IOException e) { logger.error("", e); }