lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

- Show a warning, in case the number of archived files is different from the number of requested files. 

- Code polishing.
- Update Gradle.
This commit is contained in:
Lampros Smyrnaios 2023-03-07 16:25:10 +02:00
parent ec09ecc7ff
commit 4da54e7a7d
4 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
networkTimeout=10000
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

2
installAndRun.sh
View File

@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then
echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>"; exit 2
fi
gradleVersion="8.0.1"
gradleVersion="8.0.2"
if [[ justRun -eq 0 ]]; then

2
src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java
View File

@ -210,7 +210,7 @@ public class AssignmentsHandler {
/**
* Post the worker report and wait for the Controller to request the publication-files.
* Once the Controller finishes with uploading the files to the S3-ObjectStore, it returns an "HTTP-200-OK" response to the Worker.
* Afterwards, the Worker, even in case of an error, deletes the fulltext and zip files.
* Afterwards, the Worker, even in case of an error, deletes the full-texts and the ".tar" and ".tar.zstd" files.
* */
public boolean postWorkerReport(Long assignmentRequestCounter)
{

18
src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java
View File

@ -19,14 +19,14 @@ public class FilesCompressor {
private static final Logger logger = LoggerFactory.getLogger(FilesCompressor.class);
static final int tenMb = 10 * 1_048_576;
static final int tenMb = (10 * 1_048_576);
public static File compressMultipleFilesIntoOne(long assignmentsCounter, int zipBatchCounter, List<String> filesToCompress, String baseDirectory)
public static File compressMultipleFilesIntoOne(long assignmentsCounter, int tarBatchCounter, List<String> filesToCompress, String baseDirectory)
{
// For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the Controller.
// For example: assignments_2_full-texts_4.tar.zstd | where < 4 > is referred to the 4th batch of files requested by the Controller.
File tarFile = getTarArchiveWithFullTexts(filesToCompress, baseDirectory, assignmentsCounter, zipBatchCounter);
File tarFile = getTarArchiveWithFullTexts(filesToCompress, baseDirectory, assignmentsCounter, tarBatchCounter);
if ( tarFile == null )
return null; // The error-cause is already logged.
@ -49,6 +49,7 @@ public class FilesCompressor {
return null;
}
logger.debug("Finished archiving and compressing the full-texts of assignments_" + assignmentsCounter + ", batch_" + tarBatchCounter);
return zStandardFile;
}
@ -59,7 +60,7 @@ public class FilesCompressor {
private static File getTarArchiveWithFullTexts(List<String> filesToTar, String baseDir, long assignmentsCounter, int tarBatchCounter) {
String tarFileFullPath = baseDir + "assignments_" + assignmentsCounter + "_full-texts_" + tarBatchCounter + ".tar";
// For example: assignments_2_full-texts_4.zip | where < 4 > is referred to the 4th batch of files requested by the Controller.
// For example: assignments_2_full-texts_4.tar.zstd | where < 4 > is referred to the 4th batch of files requested by the Controller.
// https://commons.apache.org/proper/commons-compress/examples.html
@ -76,7 +77,10 @@ public class FilesCompressor {
logger.error("Exception when creating the tar-file: " + tarFileFullPath, e);
return null;
}
logger.debug("Tarred " + numTarredFiles + " (out of " + filesToTar.size() + ") files for assignments_" + assignmentsCounter + ", batch_" + tarBatchCounter);
if ( numTarredFiles != filesToTar.size() )
logger.warn("The number of \"numTarredFiles\" (" + numTarredFiles + ") is different from the number of files requested to be tarred (" + filesToTar.size() + "), for assignments_" + assignmentsCounter + ", batch_" + tarBatchCounter);
return tarFile;
}
@ -103,7 +107,7 @@ public class FilesCompressor {
} finally {
if ( shouldCloseEntry ) {
try {
taos.closeArchiveEntry(); // close just the ZipEntry here (not the ZipOutputStream)
taos.closeArchiveEntry(); // close just the TarEntry here (not the TarArchiveOutputStream)
} catch (IOException e) {
logger.error("", e);
}