lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

- Code optimization and polishing. 

- Update dependencies.
This commit is contained in:
Lampros Smyrnaios 2023-08-28 16:11:26 +03:00
parent dc97b323c9
commit b579296ada
5 changed files with 19 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@ -1,6 +1,6 @@
plugins {
id 'org.springframework.boot' version '2.7.14'
id 'io.spring.dependency-management' version '1.1.2'
id 'org.springframework.boot' version '2.7.15'
id 'io.spring.dependency-management' version '1.1.3'
id 'java'
}
@ -39,7 +39,7 @@ dependencies {
exclude group: 'io.minio' // This is not used in the Worker, since it's the Controller which uploads the full-texts to S3. It also includes an older "commons-compress" version which causes problems.
}
implementation group: 'com.google.guava', name: 'guava', version: '32.1.1-jre'
implementation group: 'com.google.guava', name: 'guava', version: '32.1.2-jre'
// https://mvnrepository.com/artifact/com.google.code.gson/gson
implementation 'com.google.code.gson:gson:2.10.1'

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
validateDistributionUrl=true
networkTimeout=10000
zipStoreBase=GRADLE_USER_HOME

2
installAndRun.sh
View File

@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then
echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>"; exit 2
fi
gradleVersion="8.2.1"
gradleVersion="8.3"
shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.

2
src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java
View File

@ -178,7 +178,7 @@ public class AssignmentsHandler {
return;
} // In this case, no assignments were processed.
PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " finished after: ");
PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " (containing " + assignmentsSize + " assignments) finished after: ");
if ( askForTest ) {
logger.debug("UrlReports:"); // DEBUG!

20
src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java
View File

@ -39,13 +39,15 @@ public class ScheduledTasks {
private String workerReportsDirPath;
private static final File rootPath = new File("/");
private static final long oneAndHalfGB = 1_610_612_736L; // 1.5 GB free space per 1.000-assignments-batch.
private static final long oneAndHalfGB = 1_610_612_736L; // We need 1.5 GB free space per 1.000-assignments-batch.
private static long requiredFreeSpace;
private static final int oneMb = (1024 * 1024);
public ScheduledTasks(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch)
{
if ( maxAssignmentsLimitPerBatch < 1_000 )
if ( maxAssignmentsLimitPerBatch <= 1_000 )
requiredFreeSpace = oneAndHalfGB;
else
requiredFreeSpace = oneAndHalfGB * (maxAssignmentsLimitPerBatch / 1_000);
@ -54,14 +56,15 @@ public class ScheduledTasks {
}
@Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes.
public void handleNewAssignments() {
public void handleNewAssignments()
{
if ( GeneralController.shouldShutdownWorker || AssignmentsHandler.shouldNotRequestMore ) {
// Here we will be right after the Worker has posted its last report. It is guaranteed that the Controller will not have processed it and have not requested the full-text files.
// We do not want to shut down the controller.
return;
}
if ( rootPath.getFreeSpace() < requiredFreeSpace ) {
if ( rootPath.getFreeSpace() <= requiredFreeSpace ) {
// It's not safe to proceed with downloading more files and risk of "noSpaceLeft" error.
// Wait for the Controller to take the full-texts and any remaining files to be deleted, so that more free-space becomes available.
// We need to have some buffer zone for the ".tar" files which will be created from the already downloaded full-texts, when the Controller starts requesting them.
@ -69,7 +72,7 @@ public class ScheduledTasks {
AssignmentsHandler.shouldNotRequestMore = true;
return;
}
logger.warn("The free space is running out (less than " + (requiredFreeSpace / (1024 * 1024)) + " Mb). The Worker will avoid getting new assignments for the next 15 minutes.");
logger.warn("The free space is running out (less than " + (requiredFreeSpace / oneMb) + " Mb). The Worker will avoid getting new assignments for the next 15 minutes.");
try {
Thread.sleep(900_000); // Sleep for 15 mins to stall the scheduler from retrying right away, thus giving time to the disk-space to be freed.
} catch (InterruptedException ie) {
@ -132,6 +135,8 @@ public class ScheduledTasks {
private static final double hoursToWaitBeforeDeletion = 36.0;
private static final int hoursDivisor = (1000 * 60 * 60); // In order to get the time-difference in hours. We divide with: /1000 to get seconds, /60 to get minutes and /60 to get hours.
@Scheduled(initialDelay = 21_600_000, fixedDelay = 21_600_000) // InitialDelay & FixedDelay = 36 hours.
//@Scheduled(initialDelay = 120_000, fixedDelay = 120_000) // Just for testing (every 2 mins).
public void checkAndDeleteOldFiles() {
@ -149,7 +154,6 @@ public class ScheduledTasks {
// Also, the ".tar.zstd" file of last batch will be deleted here, as well as the whole directory itself.
logger.debug("Going to check if any leftover full-texts exist and delete them.");
int usableDirsNum = 0;
try {
File fullTextsBaseDir = new File(fileStorageService.assignmentsBaseLocation);
@ -180,7 +184,7 @@ public class ScheduledTasks {
logger.trace("The subDir \"" + subDir.getName() + "\" was last accessed in: " + new Date(lastModified));
// Get the difference in hours. /1000 to get seconds, /60 to get minutes and /60 to get hours.
double elapsedHours = (double) (currentTime - lastModified) / (1000 * 60 * 60);
double elapsedHours = (double) (currentTime - lastModified) / hoursDivisor;
if ( elapsedHours > hoursToWaitBeforeDeletion ) {
// Enough time has passed, the directory should be deleted immediately.
String subDirName = subDir.getName();
@ -193,7 +197,7 @@ public class ScheduledTasks {
String assingmentsCounterString = matcher.group(1);
if ( (assingmentsCounterString != null) && !assingmentsCounterString.isEmpty()) {
if ( FullTextsController.deleteFile(this.workerReportsDirPath + this.workerId + "_assignments_" + assingmentsCounterString + "_report.json") )
logger.warn("The subDir \"" + subDirName + "\" probably contains some failed file, since the workerReport for assignments_" + assingmentsCounterString + " was deleted only now, which means the Controller failed to successfully process the results of those assignments.");
logger.warn("The subDir \"" + subDirName + "\" probably contains some failed files, since the workerReport for assignments_" + assingmentsCounterString + " was deleted only now, which means the Controller failed to successfully process the results of those assignments.");
}
else
logger.error("The subDir \"" + subDirName + "\" has an invalid name! It does not contains the assignmentsCounter!");