lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

- Code optimization and polishing. 

- Update dependencies.
This commit is contained in:
Lampros Smyrnaios 2023-08-28 16:11:26 +03:00
parent dc97b323c9
commit b579296ada
5 changed files with 19 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@ -1,6 +1,6 @@
plugins {
id 'org.springframework.boot' version '2.7.14'
id 'io.spring.dependency-management' version '1.1.2'
id 'org.springframework.boot' version '2.7.15'
id 'io.spring.dependency-management' version '1.1.3'
id 'java'
}
@ -39,7 +39,7 @@ dependencies {
exclude group: 'io.minio' // This is not used in the Worker, since it's the Controller which uploads the full-texts to S3. It also includes an older "commons-compress" version which causes problems.
}
implementation group: 'com.google.guava', name: 'guava', version: '32.1.1-jre'
implementation group: 'com.google.guava', name: 'guava', version: '32.1.2-jre'
// https://mvnrepository.com/artifact/com.google.code.gson/gson
implementation 'com.google.code.gson:gson:2.10.1'

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
validateDistributionUrl=true
networkTimeout=10000
zipStoreBase=GRADLE_USER_HOME

2
installAndRun.sh
View File

@ -22,7 +22,7 @@ elif [[ $# -gt 2 ]]; then
echo -e "Wrong number of arguments given: ${#} (more than 2)\nPlease execute it like: script.sh <justRun: 0 | 1> <avoidReInstallingPublicationsRetriever: 0 | 1>"; exit 2
fi
gradleVersion="8.2.1"
gradleVersion="8.3"
shouldBeCarefulWithMaxHeap=0 # This is NOT a cmd-arg.

2
src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java
View File

@ -178,7 +178,7 @@ public class AssignmentsHandler {
return;
} // In this case, no assignments were processed.
PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " finished after: ");
PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " (containing " + assignmentsSize + " assignments) finished after: ");
if ( askForTest ) {
logger.debug("UrlReports:"); // DEBUG!

22
src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java
View File

@ -39,13 +39,15 @@ public class ScheduledTasks {
private String workerReportsDirPath;
private static final File rootPath = new File("/");
private static final long oneAndHalfGB = 1_610_612_736L; // 1.5 GB free space per 1.000-assignments-batch.
private static final long oneAndHalfGB = 1_610_612_736L; // We need 1.5 GB free space per 1.000-assignments-batch.
private static long requiredFreeSpace;
private static final int oneMb = (1024 * 1024);
public ScheduledTasks(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch)
{
if ( maxAssignmentsLimitPerBatch < 1_000 )
if ( maxAssignmentsLimitPerBatch <= 1_000 )
requiredFreeSpace = oneAndHalfGB;
else
requiredFreeSpace = oneAndHalfGB * (maxAssignmentsLimitPerBatch / 1_000);
@ -54,14 +56,15 @@ public class ScheduledTasks {
}
@Scheduled(fixedDelay = 1) // Request the next batch immediately after the last one finishes.
public void handleNewAssignments() {
public void handleNewAssignments()
{
if ( GeneralController.shouldShutdownWorker || AssignmentsHandler.shouldNotRequestMore ) {
// Here we will be right after the Worker has posted its last report. It is guaranteed that the Controller will not have processed it and have not requested the full-text files.
// We do not want to shut down the controller.
return;
}
if ( rootPath.getFreeSpace() < requiredFreeSpace ) {
if ( rootPath.getFreeSpace() <= requiredFreeSpace ) {
// It's not safe to proceed with downloading more files and risk of "noSpaceLeft" error.
// Wait for the Controller to take the full-texts and any remaining files to be deleted, so that more free-space becomes available.
// We need to have some buffer zone for the ".tar" files which will be created from the already downloaded full-texts, when the Controller starts requesting them.
@ -69,7 +72,7 @@ public class ScheduledTasks {
AssignmentsHandler.shouldNotRequestMore = true;
return;
}
logger.warn("The free space is running out (less than " + (requiredFreeSpace / (1024 * 1024)) + " Mb). The Worker will avoid getting new assignments for the next 15 minutes.");
logger.warn("The free space is running out (less than " + (requiredFreeSpace / oneMb) + " Mb). The Worker will avoid getting new assignments for the next 15 minutes.");
try {
Thread.sleep(900_000); // Sleep for 15 mins to stall the scheduler from retrying right away, thus giving time to the disk-space to be freed.
} catch (InterruptedException ie) {
@ -104,7 +107,7 @@ public class ScheduledTasks {
return;
// Check if the full-texts have been delivered to the Controller.
// In case some files have been left behind due to an error. DO not shutdown, but wait for the other scheduled task to clean the in the right time and then shutdown.
// In case some files have been left behind due to an error. DO not shut down, but wait for the other scheduled task to clean the in the right time and then shutdown.
File fullTextsBaseDir = new File(fileStorageService.assignmentsBaseLocation);
if ( fullTextsBaseDir.isDirectory() ) {
@ -132,6 +135,8 @@ public class ScheduledTasks {
private static final double hoursToWaitBeforeDeletion = 36.0;
private static final int hoursDivisor = (1000 * 60 * 60); // In order to get the time-difference in hours. We divide with: /1000 to get seconds, /60 to get minutes and /60 to get hours.
@Scheduled(initialDelay = 21_600_000, fixedDelay = 21_600_000) // InitialDelay & FixedDelay = 36 hours.
//@Scheduled(initialDelay = 120_000, fixedDelay = 120_000) // Just for testing (every 2 mins).
public void checkAndDeleteOldFiles() {
@ -149,7 +154,6 @@ public class ScheduledTasks {
// Also, the ".tar.zstd" file of last batch will be deleted here, as well as the whole directory itself.
logger.debug("Going to check if any leftover full-texts exist and delete them.");
int usableDirsNum = 0;
try {
File fullTextsBaseDir = new File(fileStorageService.assignmentsBaseLocation);
@ -180,7 +184,7 @@ public class ScheduledTasks {
logger.trace("The subDir \"" + subDir.getName() + "\" was last accessed in: " + new Date(lastModified));
// Get the difference in hours. /1000 to get seconds, /60 to get minutes and /60 to get hours.
double elapsedHours = (double) (currentTime - lastModified) / (1000 * 60 * 60);
double elapsedHours = (double) (currentTime - lastModified) / hoursDivisor;
if ( elapsedHours > hoursToWaitBeforeDeletion ) {
// Enough time has passed, the directory should be deleted immediately.
String subDirName = subDir.getName();
@ -193,7 +197,7 @@ public class ScheduledTasks {
String assingmentsCounterString = matcher.group(1);
if ( (assingmentsCounterString != null) && !assingmentsCounterString.isEmpty()) {
if ( FullTextsController.deleteFile(this.workerReportsDirPath + this.workerId + "_assignments_" + assingmentsCounterString + "_report.json") )
logger.warn("The subDir \"" + subDirName + "\" probably contains some failed file, since the workerReport for assignments_" + assingmentsCounterString + " was deleted only now, which means the Controller failed to successfully process the results of those assignments.");
logger.warn("The subDir \"" + subDirName + "\" probably contains some failed files, since the workerReport for assignments_" + assingmentsCounterString + " was deleted only now, which means the Controller failed to successfully process the results of those assignments.");
}
else
logger.error("The subDir \"" + subDirName + "\" has an invalid name! It does not contains the assignmentsCounter!");