lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

- Improve elapsed time precision for the "lastModified" metadata of the assignments-fulltext subDirectories. 

- Code polishing.
This commit is contained in:
Lampros Smyrnaios 2023-05-25 00:37:44 +03:00
parent 1bf27a5a4e
commit f57314908a
2 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java
View File

@ -66,6 +66,7 @@ public class ScheduledTasks {
@Scheduled(initialDelay = 900_000, fixedDelay = 1_800_000) // InitialDelay = 15 mins, FixedDelay = 30 mins.
//@Scheduled(initialDelay = 20_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
public void checkIfShouldShutdown()
{
if ( !GeneralController.shouldShutdownWorker && !AssignmentsHandler.shouldNotRequestMore )
@ -79,15 +80,16 @@ public class ScheduledTasks {
File[] fulltextSubDirs = fullTextsBaseDir.listFiles(File::isDirectory);
if ( fulltextSubDirs == null ) {
logger.error("There was an error when getting the subDirs of \"fullTextsBaseDir\": " + fullTextsBaseDir);
return;
return; // It's NOT safe to shut down.
}
if ( fulltextSubDirs.length > 0 ) {
logger.warn("The base full-texts directory still has sub-directories with full-texts, wait for the Controller to take all the files or some time to past before they are deleted. Then the Worker will shut down.");
logger.warn("The base full-texts directory still has sub-directories with full-texts, wait for the Controller to take all the files, or wait some time to past before they are deleted. Then the Worker will shut down.");
// Some subDirs may be left behind due to some error when processing the WorkerReport. In that case,
return;
} else
logger.debug("The \"fullTextsBaseDir\" is empty. Shutting down..");
} else
logger.warn("The base full-texts directory was not found Shutting down..");
logger.warn("The base full-texts directory was not found! Shutting down.."); // This base-directory should exist during run-time, but we can proceed with shutting down the Service.
connWithController.postShutdownReportToController(workerId);
UrlsWorkerApplication.gentleAppShutdown();
@ -97,9 +99,10 @@ public class ScheduledTasks {
private static final Pattern ASSIGNMENTS_COUNTER = Pattern.compile(".*assignments_([\\d]+).*");
private static final int hoursToWaitBeforeDeletion = 36;
private static final double hoursToWaitBeforeDeletion = 36.0;
@Scheduled(initialDelay = 21_600_000, fixedDelay = 21_600_000) // InitialDelay & FixedDelay = 36 hours.
//@Scheduled(initialDelay = 20_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
public void checkAndDeleteOldFiles() {
// For any reason the Worker-report connection with the Controller may fail, but the Controller will continue requesting the full-text batches.
@ -146,7 +149,7 @@ public class ScheduledTasks {
logger.trace("The subDir \"" + subDir.getName() + "\" was last accessed in: " + new Date(lastModified));
// Get the difference in hours. /1000 to get seconds, /60 to get minutes and /60 to get hours.
long elapsedHours = (currentTime - lastModified) / (1000 * 60 * 60);
double elapsedHours = (double) (currentTime - lastModified) / (1000 * 60 * 60);
if ( elapsedHours > hoursToWaitBeforeDeletion ) {
// Enough time has passed, the directory should be deleted immediately.
String subDirName = subDir.getName();

2
src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
View File

@ -56,7 +56,7 @@ public class FullTextsController {
return ResponseEntity.badRequest().body(errorMsg);
}
logger.info("Received a \"getFullTexts\" request for returning a tar-file containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ").");
logger.info("Received a \"getFullTexts\" request for returning a \".tar.zstd\" file, containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ").");
String currentAssignmentsBaseFullTextsPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator;