lsmyrnaios
/
UrlsController
1
0
Fork 2
Code Issues Pull Requests Releases 18 Wiki Activity

- Automatically delete the unsuccessful WorkerReports, which are more than 7 days old. 

- Optimize the Service's startup speed, by setting "initialDelays" to the scheduled tasks.
- Optimize documentation.
This commit is contained in:
Lampros Smyrnaios 2023-05-24 16:59:42 +03:00
parent 551c4acef5
commit 164245cb53
2 changed files with 78 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java
View File

@ -4,13 +4,18 @@ import eu.openaire.urls_controller.Application;
import eu.openaire.urls_controller.controllers.ShutdownController;
import eu.openaire.urls_controller.controllers.UrlsController;
import eu.openaire.urls_controller.services.FullTextsServiceImpl;
import eu.openaire.urls_controller.util.FileUtils;
import eu.openaire.urls_controller.util.GenericUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
@ -23,9 +28,24 @@ public class ScheduledTasks {
private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
@Autowired
FileUtils fileUtils;
@Scheduled(fixedDelay = 3_600_000) // Execute this method 1 hour after the last execution, in order for some tasks to have been gathered.
//@Scheduled(fixedDelay = 20_000) // Just for testing (every 20 secs).
private final String workerReportsDirPath;
public ScheduledTasks(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath)
{
if ( !workerReportsDirPath.endsWith("/") )
workerReportsDirPath += "/";
this.workerReportsDirPath = workerReportsDirPath; // This dir will be created later.
}
@Scheduled(initialDelay = 3_600_000, fixedDelay = 3_600_000) // Execute this method 1 hour from the start and 1 hour after each last execution, in order for some tasks to have been gathered.
//@Scheduled(initialDelay = 3_600_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
public void executeBackgroundTasks()
{
List<Callable<Boolean>> tempList = new ArrayList<>(FullTextsServiceImpl.backgroundCallableTasks); // Copy the list in order to know what was executed and delete only that data later.
@ -60,8 +80,8 @@ public class ScheduledTasks {
}
@Scheduled(fixedDelay = 7_200_000) // Check every 2 hours.
//@Scheduled(fixedDelay = 20_000) // Just for testing (every 20 secs).
@Scheduled(initialDelay = 60_000, fixedDelay = 7_200_000) // Check every 2 hours.
//@Scheduled(initialDelay = 60_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
public void checkIfServiceIsReadyForShutdown()
{
if ( ! ShutdownController.shouldShutdownService )
@ -76,4 +96,56 @@ public class ScheduledTasks {
Application.gentleAppShutdown();
}
private static final int daysToWaitBeforeDeletion = 7;
@Scheduled(initialDelay = 120_000, fixedDelay = 604_800_000) // Run every 7 days.
//@Scheduled(initialDelay = 120_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
public void checkAndDeleteUnsuccessfulWorkerReports()
{
logger.debug("Going to check and remove any unsuccessful workerReports, which are more than 7 days old.");
int usableDirsNum = 0;
try {
File workerReportsDir = new File(workerReportsDirPath);
if ( !workerReportsDir.isDirectory() ) {
logger.error("The \"workerReportsDir\" (" + workerReportsDirPath + ") does not exist!"); // This base dir should always exist!
return;
}
File[] workerReports = workerReportsDir.listFiles(File::isFile);
if ( workerReports == null ) {
logger.error("There was an error when getting the subDirs of \"workerReportsDir\": " + workerReportsDir);
return;
}
usableDirsNum = workerReports.length;
if ( usableDirsNum == 0 ) {
logger.debug("The \"workerReportsDir\" is empty, so there is nothing to delete.");
return;
}
long currentTime = System.currentTimeMillis();
// Loop through the array and print only the directories
for ( File workerReport : workerReports ) {
long lastModified = workerReport.lastModified();
if ( logger.isTraceEnabled() )
logger.trace("The workerReport \"" + workerReport.getName() + "\" was last accessed in: " + new Date(lastModified));
// Get the difference in hours. /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days.
long elapsedWeeks = (currentTime - lastModified) / (1000 * 60 * 60 * 24);
if ( elapsedWeeks > daysToWaitBeforeDeletion ) {
// Enough time has passed, the directory should be deleted immediately.
String workerReportName = workerReport.getName();
logger.warn("The workerReport \"" + workerReportName + "\" was accessed " + elapsedWeeks + " days ago (passed the " + daysToWaitBeforeDeletion + " days limit) and will be deleted.");
fileUtils.deleteFile(workerReport.getAbsolutePath());
}
}
} catch (Exception e) {
logger.error("", e);
}
}
}

3
src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java
View File

@ -46,7 +46,7 @@ public class UrlsController {
public static final ConcurrentHashMap<String, WorkerInfo> workersInfoMap = new ConcurrentHashMap<>(6);
private String workerReportsDirPath;
private final String workerReportsDirPath;
public UrlsController(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath)
@ -174,6 +174,7 @@ public class UrlsController {
// Create the report file and write the worker-report to it.
String workerReportFile = currentWorkerReportLocationDir + "/" + curWorkerId + "_assignments_" + curReportAssignmentsCounter + "_report.json";
// In case a file with the same name already exists (e.g. from a previous run of the Service), then it will be overwritten.
logger.debug("Going to write the worker report to json-file: \"" + workerReportFile + "\".");
fileUtils.writeToFile(workerReportFile, workerReport.getJsonReport(), false); // Only one thread is writing to this specific file.
// The above method will overwrite a possibly existing file. So in case of a crash, it's better to back up the reports before starting the Controller again (as the assignments-counter will start over, from 0).