152 lines
7.3 KiB
Java
152 lines
7.3 KiB
Java
package eu.openaire.urls_controller.components;
|
|
|
|
import eu.openaire.urls_controller.Application;
|
|
import eu.openaire.urls_controller.controllers.ShutdownController;
|
|
import eu.openaire.urls_controller.controllers.UrlsController;
|
|
import eu.openaire.urls_controller.services.FullTextsServiceImpl;
|
|
import eu.openaire.urls_controller.util.FileUtils;
|
|
import eu.openaire.urls_controller.util.GenericUtils;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import java.io.File;
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.List;
|
|
import java.util.concurrent.Callable;
|
|
import java.util.concurrent.CancellationException;
|
|
import java.util.concurrent.ExecutionException;
|
|
import java.util.concurrent.Future;
|
|
|
|
|
|
@Component
|
|
public class ScheduledTasks {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class);
|
|
|
|
@Autowired
|
|
FileUtils fileUtils;
|
|
|
|
|
|
private final String workerReportsDirPath;
|
|
|
|
|
|
public ScheduledTasks(@Value("${services.pdfaggregation.controller.workerReportsDirPath}") String workerReportsDirPath)
|
|
{
|
|
if ( !workerReportsDirPath.endsWith("/") )
|
|
workerReportsDirPath += "/";
|
|
|
|
this.workerReportsDirPath = workerReportsDirPath; // This dir will be created later.
|
|
}
|
|
|
|
|
|
@Scheduled(initialDelay = 3_600_000, fixedDelay = 3_600_000) // Execute this method 1 hour from the start and 1 hour after each last execution, in order for some tasks to have been gathered.
|
|
//@Scheduled(initialDelay = 3_600_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
|
|
public void executeBackgroundTasks()
|
|
{
|
|
List<Callable<Boolean>> tempList = new ArrayList<>(FullTextsServiceImpl.backgroundCallableTasks); // Copy the list in order to know what was executed and delete only that data later.
|
|
// So the items added while this execution happens, will be left in the list, while the other will be deleted.
|
|
int numOfTasks = tempList.size(); // Since the temp-list is a deep-copy and not a reference, new tasks that are added will not be executed.
|
|
if ( numOfTasks == 0 )
|
|
return;
|
|
|
|
logger.debug(numOfTasks + " background tasks were found inside the \"backgroundCallableTasks\" list and are about to be executed.");
|
|
// Execute the tasks and wait for them to finish.
|
|
try {
|
|
List<Future<Boolean>> futures = FullTextsServiceImpl.backgroundExecutor.invokeAll(tempList);
|
|
int sizeOfFutures = futures.size();
|
|
for ( int i = 0; i < sizeOfFutures; ++i ) {
|
|
try {
|
|
Boolean value = futures.get(i).get(); // Get and see if an exception is thrown..
|
|
// Add check for the value, if wanted.. (we don't care at the moment)
|
|
} catch (ExecutionException ee) {
|
|
String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP).
|
|
logger.error("Task_" + (i+1) + " failed with: " + ee.getMessage() + "\n" + stackTraceMessage);
|
|
} catch (CancellationException ce) {
|
|
logger.error("Task_" + (i+1) + " was cancelled: " + ce.getMessage());
|
|
} catch (IndexOutOfBoundsException ioobe) {
|
|
logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage());
|
|
} finally {
|
|
FullTextsServiceImpl.backgroundCallableTasks.remove(tempList.get(i)); // Remove this object from the global list. Do not use indexes, since they will be different after each deletion and addition.
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
|
|
|
|
@Scheduled(initialDelay = 60_000, fixedDelay = 7_200_000) // Check every 2 hours.
|
|
//@Scheduled(initialDelay = 60_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
|
|
public void checkIfServiceIsReadyForShutdown()
|
|
{
|
|
if ( ! ShutdownController.shouldShutdownService )
|
|
return; // Either the service was never instructed to shut down, or the user canceled the request.
|
|
|
|
for ( String workerId : UrlsController.workersInfoMap.keySet() ) {
|
|
if ( ! UrlsController.workersInfoMap.get(workerId).getHasShutdown() ) // The workerId is certainly inside the map and has a workerInfo value.
|
|
return; // If at least 1 worker is still active, then do not shut down the server.
|
|
}
|
|
|
|
logger.info("All workers have already shutdown. Shutting down the Controller..");
|
|
Application.gentleAppShutdown();
|
|
}
|
|
|
|
|
|
private static final int daysToWaitBeforeDeletion = 7;
|
|
|
|
|
|
@Scheduled(initialDelay = 120_000, fixedDelay = 604_800_000) // Run every 7 days.
|
|
//@Scheduled(initialDelay = 120_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
|
|
public void checkAndDeleteUnsuccessfulWorkerReports()
|
|
{
|
|
logger.debug("Going to check and remove any unsuccessful workerReports, which are more than 7 days old.");
|
|
int usableDirsNum = 0;
|
|
try {
|
|
File workerReportsDir = new File(workerReportsDirPath);
|
|
if ( !workerReportsDir.isDirectory() ) {
|
|
logger.error("The \"workerReportsDir\" (" + workerReportsDirPath + ") does not exist!"); // This base dir should always exist!
|
|
return;
|
|
}
|
|
|
|
File[] workerReports = workerReportsDir.listFiles(File::isFile);
|
|
if ( workerReports == null ) {
|
|
logger.error("There was an error when getting the subDirs of \"workerReportsDir\": " + workerReportsDir);
|
|
return;
|
|
}
|
|
|
|
usableDirsNum = workerReports.length;
|
|
if ( usableDirsNum == 0 ) {
|
|
logger.debug("The \"workerReportsDir\" is empty, so there is nothing to delete.");
|
|
return;
|
|
}
|
|
|
|
long currentTime = System.currentTimeMillis();
|
|
|
|
// Loop through the array and print only the directories
|
|
for ( File workerReport : workerReports ) {
|
|
long lastModified = workerReport.lastModified();
|
|
|
|
if ( logger.isTraceEnabled() )
|
|
logger.trace("The workerReport \"" + workerReport.getName() + "\" was last accessed in: " + new Date(lastModified));
|
|
|
|
// Get the difference in hours. /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days.
|
|
long elapsedWeeks = (currentTime - lastModified) / (1000 * 60 * 60 * 24);
|
|
if ( elapsedWeeks > daysToWaitBeforeDeletion ) {
|
|
// Enough time has passed, the directory should be deleted immediately.
|
|
String workerReportName = workerReport.getName();
|
|
logger.warn("The workerReport \"" + workerReportName + "\" was accessed " + elapsedWeeks + " days ago (passed the " + daysToWaitBeforeDeletion + " days limit) and will be deleted.");
|
|
fileUtils.deleteFile(workerReport.getAbsolutePath());
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
logger.error("", e);
|
|
}
|
|
}
|
|
|
|
}
|