2021-03-16 14:25:15 +01:00
package eu.openaire.urls_controller.components ;
2023-05-24 12:42:29 +02:00
import eu.openaire.urls_controller.Application ;
import eu.openaire.urls_controller.controllers.ShutdownController ;
import eu.openaire.urls_controller.controllers.UrlsController ;
2023-05-11 02:07:55 +02:00
import eu.openaire.urls_controller.services.FullTextsServiceImpl ;
2023-05-24 15:59:42 +02:00
import eu.openaire.urls_controller.util.FileUtils ;
2023-05-11 02:07:55 +02:00
import eu.openaire.urls_controller.util.GenericUtils ;
2021-03-16 14:25:15 +01:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
2023-05-24 15:59:42 +02:00
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.beans.factory.annotation.Value ;
2023-05-11 02:07:55 +02:00
import org.springframework.scheduling.annotation.Scheduled ;
2021-03-16 14:25:15 +01:00
import org.springframework.stereotype.Component ;
2023-05-24 15:59:42 +02:00
import java.io.File ;
2023-05-11 02:07:55 +02:00
import java.util.ArrayList ;
2023-05-24 15:59:42 +02:00
import java.util.Date ;
2023-05-11 02:07:55 +02:00
import java.util.List ;
import java.util.concurrent.Callable ;
import java.util.concurrent.CancellationException ;
import java.util.concurrent.ExecutionException ;
import java.util.concurrent.Future ;
2021-03-16 14:25:15 +01:00
@Component
public class ScheduledTasks {
private static final Logger logger = LoggerFactory . getLogger ( ScheduledTasks . class ) ;
2023-05-24 15:59:42 +02:00
@Autowired
FileUtils fileUtils ;
2021-03-16 14:25:15 +01:00
2023-05-24 15:59:42 +02:00
private final String workerReportsDirPath ;
public ScheduledTasks ( @Value ( " ${services.pdfaggregation.controller.workerReportsDirPath} " ) String workerReportsDirPath )
{
if ( ! workerReportsDirPath . endsWith ( " / " ) )
workerReportsDirPath + = " / " ;
this . workerReportsDirPath = workerReportsDirPath ; // This dir will be created later.
}
2023-05-24 23:34:36 +02:00
@Scheduled ( initialDelay = 1_800_000 , fixedDelay = 1_800_000 ) // Execute this method 30 mins from the start and 30 mins after each last execution, in order for some tasks to have been gathered.
//@Scheduled(initialDelay = 20_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
2023-05-11 02:07:55 +02:00
public void executeBackgroundTasks ( )
{
2023-05-26 12:08:00 +02:00
List < Callable < Boolean > > tempList = new ArrayList < > ( FullTextsServiceImpl . backgroundCallableTasks ) ; // Copy the list in order to know what was executed.
// So the items added while this execution happens, will be remain in the global-list, while the other will have already be deleted.
2023-05-11 02:07:55 +02:00
int numOfTasks = tempList . size ( ) ; // Since the temp-list is a deep-copy and not a reference, new tasks that are added will not be executed.
if ( numOfTasks = = 0 )
return ;
2023-05-26 12:08:00 +02:00
// Immediately delete the selected tasks form the global list, so that if these tasks are not finished before the scheduler runs again, they will not be re-executed.
for ( Callable < Boolean > selectedTask : tempList ) {
FullTextsServiceImpl . backgroundCallableTasks . remove ( selectedTask ) ;
}
2023-05-11 02:07:55 +02:00
logger . debug ( numOfTasks + " background tasks were found inside the \" backgroundCallableTasks \" list and are about to be executed. " ) ;
// Execute the tasks and wait for them to finish.
try {
List < Future < Boolean > > futures = FullTextsServiceImpl . backgroundExecutor . invokeAll ( tempList ) ;
int sizeOfFutures = futures . size ( ) ;
for ( int i = 0 ; i < sizeOfFutures ; + + i ) {
try {
Boolean value = futures . get ( i ) . get ( ) ; // Get and see if an exception is thrown..
// Add check for the value, if wanted.. (we don't care at the moment)
} catch ( ExecutionException ee ) {
String stackTraceMessage = GenericUtils . getSelectiveStackTrace ( ee , null , 15 ) ; // These can be serious errors like an "out of memory exception" (Java HEAP).
logger . error ( " Task_ " + ( i + 1 ) + " failed with: " + ee . getMessage ( ) + " \ n " + stackTraceMessage ) ;
} catch ( CancellationException ce ) {
logger . error ( " Task_ " + ( i + 1 ) + " was cancelled: " + ce . getMessage ( ) ) ;
} catch ( IndexOutOfBoundsException ioobe ) {
logger . error ( " IOOBE for task_ " + i + " in the futures-list! " + ioobe . getMessage ( ) ) ;
}
}
} catch ( Exception e ) {
logger . error ( " " , e ) ;
}
2021-03-16 14:25:15 +01:00
}
2023-05-11 02:07:55 +02:00
2023-05-24 12:42:29 +02:00
2023-05-24 15:59:42 +02:00
@Scheduled ( initialDelay = 60_000 , fixedDelay = 7_200_000 ) // Check every 2 hours.
//@Scheduled(initialDelay = 60_000, fixedDelay = 20_000) // Just for testing (every 20 secs).
2023-05-24 12:42:29 +02:00
public void checkIfServiceIsReadyForShutdown ( )
{
if ( ! ShutdownController . shouldShutdownService )
return ; // Either the service was never instructed to shut down, or the user canceled the request.
for ( String workerId : UrlsController . workersInfoMap . keySet ( ) ) {
if ( ! UrlsController . workersInfoMap . get ( workerId ) . getHasShutdown ( ) ) // The workerId is certainly inside the map and has a workerInfo value.
return ; // If at least 1 worker is still active, then do not shut down the server.
}
logger . info ( " All workers have already shutdown. Shutting down the Controller.. " ) ;
Application . gentleAppShutdown ( ) ;
}
2023-05-24 15:59:42 +02:00
2023-05-24 23:34:36 +02:00
private static final double daysToWaitBeforeDeletion = 7 . 0 ;
2023-05-24 15:59:42 +02:00
2023-05-24 23:34:36 +02:00
@Scheduled ( initialDelay = 604_800_000 , fixedDelay = 604_800_000 ) // Run every 7 days.
//@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs).
2023-05-24 15:59:42 +02:00
public void checkAndDeleteUnsuccessfulWorkerReports ( )
{
logger . debug ( " Going to check and remove any unsuccessful workerReports, which are more than 7 days old. " ) ;
try {
File workerReportsDir = new File ( workerReportsDirPath ) ;
if ( ! workerReportsDir . isDirectory ( ) ) {
logger . error ( " The \" workerReportsDir \" ( " + workerReportsDirPath + " ) does not exist! " ) ; // This base dir should always exist!
return ;
}
File [ ] workerReports = workerReportsDir . listFiles ( File : : isFile ) ;
if ( workerReports = = null ) {
logger . error ( " There was an error when getting the subDirs of \" workerReportsDir \" : " + workerReportsDir ) ;
return ;
}
2023-05-24 23:34:36 +02:00
else if ( workerReports . length = = 0 ) {
2023-05-24 15:59:42 +02:00
logger . debug ( " The \" workerReportsDir \" is empty, so there is nothing to delete. " ) ;
return ;
}
long currentTime = System . currentTimeMillis ( ) ;
// Loop through the array and print only the directories
for ( File workerReport : workerReports ) {
long lastModified = workerReport . lastModified ( ) ;
if ( logger . isTraceEnabled ( ) )
logger . trace ( " The workerReport \" " + workerReport . getName ( ) + " \" was last accessed in: " + new Date ( lastModified ) ) ;
// Get the difference in hours. /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days.
2023-05-24 23:34:36 +02:00
double elapsedDays = ( double ) ( currentTime - lastModified ) / ( 1000 * 60 * 60 * 24 ) ;
if ( elapsedDays > daysToWaitBeforeDeletion ) {
2023-05-24 15:59:42 +02:00
// Enough time has passed, the directory should be deleted immediately.
String workerReportName = workerReport . getName ( ) ;
2023-05-24 23:34:36 +02:00
logger . warn ( " The workerReport \" " + workerReportName + " \" was accessed " + elapsedDays + " days ago (passed the " + daysToWaitBeforeDeletion + " days limit) and will be deleted. " ) ;
2023-05-24 15:59:42 +02:00
fileUtils . deleteFile ( workerReport . getAbsolutePath ( ) ) ;
}
}
} catch ( Exception e ) {
logger . error ( " " , e ) ;
}
}
2023-03-21 16:04:28 +01:00
}