2021-03-16 17:38:53 +01:00
package eu.openaire.urls_worker ;
2021-06-22 04:58:07 +02:00
import eu.openaire.publications_retriever.PublicationsRetriever ;
2021-09-21 15:21:39 +02:00
import eu.openaire.publications_retriever.util.file.FileUtils ;
2021-11-26 16:04:31 +01:00
import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin ;
2021-12-06 23:52:40 +01:00
import eu.openaire.urls_worker.util.AssignmentsHandler ;
2021-11-26 16:04:31 +01:00
import eu.openaire.urls_worker.util.UriBuilder ;
2021-12-06 23:52:40 +01:00
import eu.openaire.urls_worker.util.WorkerConstants ;
2021-06-22 04:58:07 +02:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
2021-11-26 16:04:31 +01:00
import org.springframework.boot.CommandLineRunner ;
2021-03-16 17:38:53 +01:00
import org.springframework.boot.SpringApplication ;
import org.springframework.boot.autoconfigure.SpringBootApplication ;
2021-11-26 16:04:31 +01:00
import org.springframework.boot.context.properties.EnableConfigurationProperties ;
2021-12-23 23:12:34 +01:00
import org.springframework.context.ConfigurableApplicationContext ;
2021-11-26 16:04:31 +01:00
import org.springframework.context.annotation.Bean ;
import org.springframework.core.env.Environment ;
2021-05-20 02:28:48 +02:00
import org.springframework.scheduling.annotation.EnableScheduling ;
2021-11-26 16:04:31 +01:00
import org.springframework.web.cors.CorsConfiguration ;
import org.springframework.web.cors.CorsConfigurationSource ;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource ;
2021-05-20 02:28:48 +02:00
2021-06-22 04:58:07 +02:00
import javax.annotation.PreDestroy ;
2021-09-21 15:21:39 +02:00
import java.io.File ;
2021-12-07 18:33:10 +01:00
import java.net.MalformedURLException ;
import java.net.URL ;
2021-11-26 16:04:31 +01:00
import java.util.Arrays ;
import java.util.Collections ;
2021-09-21 15:21:39 +02:00
import java.util.Scanner ;
2021-06-22 04:58:07 +02:00
import java.util.concurrent.TimeUnit ;
2021-03-16 17:38:53 +01:00
@SpringBootApplication
2021-11-26 16:04:31 +01:00
@EnableConfigurationProperties
2021-05-20 02:28:48 +02:00
@EnableScheduling
2021-03-16 17:38:53 +01:00
public class UrlsWorkerApplication {
2021-06-22 04:58:07 +02:00
private static final Logger logger = LoggerFactory . getLogger ( UrlsWorkerApplication . class ) ;
2021-09-22 15:36:48 +02:00
private static final String inputDataFilePath = FileUtils . workingDir + " inputData.txt " ;
public static String workerId = null ;
2021-12-06 23:52:40 +01:00
public static int maxAssignmentsLimitPerBatch = 0 ;
2021-12-23 23:12:34 +01:00
public static int maxAssignmentsBatchesToHandleBeforeRestart = - 1 ; // Default value: -1 = argument-absent, 0 = infinite-batches
2021-09-21 15:21:39 +02:00
public static String controllerBaseUrl = null ; // BaseUrl template: "http://IP:PORT/api/"
2021-12-23 23:12:34 +01:00
private static ConfigurableApplicationContext context ;
2021-09-22 15:36:48 +02:00
2021-03-16 17:38:53 +01:00
public static void main ( String [ ] args ) {
2021-09-21 15:21:39 +02:00
2021-09-22 15:36:48 +02:00
setInputData ( ) ; // This may cause the Server to terminate early, in case the workerId or the controllerBaseUrl cannot be found.
2021-11-26 16:04:31 +01:00
new PublicationsRetrieverPlugin ( ) ;
2021-12-06 23:52:40 +01:00
new AssignmentsHandler ( ) ;
2021-09-21 15:21:39 +02:00
2021-12-23 23:12:34 +01:00
context = SpringApplication . run ( UrlsWorkerApplication . class , args ) ;
2021-12-03 03:09:40 +01:00
Runtime javaRuntime = Runtime . getRuntime ( ) ;
logger . debug ( " HeapSize: " + javaRuntime . totalMemory ( ) ) ;
logger . debug ( " HeapMaxSize: " + javaRuntime . maxMemory ( ) ) ;
logger . debug ( " HeapFreeSize: " + javaRuntime . freeMemory ( ) ) ;
2021-03-16 17:38:53 +01:00
}
2021-09-22 15:36:48 +02:00
2021-12-23 23:12:34 +01:00
public static void gentleShutdown ( )
{
int exitCode = SpringApplication . exit ( context , ( ) - > 0 ) ; // The "PreDestroy" method will be called.
System . exit ( exitCode ) ;
}
2021-06-22 04:58:07 +02:00
@PreDestroy
public static void preDestroy ( )
{
if ( PublicationsRetriever . executor ! = null )
{
2021-12-23 23:12:34 +01:00
logger . info ( " Shutting down the threads used by \" PublicationsRetriever \" -plugin.. " ) ;
2021-06-22 04:58:07 +02:00
PublicationsRetriever . executor . shutdown ( ) ; // Define that no new tasks will be scheduled.
try {
if ( ! PublicationsRetriever . executor . awaitTermination ( 1 , TimeUnit . MINUTES ) ) {
logger . warn ( " The working threads did not finish on time! Stopping them immediately.. " ) ;
PublicationsRetriever . executor . shutdownNow ( ) ;
}
2021-11-26 16:04:31 +01:00
} catch ( SecurityException se ) {
logger . error ( " Could not shutdown the threads in any way..! " , se ) ;
} catch ( InterruptedException ie ) {
try {
PublicationsRetriever . executor . shutdownNow ( ) ;
} catch ( SecurityException se ) {
logger . error ( " Could not shutdown the threads in any way..! " , se ) ;
}
2021-06-22 04:58:07 +02:00
}
}
}
2021-11-26 16:04:31 +01:00
@Bean
public CorsConfigurationSource corsConfigurationSource ( ) {
CorsConfiguration configuration = new CorsConfiguration ( ) ;
configuration . setAllowedOrigins ( Collections . singletonList ( " * " ) ) ;
configuration . setAllowedMethods ( Arrays . asList ( " GET " , " POST " , " PUT " , " PATCH " , " DELETE " , " OPTIONS " ) ) ;
configuration . setAllowedHeaders ( Arrays . asList ( " authorization " , " content-type " , " x-auth-token " ) ) ;
configuration . setExposedHeaders ( Collections . singletonList ( " x-auth-token " ) ) ;
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource ( ) ;
source . registerCorsConfiguration ( " /** " , configuration ) ;
return source ;
}
@Bean
public CommandLineRunner setServerBaseUrl ( Environment environment )
{
return args - > new UriBuilder ( environment ) ;
}
2021-09-21 15:21:39 +02:00
2021-09-22 15:36:48 +02:00
private static void setInputData ( )
2021-09-21 15:21:39 +02:00
{
2021-09-22 15:36:48 +02:00
// Take the workerId and the controllerBaseUrl from the file.
2021-09-21 15:21:39 +02:00
Scanner myReader = null ;
try {
2021-09-22 15:36:48 +02:00
File inputDataFile = new File ( inputDataFilePath ) ;
if ( ! inputDataFile . exists ( ) ) {
String errorMsg = " controllerBaseUrlFile \" " + inputDataFilePath + " \" does not exists! " ;
2021-09-21 15:21:39 +02:00
logger . error ( errorMsg ) ;
System . err . println ( errorMsg ) ;
System . exit ( 60 ) ;
}
2021-09-22 15:36:48 +02:00
myReader = new Scanner ( inputDataFile ) ;
if ( myReader . hasNextLine ( ) ) {
String [ ] data = myReader . nextLine ( ) . split ( " , " ) ;
2021-12-23 23:12:34 +01:00
if ( data . length < 4 ) {
2021-09-22 15:36:48 +02:00
String errorMsg = " Not all data were retrieved from file \" " + inputDataFilePath + " \" ! " ;
logger . error ( errorMsg ) ;
System . err . println ( errorMsg ) ;
System . exit ( 61 ) ;
}
workerId = data [ 0 ] . trim ( ) ;
2021-12-06 23:52:40 +01:00
String maxAssignmentsLimitStr = data [ 1 ] . trim ( ) ;
try {
maxAssignmentsLimitPerBatch = Integer . parseInt ( maxAssignmentsLimitStr ) ;
} catch ( NumberFormatException nfe ) {
logger . warn ( " The given \" maxAssignmentsLimitPerBatch \" ( " + maxAssignmentsLimitStr + " ) was not a number! Will use the default one: " + WorkerConstants . ASSIGNMENTS_LIMIT ) ;
maxAssignmentsLimitPerBatch = WorkerConstants . ASSIGNMENTS_LIMIT ;
}
2021-12-23 23:12:34 +01:00
String maxAssignmentsBatchesStr = data [ 2 ] . trim ( ) ;
try {
maxAssignmentsBatchesToHandleBeforeRestart = Integer . parseInt ( maxAssignmentsBatchesStr ) ;
} catch ( NumberFormatException nfe ) {
logger . warn ( " The given \" maxAssignmentsBatchesToHandleBeforeRestart \" ( " + maxAssignmentsBatchesStr + " ) was not a number! Will handle an infinite number of batches! " ) ;
maxAssignmentsBatchesToHandleBeforeRestart = 0 ;
}
controllerBaseUrl = data [ 3 ] . trim ( ) ;
2021-12-07 18:33:10 +01:00
try {
new URL ( controllerBaseUrl ) ;
} catch ( MalformedURLException mue ) {
String errorMsg = " The given \" controllerBaseUrl \" ( \" " + controllerBaseUrl + " \" ) was malformed! Please restart the program and give a valid URL. " ;
logger . error ( errorMsg ) ;
System . err . println ( errorMsg ) ;
System . exit ( 62 ) ;
}
2021-09-22 15:36:48 +02:00
if ( ! controllerBaseUrl . endsWith ( " / " ) )
2021-12-06 23:52:40 +01:00
controllerBaseUrl + = " / " ; // Make sure the other urls will not break later.
2021-09-22 15:36:48 +02:00
}
2021-09-21 15:21:39 +02:00
2021-12-23 23:12:34 +01:00
if ( ( workerId = = null ) | | ( maxAssignmentsLimitPerBatch = = 0 ) | | ( maxAssignmentsBatchesToHandleBeforeRestart = = - 1 ) | | ( controllerBaseUrl = = null ) ) {
String errorMsg = " No \" workerId \" or/and \" maxAssignmentsLimitPerBatch \" or/and \" maxAssignmentsBatchesToHandleBeforeRestart \" or/and \" controllerBaseUrl \" could be retrieved from the file: " + inputDataFilePath ;
2021-09-21 15:21:39 +02:00
logger . error ( errorMsg ) ;
System . err . println ( errorMsg ) ;
2021-12-07 18:33:10 +01:00
System . exit ( 63 ) ;
2021-09-21 15:21:39 +02:00
}
2021-12-23 23:12:34 +01:00
logger . info ( " workerId: " + workerId + " , maxAssignmentsLimitPerBatch: " + maxAssignmentsLimitPerBatch + " , maxAssignmentsBatchesToHandleBeforeRestart: " + maxAssignmentsBatchesToHandleBeforeRestart + " , controllerBaseUrl: " + controllerBaseUrl ) ; // It's safe and helpful to show them in the logs.
2021-09-21 15:21:39 +02:00
} catch ( Exception e ) {
2021-09-22 15:36:48 +02:00
String errorMsg = " An error prevented the retrieval of the workerId and the controllerBaseUrl from the file: " + inputDataFilePath + " \ n " + e . getMessage ( ) ;
2021-11-26 16:04:31 +01:00
logger . error ( errorMsg , e ) ;
2021-09-21 15:21:39 +02:00
System . err . println ( errorMsg ) ;
2021-12-07 18:33:10 +01:00
System . exit ( 64 ) ;
2021-09-21 15:21:39 +02:00
} finally {
if ( myReader ! = null )
myReader . close ( ) ;
}
}
2021-03-16 17:38:53 +01:00
}