package eu.openaire.urls_worker; import eu.openaire.publications_retriever.PublicationsRetriever; import eu.openaire.publications_retriever.util.file.FileUtils; import eu.openaire.urls_worker.plugins.PublicationsRetrieverPlugin; import eu.openaire.urls_worker.util.AssignmentsHandler; import eu.openaire.urls_worker.util.UriBuilder; import eu.openaire.urls_worker.util.WorkerConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.cors.CorsConfiguration; import org.springframework.web.cors.CorsConfigurationSource; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import javax.annotation.PreDestroy; import java.io.File; import java.net.MalformedURLException; import java.net.URL; import java.util.Arrays; import java.util.Collections; import java.util.Scanner; import java.util.concurrent.TimeUnit; @SpringBootApplication @EnableConfigurationProperties @EnableScheduling public class UrlsWorkerApplication { private static final Logger logger = LoggerFactory.getLogger(UrlsWorkerApplication.class); private static final String inputDataFilePath = FileUtils.workingDir + "inputData.txt"; public static String workerId = null; public static int maxAssignmentsLimitPerBatch = 0; public static int maxAssignmentsBatchesToHandleBeforeRestart = -1; // Default value: -1 = argument-absent, 0 = infinite-batches public static String controllerBaseUrl = null; // BaseUrl template: "http://IP:PORT/api/" private static ConfigurableApplicationContext context; public static void main(String[] args) { setInputData(); // This may cause the Server to terminate early, in case the workerId or the controllerBaseUrl cannot be found. new PublicationsRetrieverPlugin(); new AssignmentsHandler(); context = SpringApplication.run(UrlsWorkerApplication.class, args); Runtime javaRuntime = Runtime.getRuntime(); logger.debug("HeapSize: " + javaRuntime.totalMemory()); logger.debug("HeapMaxSize: " + javaRuntime.maxMemory()); logger.debug("HeapFreeSize: " + javaRuntime.freeMemory()); } public static void gentleShutdown() { int exitCode = SpringApplication.exit(context, () -> 0); // The "PreDestroy" method will be called. System.exit(exitCode); } @PreDestroy public static void preDestroy() { if ( PublicationsRetriever.executor != null ) { logger.info("Shutting down the threads used by \"PublicationsRetriever\"-plugin.."); PublicationsRetriever.executor.shutdown(); // Define that no new tasks will be scheduled. try { if ( !PublicationsRetriever.executor.awaitTermination(1, TimeUnit.MINUTES) ) { logger.warn("The working threads did not finish on time! Stopping them immediately.."); PublicationsRetriever.executor.shutdownNow(); } } catch (SecurityException se) { logger.error("Could not shutdown the threads in any way..!", se); } catch (InterruptedException ie) { try { PublicationsRetriever.executor.shutdownNow(); } catch (SecurityException se) { logger.error("Could not shutdown the threads in any way..!", se); } } } } @Bean public CorsConfigurationSource corsConfigurationSource() { CorsConfiguration configuration = new CorsConfiguration(); configuration.setAllowedOrigins(Collections.singletonList("*")); configuration.setAllowedMethods(Arrays.asList("GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS")); configuration.setAllowedHeaders(Arrays.asList("authorization", "content-type", "x-auth-token")); configuration.setExposedHeaders(Collections.singletonList("x-auth-token")); UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); source.registerCorsConfiguration("/**", configuration); return source; } @Bean public CommandLineRunner setServerBaseUrl(Environment environment) { return args -> new UriBuilder(environment); } private static void setInputData() { // Take the workerId and the controllerBaseUrl from the file. Scanner myReader = null; try { File inputDataFile = new File(inputDataFilePath); if ( !inputDataFile.exists() ) { String errorMsg = "controllerBaseUrlFile \"" + inputDataFilePath + "\" does not exists!"; logger.error(errorMsg); System.err.println(errorMsg); System.exit(60); } myReader = new Scanner(inputDataFile); if ( myReader.hasNextLine() ) { String[] data = myReader.nextLine().split(","); if ( data.length < 4 ) { String errorMsg = "Not all data were retrieved from file \"" + inputDataFilePath + "\"!"; logger.error(errorMsg); System.err.println(errorMsg); System.exit(61); } workerId = data[0].trim(); String maxAssignmentsLimitStr = data[1].trim(); try { maxAssignmentsLimitPerBatch = Integer.parseInt(maxAssignmentsLimitStr); } catch (NumberFormatException nfe) { logger.warn("The given \"maxAssignmentsLimitPerBatch\" (" + maxAssignmentsLimitStr + ") was not a number! Will use the default one: " + WorkerConstants.ASSIGNMENTS_LIMIT); maxAssignmentsLimitPerBatch = WorkerConstants.ASSIGNMENTS_LIMIT; } String maxAssignmentsBatchesStr = data[2].trim(); try { maxAssignmentsBatchesToHandleBeforeRestart = Integer.parseInt(maxAssignmentsBatchesStr); } catch (NumberFormatException nfe) { logger.warn("The given \"maxAssignmentsBatchesToHandleBeforeRestart\" (" + maxAssignmentsBatchesStr + ") was not a number! Will handle an infinite number of batches!"); maxAssignmentsBatchesToHandleBeforeRestart = 0; } controllerBaseUrl = data[3].trim(); try { new URL(controllerBaseUrl); } catch (MalformedURLException mue) { String errorMsg = "The given \"controllerBaseUrl\" (\"" + controllerBaseUrl + "\") was malformed! Please restart the program and give a valid URL."; logger.error(errorMsg); System.err.println(errorMsg); System.exit(62); } if ( !controllerBaseUrl.endsWith("/") ) controllerBaseUrl += "/"; // Make sure the other urls will not break later. } if ( (workerId == null) || (maxAssignmentsLimitPerBatch == 0) || (maxAssignmentsBatchesToHandleBeforeRestart == -1) || (controllerBaseUrl == null) ) { String errorMsg = "No \"workerId\" or/and \"maxAssignmentsLimitPerBatch\" or/and \"maxAssignmentsBatchesToHandleBeforeRestart\" or/and \"controllerBaseUrl\" could be retrieved from the file: " + inputDataFilePath; logger.error(errorMsg); System.err.println(errorMsg); System.exit(63); } logger.info("workerId: " + workerId + ", maxAssignmentsLimitPerBatch: " + maxAssignmentsLimitPerBatch + ", maxAssignmentsBatchesToHandleBeforeRestart: " + maxAssignmentsBatchesToHandleBeforeRestart + ", controllerBaseUrl: " + controllerBaseUrl); // It's safe and helpful to show them in the logs. } catch (Exception e) { String errorMsg = "An error prevented the retrieval of the workerId and the controllerBaseUrl from the file: " + inputDataFilePath + "\n" + e.getMessage(); logger.error(errorMsg, e); System.err.println(errorMsg); System.exit(64); } finally { if ( myReader != null ) myReader.close(); } } }