From 0f12a9305cb5255681e3cd0827ae8a5a9e922c8b Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Thu, 14 Oct 2021 03:03:47 +0300 Subject: [PATCH] - Decrease the time interval for the scheduled task "handleNewAssignments". This helps to reduce the "dead-time" between reporting the current assignments and requesting the new ones. - Avoid a potential NPE when giving information about the received AssignmentRequest. - Log and return, when the received assignments-list is empty. - Improve some logging-messages. - Update the logs' fileName and change the preferred appender to "File". - Code cleanup. --- .../components/ScheduledTasks.java | 4 ++-- .../plugins/PublicationsRetrieverPlugin.java | 7 ++---- .../urls_worker/util/AssignmentHandler.java | 22 ++++++++++++------- src/main/resources/logback-spring.xml | 6 ++--- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java index a3112f5..533d752 100644 --- a/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_worker/components/ScheduledTasks.java @@ -23,8 +23,8 @@ public class ScheduledTasks { logger.info("Server is live! Time is now {}", dateFormat.format(new Date())); } - @Scheduled(fixedRate = 1800_000) // Every 30 mins: 1800_000 - public void handleAssignment() { + @Scheduled(fixedRate = 900_000) // Every 15 mins: 900_000 + public void handleNewAssignments() { if ( AssignmentHandler.isAvailableForWork ) AssignmentHandler.handleAssignments(); else diff --git a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java index 1531ea0..122b854 100644 --- a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java @@ -49,12 +49,11 @@ public class PublicationsRetrieverPlugin { LoaderAndChecker.retrieveDocuments = true; LoaderAndChecker.retrieveDatasets = false; FileUtils.shouldDownloadDocFiles = true; - FileUtils.shouldUploadFilesToS3 = true; FileUtils.docFileNameType = FileUtils.DocFileNameType.idName; PublicationsRetriever.targetUrlType = "docUrl"; - if ( FileUtils.shouldUploadFilesToS3 ) - new S3ObjectStoreMinIO(); // Check here on how to create the credentials-file: https://github.com/LSmyrnaios/PublicationsRetriever/blob/master/README.md + FileUtils.shouldUploadFilesToS3 = true; + new S3ObjectStoreMinIO(); // Check here on how to create the credentials-file: https://github.com/LSmyrnaios/PublicationsRetriever/blob/master/README.md int workerThreadsCount = Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier; logger.info("Use " + workerThreadsCount + " worker-threads."); @@ -65,9 +64,7 @@ public class PublicationsRetrieverPlugin { public static void processAssignments(Long assignmentRequestCounter, Collection assignments) throws RuntimeException, FileNotFoundException { ConnSupportUtils.setKnownMimeTypes(); - FileUtils.storeDocFilesDir = assignmentsBaseFullTextsPath + "assignment_" + assignmentRequestCounter + "_fullTexts" + File.separator; // It needs the last separator, because of how the docFiles are named and stored. - FileUtils.setOutput(new FileOutputStream(assignmentsBasePath + "assignment_" + assignmentRequestCounter + "_generic_results.json")); int tasksNumber = assignments.size(); diff --git a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java b/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java index 0ff61ae..fccc3f9 100644 --- a/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java +++ b/src/main/java/eu/openaire/urls_worker/util/AssignmentHandler.java @@ -30,9 +30,7 @@ public class AssignmentHandler { private static final Logger logger = LoggerFactory.getLogger(AssignmentHandler.class); public static boolean isAvailableForWork = true; - public static List urlReports = null; - private static final boolean askForTest = false; // Enable this only for testing. private static final Duration requestConnectTimeoutDuration = Duration.ofMinutes(1); // 1 minute. @@ -60,11 +58,15 @@ public class AssignmentHandler { } } + /** + * This is the HTTP-error-handler. + * */ @Override public void handleError(ClientHttpResponse response) throws IOException { int responseCode = response.getRawStatusCode(); String statusText = response.getStatusText(); - String additionalErrorMessage = ((!"".equals(statusText)) ? statusText : "The HTTP-response-code was: " + responseCode); + String extraBodyMessage = "\n" + response.getBody(); + String additionalErrorMessage = ((!"".equals(statusText)) ? statusText + extraBodyMessage : "The HTTP-response-code was: " + responseCode + extraBodyMessage); if ( (responseCode >= 500) && responseCode <= 599 ) logger.error("The Controller encountered a problem! " + additionalErrorMessage); else if ( (responseCode >= 400) && (responseCode <= 499) ) @@ -98,9 +100,6 @@ public class AssignmentHandler { } //logger.debug(assignmentRequest.toString()); // DEBUG! - - logger.info("AssignmentRequest < " + assignmentRequest.getAssignmentCounter() + " > was received and it's ready to be processed. It contains " + assignmentRequest.getAssignments().size() + " tasks."); - return assignmentRequest; } @@ -114,13 +113,20 @@ public class AssignmentHandler { } Long assignmentRequestCounter = assignmentRequest.getAssignmentCounter(); - List assignments = assignmentRequest.getAssignments(); if ( assignments == null ) { logger.warn("The assignments were found to be null for assignmentRequestCounter = " + assignmentRequestCounter); return; } + int assignmentsSize = assignments.size(); + if ( assignmentsSize == 0 ) { + logger.warn("The assignmentsSize was < 0 > for assignmentRequestCounter = " + assignmentRequestCounter); + return; + } + + logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " tasks."); + // Start handling the assignments, the worker is busy. isAvailableForWork = false; @@ -167,7 +173,7 @@ public class AssignmentHandler { public static boolean postWorkerReport(Long assignmentRequestCounter) { String postUrl = UrlsWorkerApplication.controllerBaseUrl + "urls/addWorkerReport"; - logger.info("Going to post the WorkerReport to the controller-server: " + postUrl); + logger.info("Going to post the WorkerReport of assignment_" + assignmentRequestCounter + " to the controller-server: " + postUrl); try { ResponseEntity responseEntity = new RestTemplateBuilder().build().postForEntity(postUrl, new WorkerReport(UrlsWorkerApplication.workerId, assignmentRequestCounter, urlReports), String.class); diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml index a92c683..59ee12e 100644 --- a/src/main/resources/logback-spring.xml +++ b/src/main/resources/logback-spring.xml @@ -1,10 +1,10 @@ - logs/urls_worker.log + logs/UrlsWorker.log - logs/urls_worker.%i.log.zip + logs/UrlsWorker.%i.log.zip @@ -24,7 +24,7 @@ - + \ No newline at end of file