From 01e378ea66db31615040afb0f0268933b275b213 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Thu, 5 Oct 2023 12:02:52 +0300 Subject: [PATCH] - Add progress-report-log for assignments-processing. - Code polishing. --- .../plugins/PublicationsRetrieverPlugin.java | 17 ++++++++++------- .../urls_worker/UrlsWorkerApplicationTests.java | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java index 901fa3f..305c7ad 100644 --- a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java @@ -37,14 +37,11 @@ public class PublicationsRetrieverPlugin { public static String assignmentsBasePath; - private static String workerId; - private static CookieStore cookieStore = null; - public PublicationsRetrieverPlugin(@Value("${info.workerId}")String workerId, @Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) { + public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) { // Specify some configurations - this.workerId = workerId; LoaderAndChecker.retrieveDocuments = true; LoaderAndChecker.retrieveDatasets = false; ConnSupportUtils.setKnownMimeTypes(); @@ -89,10 +86,17 @@ public class PublicationsRetrieverPlugin { throw new RuntimeException("Failed to create the full-texts directory for assignments_" + assignmentRequestCounter + ": " + e.getMessage()); } + final int[] urlsCounter = {0}; + int numOfAssignments = assignments.size(); + // Start loading and checking urls. for ( Assignment assignment : assignments ) { callableTasks.add(() -> { + urlsCounter[0]++; + if ( (urlsCounter[0] % 250) == 0 ) // Every 500 urls, display a "progress" message. + logger.debug("Assignments_" + assignmentRequestCounter + " progress: will process url-" + urlsCounter[0] + " out of " + numOfAssignments); + String id = assignment.getId(); String url = assignment.getOriginalUrl(); @@ -103,11 +107,10 @@ public class PublicationsRetrieverPlugin { return false; } - if ( (url = LoaderAndChecker.handleUrlChecks(id, url)) == null ) { + if ( (url = LoaderAndChecker.handleUrlChecks(id, url)) == null ) return false; - } // The "url" might have changed (inside "handleUrlChecks()"). - String urlToCheck = url; + String urlToCheck = url; // The "url" might have changed (inside "handleUrlChecks()"). String sourceUrl = urlToCheck; // Hold it here for the logging-messages. if ( (urlToCheck = LoaderAndChecker.basicURLNormalizer.filter(sourceUrl)) == null ) { logger.warn("Could not normalize url: " + sourceUrl); diff --git a/src/test/java/eu/openaire/urls_worker/UrlsWorkerApplicationTests.java b/src/test/java/eu/openaire/urls_worker/UrlsWorkerApplicationTests.java index 2e2a3a3..7a87f0b 100644 --- a/src/test/java/eu/openaire/urls_worker/UrlsWorkerApplicationTests.java +++ b/src/test/java/eu/openaire/urls_worker/UrlsWorkerApplicationTests.java @@ -1,6 +1,6 @@ package eu.openaire.urls_worker; -import org.junit.jupiter.api.Test; +//import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest