diff --git a/build.gradle b/build.gradle index 16b61ce..5058de3 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ dependencies { implementation("org.springframework.security:spring-security-web") implementation("org.springframework.security:spring-security-config") - implementation 'org.projectlombok:lombok:1.18.24' + implementation 'org.projectlombok:lombok:1.18.26' //implementation group: 'io.jsonwebtoken', name: 'jjwt-api', version: '0.11.5' // Use this in case we use auth-tokens later on. @@ -42,7 +42,7 @@ dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.1-jre' implementation 'org.apache.commons:commons-compress:1.22' - implementation 'com.github.luben:zstd-jni:1.5.4-1' + implementation 'com.github.luben:zstd-jni:1.5.4-2' testImplementation 'org.springframework.security:spring-security-test' testImplementation "org.springframework.boot:spring-boot-starter-test" diff --git a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java index cc8327b..6454c97 100644 --- a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java @@ -2,6 +2,7 @@ package eu.openaire.urls_worker.components; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import eu.openaire.publications_retriever.PublicationsRetriever; import eu.openaire.publications_retriever.util.url.GenericUtils; import eu.openaire.publications_retriever.util.url.UrlUtils; import eu.openaire.urls_worker.UrlsWorkerApplication; @@ -24,6 +25,7 @@ import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; import java.time.Duration; +import java.time.Instant; import java.util.*; @@ -123,7 +125,9 @@ public class AssignmentsHandler { logger.info("AssignmentRequest < " + assignmentRequestCounter + " > was received and it's ready to be processed. It contains " + assignmentsSize + " tasks."); - // Make sure there are no multiple occurrences of urls with the same domains next to each other on the list. + Instant startTime = Instant.now(); + + // Make sure there are no multiple occurrences of urls with the same domains are present, next to each other, inside the list. // If the same domains appear too close in the list, then this means we have large waiting-times between url-connections, due to "politeness-delays" to avoid server-overloading. assignments = getAssignmentsSpacedOutByDomain(assignments, assignmentsSize, false); @@ -144,6 +148,7 @@ public class AssignmentsHandler { // TODO - Decide which tasks run with what plugin (depending on their datasource). // First run -in parallel- the tasks which require some specific plugin. // Then, after the above plugins are finished, run the remaining tasks in the generic plugin (which handles parallelism itself). + // TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe. // For now, let's just run all tasks in the generic plugin. try { @@ -152,7 +157,7 @@ public class AssignmentsHandler { logger.error("Exception when processing the assignments_" + assignmentRequestCounter, e); } // In this case, we will either have an empty WorkerReport or a half-filled one. Either way, we want to report back to the Controller. - // TODO - If we have more than one plugin running at the same time, then make the "AssignmentsHandler.urlReports"-list thread-safe. + PublicationsRetriever.calculateAndPrintElapsedTime(startTime, Instant.now(), "The processing of assignments_" + assignmentRequestCounter + " finished after: "); if ( askForTest ) { logger.debug("UrlReports:"); // DEBUG! diff --git a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java index ae4d0a1..d20319f 100644 --- a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java @@ -191,6 +191,7 @@ public class PublicationsRetrieverPlugin { for ( DataToBeLogged data : FileUtils.dataToBeLoggedList ) { // TODO - Consider adding multi-thread execution for the following code. + // In that case, use "ConcurrentHashMap". UrlReport.StatusType status = null; String fileLocation = null, comment = data.getComment(), mimeType = null, hash = data.getHash();