From 6355b3e397412443ab7e6df315d93f414c098f30 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Tue, 30 Nov 2021 01:02:06 +0200 Subject: [PATCH] - Increase the "PublicationsRetriever.threadsMultiplier" to "6", as the threads are mostly network-blocked. - Make sure the "maven" package is installed before compiling the "PublicationsRetriever" library. - Update dependencies. --- build.gradle | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- installAndRun.sh | 4 ++-- .../urls_worker/plugins/PublicationsRetrieverPlugin.java | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 779d799..a75ca9e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'org.springframework.boot' version '2.6.0' + id 'org.springframework.boot' version '2.6.1' id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'java' } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ffed3a2..e750102 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/installAndRun.sh b/installAndRun.sh index 02cbd52..34615b0 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -25,7 +25,7 @@ if [[ ! -f $inputDataFile ]]; then echo -e "\n\n" fi -gradleVersion="7.2" +gradleVersion="7.3" if [[ justInstall -eq 0 ]]; then @@ -39,7 +39,7 @@ if [[ justInstall -eq 0 ]]; then # Keep a backup of the existing JAR file. mv ./publications_retriever-1.0-SNAPSHOT.jar ./publications_retriever-1.0-SNAPSHOT_BACKUP.jar - cd PublicationsRetriever && mvn clean install + cd PublicationsRetriever && sudo apt install -y maven && mvn clean install # Copy the created JAR file to the top libs directory. cp target/publications_retriever-1.0-SNAPSHOT.jar ../publications_retriever-1.0-SNAPSHOT.jar diff --git a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java index 5473904..ae24fcc 100644 --- a/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java @@ -42,6 +42,7 @@ public class PublicationsRetrieverPlugin { PublicationsRetriever.targetUrlType = "docUrl"; FileUtils.jsonBatchSize = WorkerConstants.ASSIGNMENTS_LIMIT; + PublicationsRetriever.threadsMultiplier = 6; int workerThreadsCount = Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier; logger.info("Use " + workerThreadsCount + " worker-threads."); PublicationsRetriever.executor = Executors.newFixedThreadPool(workerThreadsCount);