antonis.lempesis
/
UrlsWorker
forked from lsmyrnaios/UrlsWorker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- Increase the "PublicationsRetriever.threadsMultiplier" to "6", as the threads are mostly network-blocked. 

- Make sure the "maven" package is installed before compiling the "PublicationsRetriever" library.
- Update dependencies.
This commit is contained in:
Lampros Smyrnaios 2021-11-30 01:02:06 +02:00
parent 045788c728
commit 6355b3e397
4 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -1,5 +1,5 @@
plugins { plugins {
id 'org.springframework.boot' version '2.6.0' id 'org.springframework.boot' version '2.6.1'
id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java' id 'java'
} }

2
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

4
installAndRun.sh
View File

@ -25,7 +25,7 @@ if [[ ! -f $inputDataFile ]]; then
echo -e "\n\n" echo -e "\n\n"
fi fi
gradleVersion="7.2" gradleVersion="7.3"
if [[ justInstall -eq 0 ]]; then if [[ justInstall -eq 0 ]]; then
@ -39,7 +39,7 @@ if [[ justInstall -eq 0 ]]; then
# Keep a backup of the existing JAR file. # Keep a backup of the existing JAR file.
mv ./publications_retriever-1.0-SNAPSHOT.jar ./publications_retriever-1.0-SNAPSHOT_BACKUP.jar mv ./publications_retriever-1.0-SNAPSHOT.jar ./publications_retriever-1.0-SNAPSHOT_BACKUP.jar
cd PublicationsRetriever && mvn clean install cd PublicationsRetriever && sudo apt install -y maven && mvn clean install
# Copy the created JAR file to the top libs directory. # Copy the created JAR file to the top libs directory.
cp target/publications_retriever-1.0-SNAPSHOT.jar ../publications_retriever-1.0-SNAPSHOT.jar cp target/publications_retriever-1.0-SNAPSHOT.jar ../publications_retriever-1.0-SNAPSHOT.jar

1
src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
View File

@ -42,6 +42,7 @@ public class PublicationsRetrieverPlugin {
PublicationsRetriever.targetUrlType = "docUrl"; PublicationsRetriever.targetUrlType = "docUrl";
FileUtils.jsonBatchSize = WorkerConstants.ASSIGNMENTS_LIMIT; FileUtils.jsonBatchSize = WorkerConstants.ASSIGNMENTS_LIMIT;
PublicationsRetriever.threadsMultiplier = 6;
int workerThreadsCount = Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier; int workerThreadsCount = Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier;
logger.info("Use " + workerThreadsCount + " worker-threads."); logger.info("Use " + workerThreadsCount + " worker-threads.");
PublicationsRetriever.executor = Executors.newFixedThreadPool(workerThreadsCount); PublicationsRetriever.executor = Executors.newFixedThreadPool(workerThreadsCount);