lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 13 Wiki Activity

Improve assignment of "PublicationsRetriever.threadsMultiplier", depending on the total available threads on the system. The previous assignment was not scaling well.

This commit is contained in:
Lampros Smyrnaios 2022-04-05 00:13:52 +03:00
parent 4976afa829
commit d682298850
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/eu/openaire/urls_worker/plugins/PublicationsRetrieverPlugin.java
View File

@ -53,8 +53,13 @@ public class PublicationsRetrieverPlugin {
ConnSupportUtils.shouldBlockMost5XXDomains = false;
LoaderAndChecker.setCouldRetryRegex();
PublicationsRetriever.threadsMultiplier = 10;
int workerThreadsCount = (Runtime.getRuntime().availableProcessors() * PublicationsRetriever.threadsMultiplier);
int availableProcessors = Runtime.getRuntime().availableProcessors();
if ( availableProcessors <= 4 )
PublicationsRetriever.threadsMultiplier = 10;
else
PublicationsRetriever.threadsMultiplier = 6;
int workerThreadsCount = (availableProcessors * PublicationsRetriever.threadsMultiplier);
logger.info("Use " + workerThreadsCount + " worker-threads.");
PublicationsRetriever.executor = Executors.newFixedThreadPool(workerThreadsCount);
}