antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Spark properties no longer hardcoded

This commit is contained in:
Ilias Kanellos 2023-05-11 15:36:47 +03:00
parent 3de35fd6a3
commit 5ddbb4ad10
1 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/job.properties
View File

@ -13,6 +13,14 @@ maven.executable=mvn
sparkDriverMemory=7G sparkDriverMemory=7G
sparkExecutorMemory=7G sparkExecutorMemory=7G
sparkExecutorCores=4 sparkExecutorCores=4
# Some memory and driver settings for more demanding tasks
sparkHighDriverMemory=20G
sparkNormalDriverMemory=10G
sparkHighExecutorMemory=20G
sparkNormalExecutorMemory=10G
# The above is given differently in an example I found online # The above is given differently in an example I found online
oozie.action.sharelib.for.spark=spark2 oozie.action.sharelib.for.spark=spark2
oozieActionShareLibForSpark2=spark2 oozieActionShareLibForSpark2=spark2