From 5ddbb4ad10f8885e6fdbc9c18e0356c2a25db63a Mon Sep 17 00:00:00 2001 From: ikanellos Date: Thu, 11 May 2023 15:36:47 +0300 Subject: [PATCH] Spark properties no longer hardcoded --- .../oa/graph/impact_indicators/oozie_app/job.properties | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/job.properties b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/job.properties index f9f5519cc..7b4bb96cf 100644 --- a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/job.properties +++ b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/job.properties @@ -13,6 +13,14 @@ maven.executable=mvn sparkDriverMemory=7G sparkExecutorMemory=7G sparkExecutorCores=4 + +# Some memory and driver settings for more demanding tasks +sparkHighDriverMemory=20G +sparkNormalDriverMemory=10G + +sparkHighExecutorMemory=20G +sparkNormalExecutorMemory=10G + # The above is given differently in an example I found online oozie.action.sharelib.for.spark=spark2 oozieActionShareLibForSpark2=spark2