diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/config-default.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/config-default.xml
index 2e0ed9aee..0980c3731 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/config-default.xml
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/config-default.xml
@@ -15,4 +15,8 @@
oozie.action.sharelib.for.spark
spark2
+
+ sparkExecutorMemoryOverhead
+ 1G
+
\ No newline at end of file
diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml
index 2302d4b8f..b724e5d0b 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml
@@ -18,11 +18,15 @@
sparkDriverMemory
- memory for driver process
+ heap memory for driver process
sparkExecutorMemory
- memory for individual executor
+ heap memory for individual executor
+
+
+ sparkExecutorMemoryOverhead
+ off-heap memory for individual executor
sparkExecutorCores
@@ -86,6 +90,7 @@
dhp-dedup-openaire-${projectVersion}.jar
--executor-memory=${sparkExecutorMemory}
+ --conf spark.executor.memoryOverhead=${sparkExecutorMemoryOverhead}
--executor-cores=${sparkExecutorCores}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
@@ -111,6 +116,7 @@
dhp-dedup-openaire-${projectVersion}.jar
--executor-memory=${sparkExecutorMemory}
+ --conf spark.executor.memoryOverhead=${sparkExecutorMemoryOverhead}
--executor-cores=${sparkExecutorCores}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
@@ -135,8 +141,9 @@
eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob
dhp-dedup-openaire-${projectVersion}.jar
- --executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
+ --conf spark.executor.memoryOverhead=${sparkExecutorMemoryOverhead}
+ --executor-cores=${sparkExecutorCores}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
@@ -159,8 +166,9 @@
eu.dnetlib.dhp.oa.merge.DispatchEntitiesSparkJob
dhp-dedup-openaire-${projectVersion}.jar
- --executor-cores=${sparkExecutorCores}
--executor-memory=${sparkExecutorMemory}
+ --conf spark.executor.memoryOverhead=${sparkExecutorMemoryOverhead}
+ --executor-cores=${sparkExecutorCores}
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}