forces executor Executor memory and driver executor memory to be 7G (trying to avoid OOM)

pull/119/head
Miriam Baglioni 3 years ago
parent 2550a73981
commit 95885bcf12

@ -154,9 +154,9 @@
<class>eu.dnetlib.doiboost.crossref.GenerateCrossrefDataset</class>
<jar>dhp-doiboost-${projectVersion}.jar</jar>
<spark-opts>
--executor-memory=${sparkExecutorMemory}
--executor-cores=2
--driver-memory=${sparkDriverMemory}
--executor-memory=7G
--executor-cores=4
--driver-memory=7G
--conf spark.sql.shuffle.partitions=3840
--conf spark.extraListeners=${spark2ExtraListeners}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}

Loading…
Cancel
Save