enrico.ottonello
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

removed dynamic allocation on wf

This commit is contained in:
Enrico Ottonello 2022-03-18 17:43:53 +01:00
parent 861f2a3306
commit db831e6f43
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/workflow.xml
View File

@ -37,6 +37,11 @@
<value>2G</value> <value>2G</value>
<description>memory for individual executor</description> <description>memory for individual executor</description>
</property> </property>
<property>
<name>sparkExecutorCores</name>
<value>4</value>
<description>number of cores used by single executor</description>
</property>
<property> <property>
<name>spark2ExtraListeners</name> <name>spark2ExtraListeners</name>
<value>com.cloudera.spark.lineage.NavigatorAppListener</value> <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
@ -73,8 +78,8 @@
<class>eu.dnetlib.dhp.rdfconverter.bioschema.SparkRdfToDatacite</class> <class>eu.dnetlib.dhp.rdfconverter.bioschema.SparkRdfToDatacite</class>
<jar>dhp-rdfconverter-${projectVersion}.jar</jar> <jar>dhp-rdfconverter-${projectVersion}.jar</jar>
<spark-opts> <spark-opts>
--conf spark.dynamicAllocation.enabled=true --executor-cores=${sparkExecutorCores}
--conf spark.dynamicAllocation.maxExecutors=${spark2RdfConversionMaxExecutors} --driver-memory=${sparkDriverMemory}
--executor-memory=${sparkExecutorMemory} --executor-memory=${sparkExecutorMemory}
--driver-memory=${sparkDriverMemory} --driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners} --conf spark.extraListeners=${spark2ExtraListeners}