increased spark.sql.shuffle.partitions in affiliation_from_semrel_propagation

2 years ago · f10066547b
parent 43733c1a18
commit f10066547b
1 changed files with 1 additions and 0 deletions
--- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfromsemrel/oozie_app/workflow.xml
@ -173,6 +173,7 @@
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.dynamicAllocation.enabled=true
                --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
+                --conf spark.sql.shuffle.partitions=3840
            </spark-opts>
            <arg>--relationPath</arg><arg>${workingDir}/preparedInfo/relation</arg>
            <arg>--outputPath</arg><arg>${outputPath}/relation</arg>