[dedup wf] revised memory settings to address the increased volume of input contents

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml

@@ -102,6 +102,8 @@
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.shuffle.partitions=15000
+                --conf spark.network.timeout=300s
+                --conf spark.shuffle.registration.timeout=50000
             </spark-opts>
             <arg>--graphBasePath</arg><arg>${graphBasePath}</arg>
             <arg>--graphOutputPath</arg><arg>${graphOutputPath}</arg>

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/scan/oozie_app/workflow.xml

@@ -33,16 +33,14 @@
             <description>max number of elements in a connected component</description>
         </property>
         <property>
-            <name>sparkDriverMemory</name>
+            <name>sparkResourceOpts</name>
-            <description>memory for driver process</description>
+            <value>--executor-memory=6G --conf spark.executor.memoryOverhead=4G --executor-cores=6 --driver-memory=8G --driver-cores=4</value>
+            <description>spark resource options</description>
         </property>
         <property>
-            <name>sparkExecutorMemory</name>
+            <name>sparkResourceOptsCreateMergeRel</name>
-            <description>memory for individual executor</description>
+            <value>--executor-memory=6G --conf spark.executor.memoryOverhead=4G --executor-cores=6 --driver-memory=8G --driver-cores=4</value>
-        </property>
+            <description>spark resource options</description>
-        <property>
-            <name>sparkExecutorCores</name>
-            <description>number of cores used by single executor</description>
         </property>
         <property>
             <name>oozieActionShareLibForSpark2</name>
@@ -119,9 +117,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkCreateSimRels</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOpts}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -146,9 +142,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkWhitelistSimRels</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOpts}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -174,9 +168,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkCreateMergeRels</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOptsCreateMergeRel}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -203,9 +195,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkCreateDedupRecord</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOpts}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -230,9 +220,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkCopyOpenorgsMergeRels</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOpts}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -257,9 +245,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkCreateOrgsDedupRecord</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOpts}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -283,9 +269,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkUpdateEntity</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOpts}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -309,9 +293,7 @@
             <class>eu.dnetlib.dhp.oa.dedup.SparkCopyRelationsNoOpenorgs</class>
             <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
             <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
+                ${sparkResourceOpts}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}