From 38ecfd57855b212cda31863ab78bbd2b5ae2e418 Mon Sep 17 00:00:00 2001
From: "miriam.baglioni" <miriam.baglioni@isti.cnr.it>
Date: Thu, 30 Apr 2020 18:28:46 +0200
Subject: [PATCH] the wf with all the three steps for blacklisting relations

---
 .../dhp/blacklist/oozie_app/workflow.xml      | 53 ++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/dhp-workflows/dhp-blacklist/src/main/resources/eu/dnetlib/dhp/blacklist/oozie_app/workflow.xml b/dhp-workflows/dhp-blacklist/src/main/resources/eu/dnetlib/dhp/blacklist/oozie_app/workflow.xml
index e38d721b9..48351e3fb 100644
--- a/dhp-workflows/dhp-blacklist/src/main/resources/eu/dnetlib/dhp/blacklist/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-blacklist/src/main/resources/eu/dnetlib/dhp/blacklist/oozie_app/workflow.xml
@@ -12,6 +12,10 @@
         <name>postgresPassword</name>
         <description>the postgres password</description>
     </property>
+    <property>
+        <name>sourcePath</name>
+        <description>the source path</description>
+    </property>
 </parameters>
 <start to="reset-outputpath"/>
 
@@ -38,9 +42,56 @@
             <arg>--postgresUser</arg><arg>${postgresUser}</arg>
             <arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
         </java>
-        <ok to="End"/>
+        <ok to="prepare_merged_relation"/>
         <error to="Kill"/>
     </action>
 
+    <action name="prepare_merged_relation">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>PrepareMergedRelation</name>
+            <class>eu.dnetlib.dhp.blacklist.PrepareMergedRelationJob</class>
+            <jar>dhp-blacklist-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-cores=${sparkExecutorCores}
+                --executor-memory=${sparkExecutorMemory}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation</arg>
+            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
+        </spark>
+        <ok to="apply_blacklist"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="apply_blacklist">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>ApplyBlacklist</name>
+            <class>eu.dnetlib.dhp.blacklist.SparkRemoveBlacklistedRelationJob</class>
+            <jar>dhp-blacklist-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-cores=${sparkExecutorCores}
+                --executor-memory=${sparkExecutorMemory}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation</arg>
+            <arg>--hdfsPath</arg><arg>${workingDir}/blacklist</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
 <end name="End"/>
 </workflow-app>
\ No newline at end of file