From a9120164aa8377d11ff6136d2be8d3c7bb6bf6cc Mon Sep 17 00:00:00 2001
From: "miriam.baglioni" <miriam.baglioni@isti.cnr.it>
Date: Thu, 16 Apr 2020 18:42:04 +0200
Subject: [PATCH] added hive parameter and a step of reset of the working dir
 in the workflow

---
 .../oozie_app/config-default.xml              |  4 ++
 .../oozie_app/workflow.xml                    | 65 ++++++-------------
 2 files changed, 24 insertions(+), 45 deletions(-)

diff --git a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml
index 56d0ac43c8..8d2c341057 100644
--- a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml
+++ b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/config-default.xml
@@ -19,6 +19,10 @@
         <name>spark2YarnHistoryServerAddress</name>
         <value>http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089</value>
     </property>
+    <property>
+        <name>hive_metastore_uris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
     <property>
         <name>spark2EventLogDir</name>
         <value>/user/spark/spark2ApplicationHistory</value>
diff --git a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml
index a3c043bf00..d20b839eea 100644
--- a/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml
@@ -17,13 +17,21 @@
         <description>writes new version of the graph after the propagation step</description>
     </property>
 </parameters>
+    <start to="reset-outputpath"/>
 
-<start to="fork_prepare_assoc_step1"/>
+
+<!--    <start to="fork_prepare_assoc_step1"/>-->
 
 <kill name="Kill">
     <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
 </kill>
-
+    <action name="reset-outputpath">
+        <fs>
+            <delete path='${workingDir}/projecttoresult_propagation/relation'/>
+        </fs>
+        <ok to="fork_prepare_assoc_step1"/>
+        <error to="Kill"/>
+    </action>
     <fork name="fork_prepare_assoc_step1">
         <path start="join_prepare_publication"/>
         <path start="join_prepare_dataset"/>
@@ -35,7 +43,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-PreparePhase1-Publications</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -63,7 +71,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-PreparePhase1-Dataset</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -91,7 +99,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-PreparePhase1-ORP</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -119,7 +127,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-PreparePhase1-Software</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep1</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -139,50 +147,17 @@
             <arg>--outputPath</arg><arg>${workingDir}/preparedInfo/targetOrcidAssoc</arg>
             <arg>--allowedsemrels</arg><arg>${allowedsemrels}</arg>
         </spark>
-        <ok to="End"/>
+        <ok to="wait"/>
         <error to="Kill"/>
     </action>
 
-
-
-<action name="OrcidToResultFromSemRelPropagation">
-    <spark xmlns="uri:oozie:spark-action:0.2">
-        <job-tracker>${jobTracker}</job-tracker>
-        <name-node>${nameNode}</name-node>
-        <master>yarn-cluster</master>
-        <mode>cluster</mode>
-        <name>OrcidToResultFromSemRelPropagation</name>
-        <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob</class>
-        <jar>dhp-propagation-${projectVersion}.jar</jar>
-        <spark-opts>
-            --num-executors=${sparkExecutorNumber}
-            --executor-memory=${sparkExecutorMemory}
-            --executor-cores=${sparkExecutorCores}
-            --driver-memory=${sparkDriverMemory}
-            --conf spark.extraListeners=${spark2ExtraListeners}
-            --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-            --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-            --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            --conf spark.dynamicAllocation.enabled=true
-            --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
-        </spark-opts>
-        <arg>-mt</arg> <arg>yarn-cluster</arg>
-        <arg>--sourcePath</arg><arg>${sourcePath}</arg>
-        <arg>--allowedsemrels</arg><arg>${allowedsemrels}</arg>
-        <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
-        <arg>--saveGraph</arg><arg>${saveGraph}</arg>
-    </spark>
-    <ok to="End"/>
-    <error to="Kill"/>
-</action>
-
     <join name="wait" to="prepare_assoc_step2"/>
 
     <action name="prepare_assoc_step2">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-PreparePhase2</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.PrepareResultOrcidAssociationStep2</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -215,7 +190,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-Publication</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -241,7 +216,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-Dataset</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -267,7 +242,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-ORP</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>
@@ -293,7 +268,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>ORCIDpropagationFromSemRel</name>
+            <name>ORCIDPropagation-Software</name>
             <class>eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob3</class>
             <jar>dhp-propagation-${projectVersion}.jar</jar>
             <spark-opts>