From c4b18e6ccb2946d9de0923b5d5bab5521907cae6 Mon Sep 17 00:00:00 2001
From: "miriam.baglioni" <miriam.baglioni@isti.cnr.it>
Date: Fri, 16 Jul 2021 15:01:25 +0200
Subject: [PATCH] changed the download.sh, added skip step to allow to not
 execute one phase and changed the workflow sequence of steps

---
 .../dhp/doiboost/preprocess/oozie_app/download.sh   |  2 +-
 .../dhp/doiboost/preprocess/oozie_app/workflow.xml  | 13 +++++++------
 .../dhp/doiboost/process/oozie_app/workflow.xml     |  1 +
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/download.sh b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/download.sh
index 98984e249..dfb0db708 100644
--- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/download.sh
+++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/download.sh
@@ -1,2 +1,2 @@
 #!bin/bash
-curl -LSs $1 | hdfs dfs -put - $2$3
\ No newline at end of file
+curl -LSs $1 | hdfs dfs -put - $2/$3
\ No newline at end of file
diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml
index d63e54b8d..a1b8804fa 100644
--- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml
@@ -70,7 +70,7 @@
             <case to="ResetMagWorkingPath">${wf:conf('resumeFrom') eq 'ResetMagWorkingPath'}</case>
             <case to="ConvertMagToDataset">${wf:conf('resumeFrom') eq 'ConvertMagToDataset'}</case>
             <case to="PreProcessORCID">${wf:conf('resumeFrom') eq 'PreProcessORCID'}</case>
-            <default to="DownloadDump"/>
+            <default to="removeFiles"/> <!-- first action to be done when downloadDump is to be performed -->
         </switch>
     </decision>
 
@@ -78,6 +78,7 @@
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
+
     <action name="DownloadDump">
         <shell xmlns="uri:oozie:shell-action:0.2">
             <job-tracker>${jobTracker}</job-tracker>
@@ -105,7 +106,7 @@
             <name-node>${nameNode}</name-node>
             <main-class>eu.dnetlib.doiboost.crossref.ExtractCrossrefRecords</main-class>
             <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
-            <arg>--crossrefFileNameTarGz</arg><arg>${crossrefDumpPath}/crossref.tar.gz</arg>
+            <arg>--crossrefFileNameTarGz</arg><arg>${crossrefdumpfilename}</arg>
             <arg>--workingPath</arg><arg>${crossrefDumpPath}</arg>
             <arg>--outputPath</arg><arg>${crossrefDumpPath}/files/</arg>
         </java>
@@ -161,16 +162,16 @@
             <arg>--targetPath</arg><arg>${inputPathCrossref}/crossref_ds</arg>
 
         </spark>
-        <ok to="removeFiles"/>
+        <ok to="ResetMagWorkingPath"/>
         <error to="Kill"/>
     </action>
 
         <action name="removeFiles">
             <fs>
-<!--                <delete path="${crossrefDumpPath}/files"/>-->
-                    <delete path="${crossrefDumpPath}/crossref_unpack/"/>
+              <delete path="${crossrefDumpPath}/files"/>
+                <delete path="${crossrefDumpPath}/crossref_unpack/"/>
             </fs>
-            <ok to="ResetMagWorkingPath"/>
+            <ok to="DownloadDump"/>
             <error to="Kill"/>
         </action>
 
diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
index f845d97f3..e75e1d8e1 100644
--- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
@@ -75,6 +75,7 @@
 
     <decision name="resume_from">
         <switch>
+            <case to="End">${wf:conf('resumeFrom') eq 'Skip'}</case>
             <case to="ProcessMAG">${wf:conf('resumeFrom') eq 'PreprocessMag'}</case>
             <case to="ProcessUW">${wf:conf('resumeFrom') eq 'PreprocessUW'}</case>
             <case to="ProcessORCID">${wf:conf('resumeFrom') eq 'ProcessORCID'}</case>