From e04da6d66afe1c24d5b611b714bd07be395a81c8 Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Thu, 26 Mar 2020 14:17:07 +0100
Subject: [PATCH] merged all oozie wf in one

---
 .../oozie_app/config-default.xml              | 30 ---------
 .../propagaterels/oozie_app/workflow.xml      | 52 ---------------
 .../entity/oozie_app/config-default.xml       | 30 ---------
 .../update/entity/oozie_app/workflow.xml      | 65 -------------------
 4 files changed, 177 deletions(-)
 delete mode 100644 dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/config-default.xml
 delete mode 100644 dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/workflow.xml
 delete mode 100644 dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/config-default.xml
 delete mode 100644 dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/workflow.xml

diff --git a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/config-default.xml b/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/config-default.xml
deleted file mode 100644
index 8d8766283..000000000
--- a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/config-default.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<configuration>
-    <property>
-        <name>jobTracker</name>
-        <value>yarnRM</value>
-    </property>
-    <property>
-        <name>nameNode</name>
-        <value>hdfs://nameservice1</value>
-    </property>
-    <property>
-        <name>oozie.use.system.libpath</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>oozie.action.sharelib.for.spark</name>
-        <value>spark2</value>
-    </property>
-    <property>
-        <name>hive_metastore_uris</name>
-        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
-    </property>
-    <property>
-        <name>hive_jdbc_url</name>
-        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
-    </property>
-    <property>
-        <name>hive_db_name</name>
-        <value>openaire</value>
-    </property>
-</configuration>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/workflow.xml
deleted file mode 100644
index fd5cd6d7f..000000000
--- a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/propagaterels/oozie_app/workflow.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-<workflow-app name="Dedup Entities" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>relationPath</name>
-            <description>the source path</description>
-        </property>
-        <property>
-            <name>mergeRelPath</name>
-            <description>the target path</description>
-        </property>
-        <property>
-            <name>sparkDriverMemory</name>
-            <description>memory for driver process</description>
-        </property>
-        <property>
-            <name>sparkExecutorMemory</name>
-            <description>memory for individual executor</description>
-        </property>
-    </parameters>
-
-    <start to="PropagateRelation"/>
-
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-    <action name="PropagateRelation">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Propagate Dedup Relations</name>
-            <class>eu.dnetlib.dedup.SparkPropagateRelationsJob</class>
-            <jar>dhp-dedup-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory ${sparkExecutorMemory}
-                --driver-memory=${sparkDriverMemory}
-                --num-executors 100
-                --conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2"
-            </spark-opts>
-            <arg>-mt</arg><arg>yarn-cluster</arg>
-            <arg>--mergeRelPath</arg><arg>${mergeRelPath}</arg>
-            <arg>--relationPath</arg><arg>${relationPath}</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-    <end name="End"/>
-</workflow-app>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/config-default.xml b/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/config-default.xml
deleted file mode 100644
index ba2df7773..000000000
--- a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/config-default.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<configuration>
-    <property>
-        <name>jobTracker</name>
-        <value>yarnRM</value>
-    </property>
-    <property>
-        <name>nameNode</name>
-        <value>hdfs://nameservice1</value>
-    </property>
-    <property>
-        <name>oozie.use.system.libpath</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>oozie.action.sharelib.for.spark</name>
-        <value>spark2</value>
-    </property>
-    <property>
-        <name>hive_metastore_uris</name>
-        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
-    </property>
-    <property>
-        <name>hive_db_name</name>
-        <value>openaire</value>
-    </property>
-    <property>
-        <name>master</name>
-        <value>yarn</value>
-    </property>
-</configuration>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/workflow.xml
deleted file mode 100644
index d98344736..000000000
--- a/dhp-workflows/dhp-dedup-scholexplorer/src/main/resources/eu/dnetlib/dhp/dedup/update/entity/oozie_app/workflow.xml
+++ /dev/null
@@ -1,65 +0,0 @@
-<workflow-app name="Dedup Entities" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>entity</name>
-            <description>the entity that should be processed</description>
-        </property>
-        <property>
-            <name>entityPath</name>
-            <description>the source path</description>
-        </property>
-        <property>
-            <name>mergeRelPath</name>
-            <description>the target path</description>
-        </property>
-        <property>
-            <name>dedupRecordPath</name>
-            <description>the target path</description>
-        </property>
-        <property>
-            <name>master</name>
-            <description>the target path</description>
-        </property>
-        <property>
-            <name>sparkDriverMemory</name>
-            <description>memory for driver process</description>
-        </property>
-        <property>
-            <name>sparkExecutorMemory</name>
-            <description>memory for individual executor</description>
-        </property>
-    </parameters>
-
-    <start to="updateEntity"/>
-
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-    <action name="updateEntity">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <master>${master}</master>
-            <mode>cluster</mode>
-            <name>Update ${entity} and add DedupRecord</name>
-            <class>eu.dnetlib.dedup.SparkUpdateEntityJob</class>
-            <jar>dhp-dedup-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory ${sparkExecutorMemory}
-                --driver-memory=${sparkDriverMemory}
-                --num-executors 100
-                --conf spark.yarn.jars="hdfs://hadoop-rm1.garr-pa1.d4science.org:8020/user/oozie/share/lib/lib_20180405103059/spark2"
-            </spark-opts>
-            <arg>-mt</arg><arg>${master}</arg>
-            <arg>--entityPath</arg><arg>${entityPath}</arg>
-            <arg>--mergeRelPath</arg><arg>${mergeRelPath}</arg>
-            <arg>--entity</arg><arg>${entity}</arg>
-            <arg>--dedupRecordPath</arg><arg>${dedupRecordPath}</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-    <end name="End"/>
-</workflow-app>
\ No newline at end of file