lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

WIP: dedup workflow, stage 2

This commit is contained in:
Claudio Atzori 2020-03-24 09:59:28 +01:00
parent 93e2291291
commit aaedbb1b8b
1 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
dhp-workflows/dhp-dedup/src/main/resources/eu/dnetlib/dhp/dedup/roots/oozie_app/workflow.xml
View File

@ -34,6 +34,21 @@
</property> </property>
</parameters> </parameters>
<global>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<configuration>
<property>
<name>mapreduce.job.queuename</name>
<value>${queueName}</value>
</property>
<property>
<name>oozie.launcher.mapred.job.queue.name</name>
<value>${oozieLauncherQueueName}</value>
</property>
</configuration>
</global>
<start to="CreateMergeRel"/> <start to="CreateMergeRel"/>
<kill name="Kill"> <kill name="Kill">
@ -45,8 +60,6 @@
<prepare> <prepare>
<delete path='${workingPath}/${actionSetId}/*_mergerel'/> <delete path='${workingPath}/${actionSetId}/*_mergerel'/>
</prepare> </prepare>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<master>yarn-cluster</master> <master>yarn-cluster</master>
<mode>cluster</mode> <mode>cluster</mode>
<name>Create Merge Relations</name> <name>Create Merge Relations</name>
@ -75,8 +88,6 @@
<prepare> <prepare>
<delete path='${workingPath}/${actionSetId}/*_deduprecord'/> <delete path='${workingPath}/${actionSetId}/*_deduprecord'/>
</prepare> </prepare>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<master>yarn-cluster</master> <master>yarn-cluster</master>
<mode>cluster</mode> <mode>cluster</mode>
<name>Create Dedup Record</name> <name>Create Dedup Record</name>