dnet-dedup/dnet-dedup-test/src/main/resources/dedup/oozie_app/workflow.xml

<workflow-app name="Deduplication WF" xmlns="uri:oozie:workflow:0.5">
    <parameters>
        <property>
            <name>entitiesPath</name>
            <description>the input entity path</description>
        </property>
        <property>
            <name>workingPath</name>
            <description>path for the working directory</description>
        </property>
        <property>
            <name>numPartitions</name>
            <description>number of partitions for the spark files</description>
        </property>
        <property>
            <name>dedupConfPath</name>
            <description>path for the dedup configuration file</description>
        </property>
        <property>
            <name>sparkDriverMemory</name>
            <description>memory for driver process</description>
        </property>
        <property>
            <name>sparkExecutorMemory</name>
            <description>memory for individual executor</description>
        </property>
        <property>
            <name>sparkExecutorCores</name>
            <description>number of cores used by single executor</description>
        </property>
        <property>
            <name>oozieActionShareLibForSpark2</name>
            <description>oozie action sharelib for spark 2.*</description>
        </property>
        <property>
            <name>spark2ExtraListeners</name>
            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
            <description>spark 2.* extra listeners classname</description>
        </property>
        <property>
            <name>spark2SqlQueryExecutionListeners</name>
            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
            <description>spark 2.* sql query execution listeners classname</description>
        </property>
        <property>
            <name>spark2YarnHistoryServerAddress</name>
            <description>spark 2.* yarn history server address</description>
        </property>
        <property>
            <name>spark2EventLogDir</name>
            <description>spark 2.* event log dir location</description>
        </property>
    </parameters>

    <global>
        <job-tracker>${jobTracker}</job-tracker>
        <name-node>${nameNode}</name-node>
        <configuration>
            <property>
                <name>mapreduce.job.queuename</name>
                <value>${queueName}</value>
            </property>
            <property>
                <name>oozie.launcher.mapred.job.queue.name</name>
                <value>${oozieLauncherQueueName}</value>
            </property>
            <property>
                <name>oozie.action.sharelib.for.spark</name>
                <value>${oozieActionShareLibForSpark2}</value>
            </property>
        </configuration>
    </global>

    <start to="resetWorkingPath"/>

    <kill name="Kill">
        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>

    <action name="resetWorkingPath">
        <fs>
            <delete path="${workingPath}"/>
        </fs>
        <ok to="CreateSimRels"/>
        <error to="Kill"/>
    </action>

    <action name="CreateSimRels">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>Create Similarity Relations</name>
            <class>eu.dnetlib.jobs.SparkCreateSimRels</class>
            <jar>dnet-dedup-test-${projectVersion}.jar</jar>
            <spark-opts>
                --num-executors=32
                --executor-memory=12G
                --executor-cores=4
                --driver-memory=4G
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.shuffle.partitions=3840
                --conf spark.dynamicAllocation.enabled=false
            </spark-opts>
            <arg>--entitiesPath</arg><arg>${entitiesPath}</arg>
            <arg>--workingPath</arg><arg>${workingPath}</arg>
            <arg>--numPartitions</arg><arg>${numPartitions}</arg>
            <arg>--dedupConfPath</arg><arg>${dedupConfPath}</arg>
            <arg>--useTree</arg><arg>${useTree}</arg>
        </spark>
        <ok to="CreateMergeRels"/>
        <error to="Kill"/>
    </action>

    <action name="CreateMergeRels">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>Create Merge Relations</name>
            <class>eu.dnetlib.jobs.SparkCreateMergeRels</class>
            <jar>dnet-dedup-test-${projectVersion}.jar</jar>
            <spark-opts>
                --num-executors=32
                --executor-memory=12G
                --executor-cores=4
                --driver-memory=4G
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.shuffle.partitions=3840
                --conf spark.dynamicAllocation.enabled=true
            </spark-opts>
            <arg>--entitiesPath</arg><arg>${entitiesPath}</arg>
            <arg>--workingPath</arg><arg>${workingPath}</arg>
            <arg>--numPartitions</arg><arg>${numPartitions}</arg>
            <arg>--dedupConfPath</arg><arg>${dedupConfPath}</arg>
        </spark>
        <ok to="ComputeStatistics"/>
        <error to="Kill"/>
    </action>

    <action name="ComputeStatistics">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>Compute Statistics</name>
            <class>eu.dnetlib.jobs.SparkComputeStatistics</class>
            <jar>dnet-dedup-test-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-memory=${sparkExecutorMemory}
                --executor-cores=${sparkExecutorCores}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.shuffle.partitions=3840
            </spark-opts>
            <arg>--entitiesPath</arg><arg>${entitiesPath}</arg>
            <arg>--workingPath</arg><arg>${workingPath}</arg>
            <arg>--numPartitions</arg><arg>${numPartitions}</arg>
        </spark>
        <ok to="End"/>
        <error to="Kill"/>
    </action>

    <!--<action name="CreateDedupEntities">-->
        <!--<spark xmlns="uri:oozie:spark-action:0.2">-->
            <!--<master>yarn</master>-->
            <!--<mode>cluster</mode>-->
            <!--<name>Create Dedup Entities</name>-->
            <!--<class>eu.dnetlib.jobs.SparkCreateDedupEntity</class>-->
            <!--<jar>dnet-dedup-test-${projectVersion}.jar</jar>-->
            <!--<spark-opts>-->
                <!--&#45;&#45;executor-memory=${sparkExecutorMemory}-->
                <!--&#45;&#45;executor-cores=${sparkExecutorCores}-->
                <!--&#45;&#45;driver-memory=${sparkDriverMemory}-->
                <!--&#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
                <!--&#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
                <!--&#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
                <!--&#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
                <!--&#45;&#45;conf spark.sql.shuffle.partitions=3840-->
            <!--</spark-opts>-->
            <!--<arg>&#45;&#45;entitiesPath</arg><arg>${entitiesPath}</arg>-->
            <!--<arg>&#45;&#45;workingPath</arg><arg>${workingPath}</arg>-->
            <!--<arg>&#45;&#45;numPartitions</arg><arg>${numPartitions}</arg>-->
            <!--<arg>&#45;&#45;dedupConfPath</arg><arg>${dedupConfPath}</arg>-->
        <!--</spark>-->
        <!--<ok to="End"/>-->
        <!--<error to="Kill"/>-->
    <!--</action>-->

    <end name="End"/>
</workflow-app>