dnet-hadoop/dhp-workflows/dhp-propagation/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml

<workflow-app name="affiliation_from_instrepo_propagation" xmlns="uri:oozie:workflow:0.5">
<parameters>
    <property>
        <name>sourcePath</name>
        <description>the source path</description>
    </property>
    <property>
        <name>writeUpdate</name>
        <description>writes the information found for the update. No double check done if the information is already present</description>
    </property>
    <property>
        <name>saveGraph</name>
        <description>writes new version of the graph after the propagation step</description>
    </property>
</parameters>

<start to="reset-outputpath"/>
<!--<start to="prepare_result_organization_association"/>-->
<kill name="Kill">
    <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
</kill>

    <action name="reset-outputpath">
        <fs>
            <delete path='${workingDir}/resulttoorganization_propagation'/>
        </fs>
        <ok to="copy_relations"/>
        <error to="Kill"/>
    </action>

    <fork name="copy_relations">
        <path start="copy_relation"/>
        <path start="copy_publication"/>
        <path start="copy_dataset"/>
        <path start="copy_orp"/>
        <path start="copy_software"/>
    </fork>

    <action name="copy_relation">
        <distcp xmlns="uri:oozie:distcp-action:0.2">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <arg>${nameNode}/${sourcePath}/relation</arg>
            <arg>${nameNode}/${workingDir}/resulttoorganization_propagation/relation</arg>
        </distcp>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="copy_publication">
        <distcp xmlns="uri:oozie:distcp-action:0.2">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <arg>${nameNode}/${sourcePath}/publication</arg>
            <arg>${nameNode}/${workingDir}/resulttoorganization_propagation/publication</arg>
        </distcp>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="copy_dataset">
        <distcp xmlns="uri:oozie:distcp-action:0.2">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <arg>${nameNode}/${sourcePath}/dataset</arg>
            <arg>${nameNode}/${workingDir}/resulttoorganization_propagation/dataset</arg>
        </distcp>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="copy_orp">
        <distcp xmlns="uri:oozie:distcp-action:0.2">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <arg>${nameNode}/${sourcePath}/otherresearchproduct</arg>
            <arg>${nameNode}/${workingDir}/resulttoorganization_propagation/otherresearchproduct</arg>
        </distcp>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>
    <action name="copy_software">
        <distcp xmlns="uri:oozie:distcp-action:0.2">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <arg>${nameNode}/${sourcePath}/software</arg>
            <arg>${nameNode}/${workingDir}/resulttoorganization_propagation/software</arg>
        </distcp>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>
    <join name="wait" to="prepare_result_organization_association"/>
    <action name="prepare_result_organization_association">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>PrepareResultOrganizationAssociation</name>
            <class>eu.dnetlib.dhp.resulttoorganizationfrominstrepo.PrepareResultInstRepoAssociation</class>
            <jar>dhp-propagation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
            </spark-opts>
            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
            <arg>--datasourceOrganizationPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization</arg>
            <arg>--alreadyLinkedPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked</arg>
        </spark>
        <ok to="fork_join_apply_resulttoorganization_propagation"/>
<!--        <ok to="End"/>-->
        <error to="Kill"/>
    </action>
    <fork name="fork_join_apply_resulttoorganization_propagation">
        <path start="join_propagation_publication"/>
        <path start="join_propagation_dataset"/>
        <path start="join_propagation_otherresearchproduct"/>
        <path start="join_propagation_software"/>
    </fork>

    <action name="join_propagation_publication">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>resultToOrganizationFromInstRepoPropagationForPublications</name>
            <class>eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2</class>
            <jar>dhp-propagation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.dynamicAllocation.enabled=true
                --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
            </spark-opts>
            <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
            <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
            <arg>--saveGraph</arg><arg>${saveGraph}</arg>
            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
            <arg>--outputPath</arg><arg>${workingDir}/resulttoorganization_propagation/relation</arg>
            <arg>--datasourceOrganizationPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization</arg>
            <arg>--alreadyLinkedPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked</arg>
        </spark>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="join_propagation_dataset">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>resultToOrganizationFromInstRepoPropagationForDataset</name>
            <class>eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2</class>
            <jar>dhp-propagation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.dynamicAllocation.enabled=true
                --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
            </spark-opts>
            <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
            <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
            <arg>--saveGraph</arg><arg>${saveGraph}</arg>
            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
            <arg>--outputPath</arg><arg>${workingDir}/resulttoorganization_propagation/relation</arg>
            <arg>--datasourceOrganizationPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization</arg>
            <arg>--alreadyLinkedPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked</arg>
        </spark>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="join_propagation_otherresearchproduct">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>resultToOrganizationFromInstRepoPropagationForORP</name>
            <class>eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2</class>
            <jar>dhp-propagation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.dynamicAllocation.enabled=true
                --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
            </spark-opts>
            <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
            <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
            <arg>--saveGraph</arg><arg>${saveGraph}</arg>
            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
            <arg>--outputPath</arg><arg>${workingDir}/resulttoorganization_propagation/relation</arg>
            <arg>--datasourceOrganizationPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization</arg>
            <arg>--alreadyLinkedPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked</arg>
        </spark>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="join_propagation_software">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>resultToOrganizationFromInstRepoPropagationForSoftware</name>
            <class>eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob2</class>
            <jar>dhp-propagation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.dynamicAllocation.enabled=true
                --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
            </spark-opts>
            <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
            <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
            <arg>--saveGraph</arg><arg>${saveGraph}</arg>
            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
            <arg>--outputPath</arg><arg>${workingDir}/resulttoorganization_propagation/relation</arg>
            <arg>--datasourceOrganizationPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/datasourceOrganization</arg>
            <arg>--alreadyLinkedPath</arg><arg>${workingDir}/resulttoorganization_propagation/preparedInfo/alreadyLinked</arg>
        </spark>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <join name="wait" to="End"/>
    <end name="End"/>

<!--    <action name="AffiliationPropagation">-->
<!--    <spark xmlns="uri:oozie:spark-action:0.2">-->
<!--        <job-tracker>${jobTracker}</job-tracker>-->
<!--        <name-node>${nameNode}</name-node>-->
<!--        <master>yarn-cluster</master>-->
<!--        <mode>cluster</mode>-->
<!--        <name>AffiliationPropagation</name>-->
<!--        <class>eu.dnetlib.dhp.resulttoorganizationfrominstrepo.SparkResultToOrganizationFromIstRepoJob</class>-->
<!--        <jar>dhp-propagation-${projectVersion}.jar</jar>-->
<!--        <spark-opts>-->
<!--            &#45;&#45;num-executors=${sparkExecutorNumber}-->
<!--            &#45;&#45;executor-memory=${sparkExecutorMemory}-->
<!--            &#45;&#45;executor-cores=${sparkExecutorCores}-->
<!--            &#45;&#45;driver-memory=${sparkDriverMemory}-->
<!--            &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
<!--            &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
<!--            &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
<!--            &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
<!--            &#45;&#45;conf spark.dynamicAllocation.enabled=true-->
<!--            &#45;&#45;conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}-->
<!--        </spark-opts>-->
<!--        <arg>-mt</arg> <arg>yarn-cluster</arg>-->
<!--        <arg>&#45;&#45;sourcePath</arg><arg>${sourcePath}</arg>-->
<!--        <arg>&#45;&#45;hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>-->
<!--        <arg>&#45;&#45;writeUpdate</arg><arg>${writeUpdate}</arg>-->
<!--        <arg>&#45;&#45;saveGraph</arg><arg>${saveGraph}</arg>-->
<!--    </spark>-->
<!--    <ok to="End"/>-->
<!--    <error to="Kill"/>-->
<!--</action>-->

</workflow-app>