dnet-hadoop/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/project/oozie_app/workflow.xml

<workflow-app name="H2020Classification" xmlns="uri:oozie:workflow:0.5">
    <parameters>
        <property>
            <name>projectFileURL</name>
            <description>the url where to get the projects file</description>
        </property>

        <property>
            <name>programmeFileURL</name>
            <description>the url where to get the programme file</description>
        </property>

        <property>
            <name>topicFileURL</name>
            <description>the url where to get the topic file</description>
        </property>
        <property>
            <name>outputPath</name>
            <description>path where to store the action set</description>
        </property>
        <property>
            <name>sheetName</name>
            <description>the name of the sheet to read</description>
        </property>
    </parameters>

    <start to="deleteoutputpath"/>
    <kill name="Kill">
        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <action name="deleteoutputpath">
        <fs>
            <delete path='${outputPath}'/>
            <mkdir path='${outputPath}'/>
            <delete path='${workingDir}'/>
            <mkdir path='${workingDir}'/>
        </fs>
        <ok to="fork_get_info"/>
        <error to="Kill"/>
    </action>


    <fork name="fork_get_info">
        <path start="fork_get_projects"/>
        <path start="get_programme_file"/>
        <path start="get_topic_file"/>

    </fork>

    <fork name="fork_get_projects">
        <path start="get_project_file"/>
        <path start="read_projects"/>
    </fork>

    <action name="get_project_file">
        <java>
            <main-class>eu.dnetlib.dhp.actionmanager.project.utils.ReadCSV</main-class>
            <arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
            <arg>--fileURL</arg><arg>${projectFileURL}</arg>
            <arg>--hdfsPath</arg><arg>${workingDir}/projects</arg>
            <arg>--classForName</arg><arg>eu.dnetlib.dhp.actionmanager.project.utils.CSVProject</arg>
        </java>
        <ok to="wait_projects"/>
        <error to="Kill"/>
    </action>

    <action name="get_programme_file">
        <java>
            <main-class>eu.dnetlib.dhp.actionmanager.project.utils.ReadCSV</main-class>
            <arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
            <arg>--fileURL</arg><arg>${programmeFileURL}</arg>
            <arg>--hdfsPath</arg><arg>${workingDir}/programme</arg>
            <arg>--classForName</arg><arg>eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme</arg>
        </java>
        <ok to="prepare_programme"/>
        <error to="Kill"/>
    </action>

    <action name="get_topic_file">
        <java>
            <main-class>eu.dnetlib.dhp.actionmanager.project.utils.ReadExcel</main-class>
            <arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
            <arg>--fileURL</arg><arg>${topicFileURL}</arg>
            <arg>--hdfsPath</arg><arg>${workingDir}/topic</arg>
            <arg>--sheetName</arg><arg>${sheetName}</arg>
            <arg>--classForName</arg><arg>eu.dnetlib.dhp.actionmanager.project.utils.EXCELTopic</arg>
        </java>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="read_projects">
        <java>
            <main-class>eu.dnetlib.dhp.actionmanager.project.ReadProjectsFromDB</main-class>
            <arg>--hdfsPath</arg><arg>${workingDir}/dbProjects</arg>
            <arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
            <arg>--postgresUrl</arg><arg>${postgresURL}</arg>
            <arg>--postgresUser</arg><arg>${postgresUser}</arg>
            <arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
        </java>
        <ok to="wait_projects"/>
        <error to="Kill"/>
    </action>

    <action name="prepare_programme">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>PrepareProgramme</name>
            <class>eu.dnetlib.dhp.actionmanager.project.PrepareProgramme</class>
            <jar>dhp-aggregation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.shuffle.partitions=3840
            </spark-opts>
            <arg>--programmePath</arg><arg>${workingDir}/programme</arg>
            <arg>--outputPath</arg><arg>${workingDir}/preparedProgramme</arg>
        </spark>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <join name="wait" to="create_updates"/>

    <join name="wait_projects" to="prepare_project"/>


    <action name="prepare_project">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>PrepareProjects</name>
            <class>eu.dnetlib.dhp.actionmanager.project.PrepareProjects</class>
            <jar>dhp-aggregation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.shuffle.partitions=3840
            </spark-opts>
            <arg>--projectPath</arg><arg>${workingDir}/projects</arg>
            <arg>--outputPath</arg><arg>${workingDir}/preparedProjects</arg>
            <arg>--dbProjectPath</arg><arg>${workingDir}/dbProjects</arg>
        </spark>
        <ok to="wait"/>
        <error to="Kill"/>
    </action>

    <action name="create_updates">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>ProjectProgrammeAS</name>
            <class>eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob</class>
            <jar>dhp-aggregation-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-cores=${sparkExecutorCores}
                --executor-memory=${sparkExecutorMemory}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.shuffle.partitions=3840
            </spark-opts>
            <arg>--projectPath</arg><arg>${workingDir}/preparedProjects</arg>
            <arg>--programmePath</arg><arg>${workingDir}/preparedProgramme</arg>
            <arg>--topicPath</arg><arg>${workingDir}/topic</arg>
            <arg>--outputPath</arg><arg>${outputPath}</arg>
        </spark>
        <ok to="End"/>
        <error to="Kill"/>
    </action>

    <end name="End"/>
</workflow-app>