BrBETA_dnet-hadoop/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml

<workflow-app name="Software-Heritage-Integration-Workflow" xmlns="uri:oozie:workflow:0.5">

    <!-- Custom parameters   -->
    <parameters>
        <property>
            <name>hiveDbName</name>
            <description>The name of the Hive DB to be used</description>
        </property>
        <property>
            <name>softwareCodeRepositoryURLs</name>
            <description>The path in the HDFS to save the software repository URLs</description>
        </property>
        <property>
            <name>lastVisitsPath</name>
            <description>The path in the HDFS to save the responses of the last visit requests</description>
        </property>
        <property>
            <name>archiveRequestsPath</name>
            <description>The path in the HDFS to save the responses of the archive requests</description>
        </property>
        <property>
            <name>actionsetsPath</name>
            <description>The path in the HDFS to save the action sets</description>
        </property>
        <property>
            <name>graphPath</name>
            <description>The path in the HDFS to the base folder of the graph</description>
        </property>
        <property>
            <name>maxNumberOfRetry</name>
            <description>Max number of retries for failed API calls</description>
        </property>
        <property>
            <name>retryDelay</name>
            <description>Retry delay for failed requests (in sec)</description>
        </property>
        <property>
            <name>requestDelay</name>
            <description>Delay between API requests (in ms)</description>
        </property>
        <property>
            <name>apiAccessToken</name>
            <description>The API Key of the SWH API</description>
        </property>
        <property>
            <name>softwareLimit</name>
            <description>Limit on the number of repo URLs to use (Optional); for debug purposes</description>
        </property>
        <property>
            <name>resumeFrom</name>
            <description>Variable that indicates the step to start from</description>
        </property>
    </parameters>

    <!-- Global variables  -->
    <global>
        <job-tracker>${jobTracker}</job-tracker>
        <name-node>${nameNode}</name-node>
        <configuration>
            <property>
                <name>oozie.action.sharelib.for.spark</name>
                <value>${oozieActionShareLibForSpark2}</value>
            </property>
            <property>
                <name>actionsetsPath</name>
                <value>${actionsetsPath}</value>
            </property>
            <property>
                <name>apiAccessToken</name>
                <value>${apiAccessToken}</value>
            </property>
        </configuration>
    </global>

    <start to="startFrom"/>

    <kill name="Kill">
        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>

    <decision name="startFrom">
        <switch>
            <case to="collect-software-repository-urls">${wf:conf('resumeFrom') eq 'collect-software-repository-urls'}</case>
            <case to="create-swh-actionsets">${wf:conf('resumeFrom') eq 'create-swh-actionsets'}</case>
            <default to="collect-software-repository-urls"/>
        </switch>
    </decision>

    <action name="collect-software-repository-urls">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>Collect software repository URLs</name>
            <class>eu.dnetlib.dhp.swh.CollectSoftwareRepositoryURLs</class>
            <jar>dhp-swh-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-memory=${sparkExecutorMemory}
                --executor-cores=${sparkExecutorCores}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
            </spark-opts>

            <arg>--softwareCodeRepositoryURLs</arg><arg>${softwareCodeRepositoryURLs}</arg>
            <arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
            <arg>--softwareLimit</arg><arg>${softwareLimit}</arg>
        </spark>
        <ok to="collect-repository-last-visit-data"/>
        <error to="Kill"/>
    </action>

    <action name="collect-repository-last-visit-data">
        <java>
            <main-class>eu.dnetlib.dhp.swh.CollectLastVisitRepositoryData</main-class>

            <arg>--namenode</arg><arg>${nameNode}</arg>
            <arg>--softwareCodeRepositoryURLs</arg><arg>${softwareCodeRepositoryURLs}</arg>
            <arg>--lastVisitsPath</arg><arg>${lastVisitsPath}</arg>

            <arg>--maxNumberOfRetry</arg><arg>${maxNumberOfRetry}</arg>
            <arg>--requestDelay</arg><arg>${requestDelay}</arg>
            <arg>--retryDelay</arg><arg>${retryDelay}</arg>
            <arg>--requestMethod</arg><arg>GET</arg>
            <arg>--apiAccessToken</arg><arg>${apiAccessToken}</arg>

        </java>
        <ok to="archive-repository-urls"/>
        <error to="Kill"/>
    </action>

    <action name="archive-repository-urls">
        <java>
            <main-class>eu.dnetlib.dhp.swh.ArchiveRepositoryURLs</main-class>

            <arg>--namenode</arg><arg>${nameNode}</arg>
            <arg>--lastVisitsPath</arg><arg>${lastVisitsPath}</arg>
            <arg>--archiveRequestsPath</arg><arg>${archiveRequestsPath}</arg>
            <arg>--archiveThresholdInDays</arg><arg>365</arg>

            <arg>--maxNumberOfRetry</arg><arg>${maxNumberOfRetry}</arg>
            <arg>--requestDelay</arg><arg>${requestDelay}</arg>
            <arg>--retryDelay</arg><arg>${retryDelay}</arg>
            <arg>--requestMethod</arg><arg>POST</arg>
            <arg>--apiAccessToken</arg><arg>${apiAccessToken}</arg>

        </java>
        <ok to="create-swh-actionsets"/>
        <error to="Kill"/>
    </action>

    <action name="create-swh-actionsets">
        <spark xmlns="uri:oozie:spark-action:0.2">
            <master>yarn</master>
            <mode>cluster</mode>
            <name>Create actionsets for SWH data</name>
            <class>eu.dnetlib.dhp.swh.PrepareSWHActionsets</class>
            <jar>dhp-swh-${projectVersion}.jar</jar>
            <spark-opts>
                --executor-memory=${sparkExecutorMemory}
                --executor-cores=${sparkExecutorCores}
                --driver-memory=${sparkDriverMemory}
                --conf spark.extraListeners=${spark2ExtraListeners}
                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
            </spark-opts>

            <arg>--lastVisitsPath</arg><arg>${lastVisitsPath}</arg>
            <arg>--actionsetsPath</arg><arg>${actionsetsPath}</arg>
            <arg>--softwareInputPath</arg><arg>${graphPath}/software</arg>
        </spark>
        <ok to="End"/>
        <error to="Kill"/>
    </action>

    <end name="End"/>

</workflow-app>