232 lines
9.3 KiB
XML
232 lines
9.3 KiB
XML
<workflow-app name="H2020Classification" xmlns="uri:oozie:workflow:0.5">
|
|
<parameters>
|
|
<property>
|
|
<name>outputPath</name>
|
|
<description>path where to store the action set</description>
|
|
</property>
|
|
</parameters>
|
|
|
|
<start to="deleteoutputpath"/>
|
|
<kill name="Kill">
|
|
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
</kill>
|
|
<action name="deleteoutputpath">
|
|
<fs>
|
|
<delete path='${outputPath}'/>
|
|
<mkdir path='${outputPath}'/>
|
|
<delete path='${workingDir}'/>
|
|
<mkdir path='${workingDir}'/>
|
|
</fs>
|
|
<ok to="fork_download_info"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<fork name="fork_download_info">
|
|
<path start="fork_get_projects"/>
|
|
<path start="download_programme_file"/>
|
|
</fork>
|
|
|
|
<fork name="fork_get_projects">
|
|
<path start="download_projects"/>
|
|
<path start="read_projects_from_db"/>
|
|
</fork>
|
|
|
|
<action name="download_projects">
|
|
<shell xmlns="uri:oozie:shell-action:0.2">
|
|
<job-tracker>${jobTracker}</job-tracker>
|
|
<name-node>${nameNode}</name-node>
|
|
<configuration>
|
|
<property>
|
|
<name>mapred.job.queue.name</name>
|
|
<value>${queueName}</value>
|
|
</property>
|
|
</configuration>
|
|
<exec>download.sh</exec>
|
|
<argument>${downloadH2020Projects}</argument>
|
|
<argument>${projectPath}</argument>
|
|
<env-var>HADOOP_USER_NAME=${wf:user()}</env-var>
|
|
<file>download.sh</file>
|
|
<capture-output/>
|
|
</shell>
|
|
<ok to="extract_projects"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="extract_projects">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.actionmanager.project.utils.ExtractFromZip</main-class>
|
|
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
|
|
<arg>--inputPath</arg><arg>${projectPath}</arg>
|
|
<arg>--outputPath</arg><arg>${workingDir}/</arg>
|
|
</java>
|
|
<ok to="read_from_folder"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<fork name="read_from_folder">
|
|
<path start="read_projects"/>
|
|
<path start="read_topic_file"/>
|
|
</fork>
|
|
|
|
<action name="read_projects">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.actionmanager.project.utils.ReadProjects</main-class>
|
|
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
|
|
<arg>--inputPath</arg><arg>${workingDir}/json/project.json</arg>
|
|
<arg>--outputPath</arg><arg>${workingDir}/projects</arg>
|
|
</java>
|
|
<ok to="wait_read_from_folder"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="download_programme_file">
|
|
<shell xmlns="uri:oozie:shell-action:0.2">
|
|
<job-tracker>${jobTracker}</job-tracker>
|
|
<name-node>${nameNode}</name-node>
|
|
<configuration>
|
|
<property>
|
|
<name>mapred.job.queue.name</name>
|
|
<value>${queueName}</value>
|
|
</property>
|
|
</configuration>
|
|
<exec>download.sh</exec>
|
|
<argument>${downloadH2020Programme}</argument>
|
|
<argument>${programmePath}</argument>
|
|
<env-var>HADOOP_USER_NAME=${wf:user()}</env-var>
|
|
<file>download.sh</file>
|
|
<capture-output/>
|
|
</shell>
|
|
<ok to="extract_programme"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
<action name="extract_programme">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.actionmanager.project.utils.ExtractFromZip</main-class>
|
|
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
|
|
<arg>--inputPath</arg><arg>${programmePath}</arg>
|
|
<arg>--outputPath</arg><arg>${workingDir}/downloadedProgramme/</arg>
|
|
</java>
|
|
<ok to="read_programme"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
<action name="read_programme">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.actionmanager.project.utils.ReadCSV</main-class>
|
|
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
|
|
<arg>--fileURL</arg><arg>${workingDir}/downloadedProgramme/csv/programme.csv</arg>
|
|
<arg>--hdfsPath</arg><arg>${workingDir}/programme</arg>
|
|
<arg>--classForName</arg><arg>eu.dnetlib.dhp.actionmanager.project.utils.model.CSVProgramme</arg>
|
|
</java>
|
|
<ok to="prepare_programme"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="read_topic_file">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.actionmanager.project.utils.ReadTopics</main-class>
|
|
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
|
|
<arg>--inputPath</arg><arg>${workingDir}/json/topics.json</arg>
|
|
<arg>--outputPath</arg><arg>${workingDir}/topic</arg>
|
|
</java>
|
|
<ok to="wait_read_from_folder"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="read_projects_from_db">
|
|
<java>
|
|
<main-class>eu.dnetlib.dhp.actionmanager.project.ReadProjectsFromDB</main-class>
|
|
<arg>--hdfsPath</arg><arg>${workingDir}/dbProjects</arg>
|
|
<arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
|
|
<arg>--postgresUrl</arg><arg>${postgresURL}</arg>
|
|
<arg>--postgresUser</arg><arg>${postgresUser}</arg>
|
|
<arg>--postgresPassword</arg><arg>${postgresPassword}</arg>
|
|
</java>
|
|
<ok to="wait_projects"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="prepare_programme">
|
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
<master>yarn</master>
|
|
<mode>cluster</mode>
|
|
<name>PrepareProgramme</name>
|
|
<class>eu.dnetlib.dhp.actionmanager.project.PrepareProgramme</class>
|
|
<jar>dhp-aggregation-${projectVersion}.jar</jar>
|
|
<spark-opts>
|
|
--executor-cores=${sparkExecutorCores}
|
|
--executor-memory=${sparkExecutorMemory}
|
|
--driver-memory=${sparkDriverMemory}
|
|
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
--conf spark.sql.shuffle.partitions=3840
|
|
</spark-opts>
|
|
<arg>--programmePath</arg><arg>${workingDir}/programme</arg>
|
|
<arg>--outputPath</arg><arg>${workingDir}/preparedProgramme</arg>
|
|
</spark>
|
|
<ok to="wait"/>
|
|
<!-- <ok to="End"/>-->
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<join name="wait_read_from_folder" to="wait_projects"/>
|
|
<join name="wait" to="create_updates"/>
|
|
|
|
<join name="wait_projects" to="prepare_project"/>
|
|
|
|
|
|
<action name="prepare_project">
|
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
<master>yarn</master>
|
|
<mode>cluster</mode>
|
|
<name>PrepareProjects</name>
|
|
<class>eu.dnetlib.dhp.actionmanager.project.PrepareProjects</class>
|
|
<jar>dhp-aggregation-${projectVersion}.jar</jar>
|
|
<spark-opts>
|
|
--executor-cores=${sparkExecutorCores}
|
|
--executor-memory=${sparkExecutorMemory}
|
|
--driver-memory=${sparkDriverMemory}
|
|
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
--conf spark.sql.shuffle.partitions=3840
|
|
</spark-opts>
|
|
<arg>--projectPath</arg><arg>${workingDir}/projects</arg>
|
|
<arg>--outputPath</arg><arg>${workingDir}/preparedProjects</arg>
|
|
<arg>--dbProjectPath</arg><arg>${workingDir}/dbProjects</arg>
|
|
</spark>
|
|
<ok to="wait"/>
|
|
<!-- <ok to="End"/>-->
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<action name="create_updates">
|
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
<master>yarn</master>
|
|
<mode>cluster</mode>
|
|
<name>ProjectProgrammeAS</name>
|
|
<class>eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob</class>
|
|
<jar>dhp-aggregation-${projectVersion}.jar</jar>
|
|
<spark-opts>
|
|
--executor-cores=${sparkExecutorCores}
|
|
--executor-memory=${sparkExecutorMemory}
|
|
--driver-memory=${sparkDriverMemory}
|
|
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
--conf spark.sql.shuffle.partitions=3840
|
|
</spark-opts>
|
|
<arg>--projectPath</arg><arg>${workingDir}/preparedProjects</arg>
|
|
<arg>--programmePath</arg><arg>${workingDir}/preparedProgramme</arg>
|
|
<arg>--topicPath</arg><arg>${workingDir}/topic</arg>
|
|
<arg>--outputPath</arg><arg>${outputPath}</arg>
|
|
</spark>
|
|
<ok to="End"/>
|
|
<error to="Kill"/>
|
|
</action>
|
|
|
|
<end name="End"/>
|
|
</workflow-app> |