forked from D-Net/dnet-hadoop
[orcid] large oozie workflow cleanup; updated workflow for the orcidnodoi actionset creation
This commit is contained in:
parent
8f309b72ff
commit
55964cbd81
|
|
@ -1,42 +0,0 @@
|
||||||
<workflow-app name="import_orcid_no_doi" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>inputPath</name>
|
|
||||||
<value>/data/orcid_activities_2020/no_doi_dataset</value>
|
|
||||||
<description>path where retrieve the already generated action set</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>outputPath</name>
|
|
||||||
<value>/data/orcid_activities_2020/test_import_orcid_no_doi</value>
|
|
||||||
<description>path where to store the action set</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<global>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.spark</name>
|
|
||||||
<value>${oozieActionShareLibForSpark2}</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
</global>
|
|
||||||
|
|
||||||
<start to="importOrcidNoDoi"/>
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="importOrcidNoDoi">
|
|
||||||
<distcp xmlns="uri:oozie:distcp-action:0.2">
|
|
||||||
<arg>${inputPath}/*</arg>
|
|
||||||
<arg>${outputPath}</arg>
|
|
||||||
</distcp>
|
|
||||||
<ok to="End"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.java</name>
|
|
||||||
<value>spark2</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.map.java.opts</name>
|
|
||||||
<value>-Xmx2g</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>jobTracker</name>
|
|
||||||
<value>yarnRM</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>nameNode</name>
|
|
||||||
<value>hdfs://nameservice1</value>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>oozie.use.system.libpath</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.spark</name>
|
|
||||||
<value>spark2</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
|
|
@ -1,542 +0,0 @@
|
||||||
<workflow-app name="Gen Orcid Works-no-doi From Activities" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>workingPath</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_0</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_0.tar.gz https://orcid.figshare.com/ndownloader/files/25002232 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_0.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_0.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_0.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 0</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_1</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_1.tar.gz https://orcid.figshare.com/ndownloader/files/25002088 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_1.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_1.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_1.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 1</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_2</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_2.tar.gz https://orcid.figshare.com/ndownloader/files/25000596 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_2.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_2.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_2.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 2</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_3</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_3.tar.gz https://orcid.figshare.com/ndownloader/files/25015150 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_3.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_3.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_3.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 3</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_4</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_4.tar.gz https://orcid.figshare.com/ndownloader/files/25033643 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_4.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_4.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_4.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 4</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_5</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_5.tar.gz https://orcid.figshare.com/ndownloader/files/25005483 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_5.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_5.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_5.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 5</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_6</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_6.tar.gz https://orcid.figshare.com/ndownloader/files/25005425 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_6.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_6.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_6.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 6</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_7</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_7.tar.gz https://orcid.figshare.com/ndownloader/files/25012016 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_7.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_7.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_7.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 7</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_8</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_8.tar.gz https://orcid.figshare.com/ndownloader/files/25012079 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_8.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_8.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_8.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 8</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_9</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_9.tar.gz https://orcid.figshare.com/ndownloader/files/25010727 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_9.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_9.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_9.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 9</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_X</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_activites_X.tar.gz https://orcid.figshare.com/ndownloader/files/25011025 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_activites_X.tar.gz /data/orcid_activities_2020/ORCID_2020_10_activites_X.tar.gz ; rm -f /tmp/ORCID_2020_10_activites_X.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file X</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<start to="ResetWorkingPath"/>
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="ResetWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath}/no_doi_works/*'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="fork_check_download_files"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<fork name = "fork_check_download_files">
|
|
||||||
<path start = "check_exist_on_hdfs_activities_0"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_1"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_2"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_3"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_4"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_5"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_6"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_7"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_8"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_9"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_X"/>
|
|
||||||
</fork>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_0">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_0.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_0" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_0">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_0}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_0">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_0.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_0.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_1">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_1.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_1" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_1">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_1}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_1">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_1.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_1.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_2">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_2.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_2" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_2">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_2}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_2">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_2.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_2.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_3">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_3.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_3" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_3">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_3}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_3">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_3.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_3.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_4">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_4.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_4" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_4">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_4}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_4">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_4.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_4.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_5">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_5.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_5" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_5">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_5}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_5">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_5.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_5.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_6">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_6.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_6" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_6">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_6}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_6">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_6.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_6.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_7">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_7.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_7" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_7">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_7}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_7">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_7.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_7.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_8">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_8.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_8" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_8">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_8}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_8">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_8.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_8.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_9">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_9.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_9" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_9">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_9}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_9">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_9.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_9.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_X">
|
|
||||||
<switch>
|
|
||||||
<case to="wait_download_phase_node">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_activites_X.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_X" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_X">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_X}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="wait_download_phase_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenOrcidAuthorWork_X">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcidnodoi.GenOrcidAuthorWork</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_X.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>no_doi_works/works_X.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>no_doi_enriched_works/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<join name = "wait_download_phase_node" to = "fork_gen_orcid_author_work"/>
|
|
||||||
|
|
||||||
<fork name = "fork_gen_orcid_author_work">
|
|
||||||
<path start = "GenOrcidAuthorWork_0"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_1"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_2"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_3"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_4"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_5"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_6"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_7"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_8"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_9"/>
|
|
||||||
<path start = "GenOrcidAuthorWork_X"/>
|
|
||||||
</fork>
|
|
||||||
|
|
||||||
<join name = "join_node" to = "End"/>
|
|
||||||
|
|
||||||
<!-- <join name = "join_node" to = "fork_gen_orcid_author_work_2"/>-->
|
|
||||||
|
|
||||||
<!-- <fork name = "fork_gen_orcid_author_work_2">-->
|
|
||||||
<!-- <path start = "GenOrcidAuthorWork_6"/>-->
|
|
||||||
<!-- <path start = "GenOrcidAuthorWork_7"/>-->
|
|
||||||
<!-- <path start = "GenOrcidAuthorWork_8"/>-->
|
|
||||||
<!-- <path start = "GenOrcidAuthorWork_9"/>-->
|
|
||||||
<!-- <path start = "GenOrcidAuthorWork_X"/>-->
|
|
||||||
<!-- </fork>-->
|
|
||||||
|
|
||||||
<!-- <join name = "join_node_2" to = "End"/>-->
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>jobTracker</name>
|
|
||||||
<value>yarnRM</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>nameNode</name>
|
|
||||||
<value>hdfs://nameservice1</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.java</name>
|
|
||||||
<value>spark2</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.map.java.opts</name>
|
|
||||||
<value>-Xmx4g</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
|
|
@ -1,505 +0,0 @@
|
||||||
<workflow-app name="Gen Orcid Authors DOIs" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>workingPath_activities</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_0</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_0.tar.gz https://orcid.figshare.com/ndownloader/files/18017660 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_0.tar.gz /data/orcid_activities/ORCID_2019_activites_0.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 0</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_1</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_1.tar.gz https://orcid.figshare.com/ndownloader/files/18017675 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_1.tar.gz /data/orcid_activities/ORCID_2019_activites_1.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 1</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_2</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_2.tar.gz https://orcid.figshare.com/ndownloader/files/18017717 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_2.tar.gz /data/orcid_activities/ORCID_2019_activites_2.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 2</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_3</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_3.tar.gz https://orcid.figshare.com/ndownloader/files/18017765 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_3.tar.gz /data/orcid_activities/ORCID_2019_activites_3.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 3</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_4</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_4.tar.gz https://orcid.figshare.com/ndownloader/files/18017831 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_4.tar.gz /data/orcid_activities/ORCID_2019_activites_4.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 4</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_5</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_5.tar.gz https://orcid.figshare.com/ndownloader/files/18017987 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_5.tar.gz /data/orcid_activities/ORCID_2019_activites_5.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 5</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_6</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_6.tar.gz https://orcid.figshare.com/ndownloader/files/18018053 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_6.tar.gz /data/orcid_activities/ORCID_2019_activites_6.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 6</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_7</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_7.tar.gz https://orcid.figshare.com/ndownloader/files/18018023 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_7.tar.gz /data/orcid_activities/ORCID_2019_activites_7.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 7</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_8</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_8.tar.gz https://orcid.figshare.com/ndownloader/files/18018248 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_8.tar.gz /data/orcid_activities/ORCID_2019_activites_8.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 8</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_9</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_9.tar.gz https://orcid.figshare.com/ndownloader/files/18018029 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_9.tar.gz /data/orcid_activities/ORCID_2019_activites_9.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file 9</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_X</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2019_activites_X.tar.gz https://orcid.figshare.com/ndownloader/files/18018182 ; hdfs dfs -copyFromLocal /tmp/ORCID_2019_activites_X.tar.gz /data/orcid_activities/ORCID_2019_activites_X.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid activity file X</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<start to="ResetWorkingPath"/>
|
|
||||||
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="ResetWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath_activities}/output'/>
|
|
||||||
<mkdir path='${workingPath_activities}/output'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="fork_gen_orcid_authors_dois"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<fork name = "fork_gen_orcid_authors_dois">
|
|
||||||
<path start = "check_exist_on_hdfs_activities_0"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_1"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_2"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_3"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_4"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_5"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_6"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_7"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_8"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_9"/>
|
|
||||||
<path start = "check_exist_on_hdfs_activities_X"/>
|
|
||||||
</fork>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_0">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_0">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_0.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_0" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_0">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_0}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_0"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_0">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_0.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_0.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_1">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_1">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_1.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_1" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_1">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_1}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_1"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_1">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_1.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_1.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_2">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_2">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_2.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_2" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_2">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_2}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_2"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_2">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_2.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_2.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_3">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_3">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_3.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_3" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_3">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_3}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_3"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_3">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_3.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_3.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_4">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_4">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_4.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_4" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_4">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_4}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_4"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_4">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_4.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_4.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_5">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_5">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_5.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_5" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_5">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_5}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_5"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_5">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_5.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_5.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_6">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_6">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_6.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_6" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_6">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_6}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_6"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_6">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_6.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_6.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_7">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_7">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_7.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_7" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_7">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_7}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_7"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_7">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_7.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_7.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_8">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_8">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_8.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_8" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_8">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_8}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_8"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_8">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_8.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_8.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_9">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_9">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_9.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_9" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_9">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_9}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_9"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_9">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_9.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_9.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_activities_X">
|
|
||||||
<switch>
|
|
||||||
<case to="Gen_Orcid_Authors_DOIs_X">
|
|
||||||
${fs:exists(concat(workingPath_activities,'/ORCID_2019_activites_X.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="Download_X" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="Download_X">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_X}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="Gen_Orcid_Authors_DOIs_X"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="Gen_Orcid_Authors_DOIs_X">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidAuthorsDOIsDataGen</main-class>
|
|
||||||
<arg>-d</arg><arg>${workingPath_activities}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2019_activites_X.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>output/authors_dois_X.seq</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<join name = "join_node" to = "End"/>
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
<workflow-app name="gen_doi_author_list" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>sparkDriverMemory</name>
|
|
||||||
<description>memory for driver process</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>sparkExecutorMemory</name>
|
|
||||||
<description>memory for individual executor</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>sparkExecutorCores</name>
|
|
||||||
<description>number of cores used by single executor</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2MaxExecutors</name>
|
|
||||||
<value>20</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozieActionShareLibForSpark2</name>
|
|
||||||
<description>oozie action sharelib for spark 2.*</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2ExtraListeners</name>
|
|
||||||
<value>com.cloudera.spark.lineage.NavigatorAppListener</value>
|
|
||||||
<description>spark 2.* extra listeners classname</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2SqlQueryExecutionListeners</name>
|
|
||||||
<value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
|
|
||||||
<description>spark 2.* sql query execution listeners classname</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2YarnHistoryServerAddress</name>
|
|
||||||
<description>spark 2.* yarn history server address</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2EventLogDir</name>
|
|
||||||
<description>spark 2.* event log dir location</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>workingPath</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<global>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.spark</name>
|
|
||||||
<value>${oozieActionShareLibForSpark2}</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
</global>
|
|
||||||
|
|
||||||
<start to="ResetWorkingPath"/>
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="ResetWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath}/doi_author_list'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="GenDoiAuthorList"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="GenDoiAuthorList">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn-cluster</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>GenDoiAuthorList</name>
|
|
||||||
<class>eu.dnetlib.doiboost.orcid.SparkGenerateDoiAuthorList</class>
|
|
||||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
--conf spark.dynamicAllocation.enabled=true
|
|
||||||
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
|
|
||||||
</spark-opts>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-a</arg><arg>authors/authors.seq</arg>
|
|
||||||
<arg>-xw</arg><arg>xml/works/*.seq</arg>
|
|
||||||
<arg>-o</arg><arg>doi_author_list/</arg>
|
|
||||||
</spark>
|
|
||||||
<ok to="End"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,232 +0,0 @@
|
||||||
<workflow-app name="Extract Orcid XML Works From Activities" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>workingPath</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<global>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.java</name>
|
|
||||||
<value>${oozieActionShareLibForSpark2}</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.map.java.opts</name>
|
|
||||||
<value>-Xmx2g</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.use.system.libpath</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
</global>
|
|
||||||
|
|
||||||
<start to="ResetWorkingPath"/>
|
|
||||||
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="ResetWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath}/xml/works'/>
|
|
||||||
<mkdir path='${workingPath}/xml/works'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="fork_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<fork name = "fork_node">
|
|
||||||
<path start = "ExtractXMLWorkActivities_0"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_1"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_2"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_3"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_4"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_5"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_6"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_7"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_8"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_9"/>
|
|
||||||
<path start = "ExtractXMLWorkActivities_X"/>
|
|
||||||
</fork>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_0">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_0.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_0.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_1">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_1.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_1.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_2">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_2.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_2.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_3">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_3.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_3.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_4">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_4.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_4.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_5">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_5.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_5.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_6">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_6.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_6.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_7">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_7.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_7.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_8">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_8.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_8.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_9">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_9.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_9.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLWorkActivities_X">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLActivitiesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_activites_X.tar.gz</arg>
|
|
||||||
<arg>-ow</arg><arg>xml/works/xml_works_X.seq</arg>
|
|
||||||
<arg>-oew</arg><arg>---</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="join_node"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<join name = "join_node" to = "End"/>
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>jobTracker</name>
|
|
||||||
<value>yarnRM</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>nameNode</name>
|
|
||||||
<value>hdfs://nameservice1</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.use.system.libpath</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.spark</name>
|
|
||||||
<value>spark2</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.map.java.opts</name>
|
|
||||||
<value>-Xmx8g</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
|
|
@ -1,40 +0,0 @@
|
||||||
<workflow-app name="Extract Orcid XML Authors From Summaries" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>workingPath</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<start to="ResetWorkingPath"/>
|
|
||||||
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="ResetWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath}/xml/authors'/>
|
|
||||||
<mkdir path='${workingPath}/xml/authors'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="ExtractXMLAuthorsSummaries"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ExtractXMLAuthorsSummaries">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.ExtractXMLSummariesData</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_summaries.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>xml/authors/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="End"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>jobTracker</name>
|
|
||||||
<value>yarnRM</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>nameNode</name>
|
|
||||||
<value>hdfs://nameservice1</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.use.system.libpath</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.spark</name>
|
|
||||||
<value>spark2</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
|
||||||
<value>true</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozie.launcher.mapreduce.map.java.opts</name>
|
|
||||||
<value>-Xmx8g</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
|
|
@ -1,68 +0,0 @@
|
||||||
<workflow-app name="Gen Orcid Authors From Summaries" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>workingPath</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>shell_cmd_0</name>
|
|
||||||
<value>wget -O /tmp/ORCID_2020_10_summaries.tar.gz https://orcid.figshare.com/ndownloader/files/25032905 ; hdfs dfs -copyFromLocal /tmp/ORCID_2020_10_summaries.tar.gz /data/orcid_activities_2020/ORCID_2020_10_summaries.tar.gz ; rm -f /tmp/ORCID_2020_10_summaries.tar.gz
|
|
||||||
</value>
|
|
||||||
<description>the shell command that downloads and puts to hdfs orcid summaries</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<start to="ResetWorkingPath"/>
|
|
||||||
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="ResetWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath}/authors'/>
|
|
||||||
<mkdir path='${workingPath}/authors'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="check_exist_on_hdfs_summaries"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<decision name="check_exist_on_hdfs_summaries">
|
|
||||||
<switch>
|
|
||||||
<case to="ImportOrcidSummaries">
|
|
||||||
${fs:exists(concat(workingPath,'/ORCID_2020_10_summaries.tar.gz'))}
|
|
||||||
</case>
|
|
||||||
<default to="DownloadSummaries" />
|
|
||||||
</switch>
|
|
||||||
</decision>
|
|
||||||
|
|
||||||
<action name="DownloadSummaries">
|
|
||||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<exec>bash</exec>
|
|
||||||
<argument>-c</argument>
|
|
||||||
<argument>${shell_cmd_0}</argument>
|
|
||||||
<capture-output/>
|
|
||||||
</shell>
|
|
||||||
<ok to="ImportOrcidSummaries"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="ImportOrcidSummaries">
|
|
||||||
<java>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<main-class>eu.dnetlib.doiboost.orcid.OrcidDSManager</main-class>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>ORCID_2020_10_summaries.tar.gz</arg>
|
|
||||||
<arg>-o</arg><arg>authors/</arg>
|
|
||||||
</java>
|
|
||||||
<ok to="End"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,163 +0,0 @@
|
||||||
<workflow-app name="update_orcid_datasets" xmlns="uri:oozie:workflow:0.5">
|
|
||||||
<parameters>
|
|
||||||
<property>
|
|
||||||
<name>spark2MaxExecutors</name>
|
|
||||||
<value>50</value>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>sparkDriverMemory</name>
|
|
||||||
<description>memory for driver process</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>sparkExecutorMemory</name>
|
|
||||||
<description>memory for individual executor</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>sparkExecutorCores</name>
|
|
||||||
<description>number of cores used by single executor</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>oozieActionShareLibForSpark2</name>
|
|
||||||
<description>oozie action sharelib for spark 2.*</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2ExtraListeners</name>
|
|
||||||
<value>com.cloudera.spark.lineage.NavigatorAppListener</value>
|
|
||||||
<description>spark 2.* extra listeners classname</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2SqlQueryExecutionListeners</name>
|
|
||||||
<value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
|
|
||||||
<description>spark 2.* sql query execution listeners classname</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2YarnHistoryServerAddress</name>
|
|
||||||
<description>spark 2.* yarn history server address</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>spark2EventLogDir</name>
|
|
||||||
<description>spark 2.* event log dir location</description>
|
|
||||||
</property>
|
|
||||||
<property>
|
|
||||||
<name>workingPath</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
|
||||||
|
|
||||||
<global>
|
|
||||||
<job-tracker>${jobTracker}</job-tracker>
|
|
||||||
<name-node>${nameNode}</name-node>
|
|
||||||
<configuration>
|
|
||||||
<property>
|
|
||||||
<name>oozie.action.sharelib.for.spark</name>
|
|
||||||
<value>${oozieActionShareLibForSpark2}</value>
|
|
||||||
</property>
|
|
||||||
</configuration>
|
|
||||||
</global>
|
|
||||||
|
|
||||||
<start to="promoteOrcidAuthorsDataset"/>
|
|
||||||
|
|
||||||
<kill name="Kill">
|
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
|
||||||
</kill>
|
|
||||||
|
|
||||||
<action name="ResetWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath}/orcid_dataset/new_authors'/>
|
|
||||||
<delete path='${workingPath}/orcid_dataset/new_works'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="UpdateOrcidAuthors"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="UpdateOrcidAuthors">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn-cluster</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>UpdateOrcidAuthors</name>
|
|
||||||
<class>eu.dnetlib.doiboost.orcid.SparkUpdateOrcidAuthors</class>
|
|
||||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--conf spark.dynamicAllocation.enabled=true
|
|
||||||
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
</spark-opts>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>-</arg>
|
|
||||||
<arg>-o</arg><arg>-</arg>
|
|
||||||
<arg>-t</arg><arg>-</arg>
|
|
||||||
</spark>
|
|
||||||
<ok to="UpdateOrcidWorks"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="UpdateOrcidWorks">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn-cluster</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>UpdateOrcidWorks</name>
|
|
||||||
<class>eu.dnetlib.doiboost.orcid.SparkUpdateOrcidWorks</class>
|
|
||||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--conf spark.dynamicAllocation.enabled=true
|
|
||||||
--conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors}
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
</spark-opts>
|
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
|
||||||
<arg>-f</arg><arg>-</arg>
|
|
||||||
<arg>-o</arg><arg>-</arg>
|
|
||||||
<arg>-t</arg><arg>-</arg>
|
|
||||||
</spark>
|
|
||||||
<ok to="End"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="promoteOrcidAuthorsDataset">
|
|
||||||
<distcp xmlns="uri:oozie:distcp-action:0.2">
|
|
||||||
<prepare>
|
|
||||||
<delete path="${workingPath}/orcid_dataset/authors"/>
|
|
||||||
<mkdir path="${workingPath}/orcid_dataset/authors"/>
|
|
||||||
</prepare>
|
|
||||||
<arg>${workingPath}/orcid_dataset/new_authors/*</arg>
|
|
||||||
<arg>${workingPath}/orcid_dataset/authors</arg>
|
|
||||||
</distcp>
|
|
||||||
<ok to="promoteOrcidWorksDataset"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="promoteOrcidWorksDataset">
|
|
||||||
<distcp xmlns="uri:oozie:distcp-action:0.2">
|
|
||||||
<prepare>
|
|
||||||
<delete path="${workingPath}/orcid_dataset/works"/>
|
|
||||||
<mkdir path="${workingPath}/orcid_dataset/works"/>
|
|
||||||
</prepare>
|
|
||||||
<arg>${workingPath}/orcid_dataset/new_works/*</arg>
|
|
||||||
<arg>${workingPath}/orcid_dataset/works</arg>
|
|
||||||
</distcp>
|
|
||||||
<ok to="CleanWorkingPath"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="CleanWorkingPath">
|
|
||||||
<fs>
|
|
||||||
<delete path='${workingPath}/orcid_dataset/new_authors'/>
|
|
||||||
<delete path='${workingPath}/orcid_dataset/new_works'/>
|
|
||||||
</fs>
|
|
||||||
<ok to="End"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<end name="End"/>
|
|
||||||
</workflow-app>
|
|
||||||
|
|
@ -1,5 +1,15 @@
|
||||||
<workflow-app name="gen_orcid_no_doi_dataset" xmlns="uri:oozie:workflow:0.5">
|
<workflow-app name="gen_orcid_no_doi_dataset" xmlns="uri:oozie:workflow:0.5">
|
||||||
<parameters>
|
<parameters>
|
||||||
|
<property>
|
||||||
|
<name>workingPath</name>
|
||||||
|
<value>/data/orcid_activities_2020</value>
|
||||||
|
<description>path where the collection workflow stores the ORCID data</description>
|
||||||
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>outputPath</name>
|
||||||
|
<description>path where to store the action set</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>spark2GenNoDoiDatasetMaxExecutors</name>
|
<name>spark2GenNoDoiDatasetMaxExecutors</name>
|
||||||
<value>40</value>
|
<value>40</value>
|
||||||
|
|
@ -35,10 +45,6 @@
|
||||||
<name>spark2EventLogDir</name>
|
<name>spark2EventLogDir</name>
|
||||||
<description>spark 2.* event log dir location</description>
|
<description>spark 2.* event log dir location</description>
|
||||||
</property>
|
</property>
|
||||||
<property>
|
|
||||||
<name>workingPath</name>
|
|
||||||
<description>the working dir base path</description>
|
|
||||||
</property>
|
|
||||||
</parameters>
|
</parameters>
|
||||||
|
|
||||||
<global>
|
<global>
|
||||||
|
|
@ -83,11 +89,20 @@
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>-w</arg><arg>${workingPath}/</arg>
|
<arg>--workingPath</arg><arg>${workingPath}/</arg>
|
||||||
<arg>-n</arg><arg>${nameNode}</arg>
|
<arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
|
||||||
<arg>-i</arg><arg>last_orcid_dataset</arg>
|
<arg>--orcidDataFolder</arg><arg>last_orcid_dataset</arg>
|
||||||
<arg>-oew</arg><arg>no_doi_dataset</arg>
|
<arg>--outputEnrichedWorksPath</arg><arg>no_doi_dataset</arg>
|
||||||
</spark>
|
</spark>
|
||||||
|
<ok to="importOrcidNoDoi"/>
|
||||||
|
<error to="Kill"/>
|
||||||
|
</action>
|
||||||
|
|
||||||
|
<action name="importOrcidNoDoi">
|
||||||
|
<distcp xmlns="uri:oozie:distcp-action:0.2">
|
||||||
|
<arg>${workingPath}/no_doi_dataset/*</arg>
|
||||||
|
<arg>${outputPath}</arg>
|
||||||
|
</distcp>
|
||||||
<ok to="End"/>
|
<ok to="End"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
</action>
|
</action>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue