sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

[orcid] download of the lambda file defined in a script

This commit is contained in:
Claudio Atzori 2021-04-22 11:22:10 +02:00
parent 55964cbd81
commit e5abbec2ba
2 changed files with 5 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_updates_download/oozie_app/get_orcid_lambda_file.sh Normal file
View File

@ -0,0 +1,3 @@
wget -O /tmp/last_modified.csv.tar http://74804fb637bd8e2fba5b-e0a029c2f87486cddec3b416996a6057.r3.cf1.rackcdn.com/last_modified.csv.tar
hdfs dfs -copyFromLocal /tmp/last_modified.csv.tar /data/orcid_activities_2020/last_modified.csv.tar
rm -f /tmp/last_modified.csv.tar

11
dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/orcid_updates_download/oozie_app/workflow.xml
View File

@ -24,12 +24,6 @@
<name>token</name> <name>token</name>
<description>access token</description> <description>access token</description>
</property> </property>
<property>
<name>shell_cmd</name>
<value>wget -O /tmp/last_modified.csv.tar http://74804fb637bd8e2fba5b-e0a029c2f87486cddec3b416996a6057.r3.cf1.rackcdn.com/last_modified.csv.tar ; hdfs dfs -copyFromLocal /tmp/last_modified.csv.tar /data/orcid_activities_2020/last_modified.csv.tar ; rm -f /tmp/last_modified.csv.tar
</value>
<description>the shell command that downloads the lambda file from orcid containing last orcid update informations</description>
</property>
<property> <property>
<name>sparkDriverMemory</name> <name>sparkDriverMemory</name>
<value>7G</value> <value>7G</value>
@ -103,9 +97,8 @@
<shell xmlns="uri:oozie:shell-action:0.1"> <shell xmlns="uri:oozie:shell-action:0.1">
<job-tracker>${jobTracker}</job-tracker> <job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node> <name-node>${nameNode}</name-node>
<exec>bash</exec> <exec>get_orcid_lambda_file.sh</exec>
<argument>-c</argument> <file>get_orcid_lambda_file.sh</file>
<argument>${shell_cmd}</argument>
<capture-output/> <capture-output/>
</shell> </shell>
<ok to="GenLastModifiedSeq"/> <ok to="GenLastModifiedSeq"/>