miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

added decision node to verify if to upload in Zenodo

This commit is contained in:
Miriam Baglioni 2020-11-25 13:44:10 +01:00
parent 305e3d0c9c
commit e7e418e444
1 changed files with 27 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/funderresults/oozie_app/workflow.xml
View File

@ -1,6 +1,11 @@
<workflow-app name="dump_funder_results" xmlns="uri:oozie:workflow:0.5"> <workflow-app name="dump_funder_results" xmlns="uri:oozie:workflow:0.5">
<parameters> <parameters>
<property>
<name>upload</name>
<value>false</value>
<description>true to upload the dump for the funders in Zenodo</description>
</property>
<property> <property>
<name>sourcePath</name> <name>sourcePath</name>
<description>the source path</description> <description>the source path</description>
@ -116,7 +121,7 @@
<delete path="${outputPath}"/> <delete path="${outputPath}"/>
<mkdir path="${outputPath}"/> <mkdir path="${outputPath}"/>
</fs> </fs>
<ok to="save_community_map"/> <ok to="make_archive"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
@ -515,13 +520,32 @@
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${workingDir}/ext</arg> <arg>--sourcePath</arg><arg>${workingDir}/ext</arg>
<arg>--outputPath</arg><arg>${outputPath}</arg> <arg>--outputPath</arg><arg>${workingDir}/resultperfunder</arg>
<arg>--relationPath</arg><arg>${sourcePath}</arg> <arg>--relationPath</arg><arg>${sourcePath}</arg>
</spark> </spark>
<ok to="End"/> <ok to="make_archive"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
<action name="make_archive">
<java>
<main-class>eu.dnetlib.dhp.oa.graph.dump.MakeTar</main-class>
<arg>--hdfsPath</arg><arg>${outputPath}</arg>
<arg>--nameNode</arg><arg>${nameNode}</arg>
<!-- <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/resultperfunder</arg>-->
<arg>--sourcePath</arg><arg>/user/miriam.baglioni/graph_dump_whole_production_funder_results</arg>
</java>
<ok to="should_upload"/>
<error to="Kill"/>
</action>
<decision name="should_upload">
<switch>
<case to="send_zenodo">${wf:conf('upload') eq true}</case>
<default to="End"/>
</switch>
</decision>
<action name="send_zenodo"> <action name="send_zenodo">
<java> <java>
<main-class>eu.dnetlib.dhp.oa.graph.dump.SendToZenodoHDFS</main-class> <main-class>eu.dnetlib.dhp.oa.graph.dump.SendToZenodoHDFS</main-class>