new wf definition

This commit is contained in:
Miriam Baglioni 2020-04-09 11:33:19 +02:00
parent 9c63c4840d
commit 627ad58a8b
4 changed files with 33 additions and 35 deletions

View File

@ -34,5 +34,17 @@
"paramLongName":"resultTableName", "paramLongName":"resultTableName",
"paramDescription": "the name of the result table we are currently working on", "paramDescription": "the name of the result table we are currently working on",
"paramRequired": true "paramRequired": true
},
{
"paramName": "out",
"paramLongName": "outputPath",
"paramDescription": "the path used to store temporary output files",
"paramRequired": true
},
{
"paramName": "p",
"paramLongName": "preparedInfoPath",
"paramDescription": "the path where prepared info have been stored",
"paramRequired": true
} }
] ]

View File

@ -1,32 +0,0 @@
[
{
"paramName":"mt",
"paramLongName":"master",
"paramDescription": "should be local or yarn",
"paramRequired": true
},
{
"paramName":"s",
"paramLongName":"sourcePath",
"paramDescription": "the path of the sequencial file to read",
"paramRequired": true
},
{
"paramName":"wl",
"paramLongName":"whitelist",
"paramDescription": "datasource id that will be considered even if not in the allowed typology list. Split by ;",
"paramRequired": true
},
{
"paramName":"at",
"paramLongName":"allowedtypes",
"paramDescription": "the types of the allowed datasources. Split by ;",
"paramRequired": true
},
{
"paramName":"h",
"paramLongName":"hive_metastore_uris",
"paramDescription": "the hive metastore uris",
"paramRequired": true
}
]

View File

@ -36,6 +36,15 @@
<name>saveGraph</name> <name>saveGraph</name>
<description>writes new version of the graph after the propagation step</description> <description>writes new version of the graph after the propagation step</description>
</property> </property>
<property>
<name>outputPath</name>
<description>the path used to store temporary output files</description>
</property>
<property>
<name>preparedInfoPath</name>
<description>the path where prepared info have been stored</description>
</property>
</parameters> </parameters>
<start to="prepare_datasource_country_association"/> <start to="prepare_datasource_country_association"/>
@ -65,13 +74,14 @@
<arg>--whitelist</arg><arg>${whitelist}</arg> <arg>--whitelist</arg><arg>${whitelist}</arg>
<arg>--allowedtypes</arg><arg>${allowedtypes}</arg> <arg>--allowedtypes</arg><arg>${allowedtypes}</arg>
<arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg> <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
</spark> </spark>
<ok to="fork_join_apply_country_propagation"/> <ok to="fork_join_apply_country_propagation"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
<fork name="fork_join_apply_country_propagation"> <fork name="fork_join_apply_country_propagation">
<path start="join_propagation_publication"/> <!-- <path start="join_propagation_publication"/>-->
<path start="join_propagation_dataset"/> <!-- <path start="join_propagation_dataset"/>-->
<path start="join_propagation_otherresearchproduct"/> <path start="join_propagation_otherresearchproduct"/>
<path start="join_propagation_software"/> <path start="join_propagation_software"/>
</fork> </fork>
@ -98,6 +108,8 @@
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg> <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
<arg>--saveGraph</arg><arg>${saveGraph}</arg> <arg>--saveGraph</arg><arg>${saveGraph}</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
</spark> </spark>
<ok to="end"/> <ok to="end"/>
<error to="Kill"/> <error to="Kill"/>
@ -125,6 +137,8 @@
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg> <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
<arg>--saveGraph</arg><arg>${saveGraph}</arg> <arg>--saveGraph</arg><arg>${saveGraph}</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
</spark> </spark>
<ok to="end"/> <ok to="end"/>
<error to="Kill"/> <error to="Kill"/>
@ -152,6 +166,8 @@
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg> <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
<arg>--saveGraph</arg><arg>${saveGraph}</arg> <arg>--saveGraph</arg><arg>${saveGraph}</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Otherresearchproduct</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
</spark> </spark>
<ok to="end"/> <ok to="end"/>
<error to="Kill"/> <error to="Kill"/>
@ -179,11 +195,13 @@
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg> <arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
<arg>--saveGraph</arg><arg>${saveGraph}</arg> <arg>--saveGraph</arg><arg>${saveGraph}</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
</spark> </spark>
<ok to="end"/> <ok to="end"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
<action name="CountryPropagation"> <action name="CountryPropagation">
<spark xmlns="uri:oozie:spark-action:0.2"> <spark xmlns="uri:oozie:spark-action:0.2">
<job-tracker>${jobTracker}</job-tracker> <job-tracker>${jobTracker}</job-tracker>