forked from D-Net/dnet-hadoop
new wf definition
This commit is contained in:
parent
9c63c4840d
commit
627ad58a8b
|
@ -34,5 +34,17 @@
|
|||
"paramLongName":"resultTableName",
|
||||
"paramDescription": "the name of the result table we are currently working on",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "out",
|
||||
"paramLongName": "outputPath",
|
||||
"paramDescription": "the path used to store temporary output files",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "p",
|
||||
"paramLongName": "preparedInfoPath",
|
||||
"paramDescription": "the path where prepared info have been stored",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
|
@ -1,32 +0,0 @@
|
|||
[
|
||||
{
|
||||
"paramName":"mt",
|
||||
"paramLongName":"master",
|
||||
"paramDescription": "should be local or yarn",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName":"s",
|
||||
"paramLongName":"sourcePath",
|
||||
"paramDescription": "the path of the sequencial file to read",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName":"wl",
|
||||
"paramLongName":"whitelist",
|
||||
"paramDescription": "datasource id that will be considered even if not in the allowed typology list. Split by ;",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName":"at",
|
||||
"paramLongName":"allowedtypes",
|
||||
"paramDescription": "the types of the allowed datasources. Split by ;",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName":"h",
|
||||
"paramLongName":"hive_metastore_uris",
|
||||
"paramDescription": "the hive metastore uris",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
|
@ -36,6 +36,15 @@
|
|||
<name>saveGraph</name>
|
||||
<description>writes new version of the graph after the propagation step</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>outputPath</name>
|
||||
<description>the path used to store temporary output files</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>preparedInfoPath</name>
|
||||
<description>the path where prepared info have been stored</description>
|
||||
</property>
|
||||
|
||||
</parameters>
|
||||
|
||||
<start to="prepare_datasource_country_association"/>
|
||||
|
@ -65,13 +74,14 @@
|
|||
<arg>--whitelist</arg><arg>${whitelist}</arg>
|
||||
<arg>--allowedtypes</arg><arg>${allowedtypes}</arg>
|
||||
<arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
|
||||
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
|
||||
</spark>
|
||||
<ok to="fork_join_apply_country_propagation"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
<fork name="fork_join_apply_country_propagation">
|
||||
<path start="join_propagation_publication"/>
|
||||
<path start="join_propagation_dataset"/>
|
||||
<!-- <path start="join_propagation_publication"/>-->
|
||||
<!-- <path start="join_propagation_dataset"/>-->
|
||||
<path start="join_propagation_otherresearchproduct"/>
|
||||
<path start="join_propagation_software"/>
|
||||
</fork>
|
||||
|
@ -98,6 +108,8 @@
|
|||
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
|
||||
<arg>--saveGraph</arg><arg>${saveGraph}</arg>
|
||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
|
||||
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
|
||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
|
||||
</spark>
|
||||
<ok to="end"/>
|
||||
<error to="Kill"/>
|
||||
|
@ -125,6 +137,8 @@
|
|||
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
|
||||
<arg>--saveGraph</arg><arg>${saveGraph}</arg>
|
||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
|
||||
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
|
||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
|
||||
</spark>
|
||||
<ok to="end"/>
|
||||
<error to="Kill"/>
|
||||
|
@ -152,6 +166,8 @@
|
|||
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
|
||||
<arg>--saveGraph</arg><arg>${saveGraph}</arg>
|
||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Otherresearchproduct</arg>
|
||||
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
|
||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
|
||||
</spark>
|
||||
<ok to="end"/>
|
||||
<error to="Kill"/>
|
||||
|
@ -179,11 +195,13 @@
|
|||
<arg>--writeUpdate</arg><arg>${writeUpdate}</arg>
|
||||
<arg>--saveGraph</arg><arg>${saveGraph}</arg>
|
||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
|
||||
<arg>--outputPath</arg><arg>${workingDir}/country_propagation/publication</arg>
|
||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/country_propagation/preparedInfo</arg>
|
||||
</spark>
|
||||
<ok to="end"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
|
||||
<action name="CountryPropagation">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
|
|
Loading…
Reference in New Issue