forked from D-Net/dnet-hadoop
added parameter to guide the dump for the result (resultAggregation). true if all the result types should be dump together, false otherwise.
This commit is contained in:
parent
d407852ac2
commit
c5858afb88
|
@ -17,7 +17,13 @@
|
||||||
"paramLongName": "isSparkSessionManaged",
|
"paramLongName": "isSparkSessionManaged",
|
||||||
"paramDescription": "true if the spark session is managed, false otherwise",
|
"paramDescription": "true if the spark session is managed, false otherwise",
|
||||||
"paramRequired": false
|
"paramRequired": false
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
"paramName": "ra",
|
||||||
|
"paramLongName": "resultAggregation",
|
||||||
|
"paramDescription": "true if all the result type should be saved under the generic result name. false to get a different dump for each result type",
|
||||||
|
"paramRequired": true
|
||||||
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -13,6 +13,10 @@
|
||||||
<name>outputPath</name>
|
<name>outputPath</name>
|
||||||
<description>the output path</description>
|
<description>the output path</description>
|
||||||
</property>
|
</property>
|
||||||
|
<property>
|
||||||
|
<name>resultAggregation</name>
|
||||||
|
<description>true if all the result type have to be dumped under result. false otherwise</description>
|
||||||
|
</property>
|
||||||
<property>
|
<property>
|
||||||
<name>accessToken</name>
|
<name>accessToken</name>
|
||||||
<description>the access token used for the deposition in Zenodo</description>
|
<description>the access token used for the deposition in Zenodo</description>
|
||||||
|
@ -268,7 +272,6 @@
|
||||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Organization</arg>
|
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Organization</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/collect/organization</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/collect/organization</arg>
|
||||||
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<!-- <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_dump"/>
|
<ok to="join_dump"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -295,7 +298,6 @@
|
||||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg>
|
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/collect/project</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/collect/project</arg>
|
||||||
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<!-- <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_dump"/>
|
<ok to="join_dump"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -322,7 +324,6 @@
|
||||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Datasource</arg>
|
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Datasource</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/collect/datasource</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/collect/datasource</arg>
|
||||||
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<!-- <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_dump"/>
|
<ok to="join_dump"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -441,7 +442,6 @@
|
||||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
|
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/relation/publication</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/relation/publication</arg>
|
||||||
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<!-- <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extract_relations"/>
|
<ok to="join_extract_relations"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -468,7 +468,6 @@
|
||||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
|
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/relation/dataset</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/relation/dataset</arg>
|
||||||
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<!-- <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extract_relations"/>
|
<ok to="join_extract_relations"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -495,7 +494,6 @@
|
||||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
|
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/relation/orp</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/relation/orp</arg>
|
||||||
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<!-- <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extract_relations"/>
|
<ok to="join_extract_relations"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -522,7 +520,6 @@
|
||||||
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
|
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/relation/software</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/relation/software</arg>
|
||||||
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<!-- <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extract_relations"/>
|
<ok to="join_extract_relations"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -561,6 +558,7 @@
|
||||||
<arg>--hdfsPath</arg><arg>${outputPath}</arg>
|
<arg>--hdfsPath</arg><arg>${outputPath}</arg>
|
||||||
<arg>--nameNode</arg><arg>${nameNode}</arg>
|
<arg>--nameNode</arg><arg>${nameNode}</arg>
|
||||||
<arg>--sourcePath</arg><arg>${workingDir}/collect</arg>
|
<arg>--sourcePath</arg><arg>${workingDir}/collect</arg>
|
||||||
|
<arg>--resultAggregation</arg><arg>${resultAggregation}</arg>
|
||||||
</java>
|
</java>
|
||||||
<ok to="send_zenodo"/>
|
<ok to="send_zenodo"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
|
Loading…
Reference in New Issue