D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

changed the input params and the workflow definition to tackle the Result as all result product produced

This commit is contained in:
Miriam Baglioni 2020-06-18 11:25:05 +02:00
parent 8211cbb9fe
commit e8b3e972f2
4 changed files with 135 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json
View File

@ -36,18 +36,6 @@
"paramLongName":"resultTableName", "paramLongName":"resultTableName",
"paramDescription": "the name of the result table we are currently working on", "paramDescription": "the name of the result table we are currently working on",
"paramRequired": true "paramRequired": true
},
{
"paramName":"dn",
"paramLongName":"dumpTableName",
"paramDescription": "the name of the corresondent dump element ",
"paramRequired": true
},
{
"paramName":"rt",
"paramLongName":"resultType",
"paramDescription": "the name of the corresondent dump element ",
"paramRequired": false
} }
] ]

219
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/oozie_app/workflow.xml
View File

@ -121,9 +121,7 @@
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/publication</arg> <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
<arg>--resultType</arg><arg>publication</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>
<arg>--outputPath</arg><arg>${workingDir}/publication</arg> <arg>--outputPath</arg><arg>${workingDir}/publication</arg>
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
@ -149,9 +147,7 @@
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg> <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
<arg>--resultType</arg><arg>dataset</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
<arg>--outputPath</arg><arg>${workingDir}/dataset</arg> <arg>--outputPath</arg><arg>${workingDir}/dataset</arg>
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
@ -177,9 +173,7 @@
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg> <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
<arg>--resultType</arg><arg>otherresearchproduct</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
<arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
<arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg> <arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg>
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
@ -205,9 +199,7 @@
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/software</arg> <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
<arg>--resultType</arg><arg>software</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--dumpTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
<arg>--outputPath</arg><arg>${workingDir}/software</arg> <arg>--outputPath</arg><arg>${workingDir}/software</arg>
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
@ -267,7 +259,6 @@
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${workingDir}/publication</arg> <arg>--sourcePath</arg><arg>${workingDir}/publication</arg>
<arg>--outputPath</arg><arg>${workingDir}/ext/publication</arg> <arg>--outputPath</arg><arg>${workingDir}/ext/publication</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg> <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark> </spark>
<ok to="join_extend"/> <ok to="join_extend"/>
@ -293,7 +284,6 @@
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${workingDir}/dataset</arg> <arg>--sourcePath</arg><arg>${workingDir}/dataset</arg>
<arg>--outputPath</arg><arg>${workingDir}/ext/dataset</arg> <arg>--outputPath</arg><arg>${workingDir}/ext/dataset</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg> <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark> </spark>
<ok to="join_extend"/> <ok to="join_extend"/>
@ -318,7 +308,6 @@
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${workingDir}/otherresearchproduct</arg> <arg>--sourcePath</arg><arg>${workingDir}/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${workingDir}/ext/orp</arg> <arg>--outputPath</arg><arg>${workingDir}/ext/orp</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg> <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark> </spark>
<ok to="join_extend"/> <ok to="join_extend"/>
@ -343,25 +332,15 @@
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${workingDir}/software</arg> <arg>--sourcePath</arg><arg>${workingDir}/software</arg>
<arg>--outputPath</arg><arg>${workingDir}/ext/software</arg> <arg>--outputPath</arg><arg>${workingDir}/ext/software</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg> <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark> </spark>
<ok to="join_extend"/> <ok to="join_extend"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
<join name="join_extend" to="splitForCommunities"/>
<join name="join_extend" to="fork_splitForCommunities"/> <action name="splitForCommunities">
<fork name="fork_splitForCommunities">
<path start="split_publication"/>
<path start="split_dataset"/>
<path start="split_orp"/>
<path start="split_software"/>
</fork>
<action name="split_publication">
<spark xmlns="uri:oozie:spark-action:0.2"> <spark xmlns="uri:oozie:spark-action:0.2">
<master>yarn</master> <master>yarn</master>
<mode>cluster</mode> <mode>cluster</mode>
@ -378,93 +357,129 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
</spark-opts> </spark-opts>
<arg>--sourcePath</arg><arg>${workingDir}/ext/publication</arg> <arg>--sourcePath</arg><arg>${workingDir}/ext</arg>
<arg>--outputPath</arg><arg>${outputPath}</arg> <arg>--outputPath</arg><arg>${outputPath}</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg> <!-- <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>-->
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
<ok to="join_split"/> <ok to="End"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>
<action name="split_dataset"> <!-- <join name="join_extend" to="fork_splitForCommunities"/>-->
<spark xmlns="uri:oozie:spark-action:0.2">
<master>yarn</master>
<mode>cluster</mode> <!-- <fork name="fork_splitForCommunities">-->
<name>Split dumped result for community</name> <!-- <path start="split_publication"/>-->
<class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class> <!-- <path start="split_dataset"/>-->
<jar>dhp-graph-mapper-${projectVersion}.jar</jar> <!-- <path start="split_orp"/>-->
<spark-opts> <!-- <path start="split_software"/>-->
--executor-memory=${sparkExecutorMemory} <!-- </fork>-->
--executor-cores=${sparkExecutorCores}
--driver-memory=${sparkDriverMemory} <!-- <action name="split_publication">-->
--conf spark.extraListeners=${spark2ExtraListeners} <!-- <spark xmlns="uri:oozie:spark-action:0.2">-->
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} <!-- <master>yarn</master>-->
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} <!-- <mode>cluster</mode>-->
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} <!-- <name>Split dumped result for community</name>-->
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} <!-- <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
</spark-opts> <!-- <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
<arg>--sourcePath</arg><arg>${workingDir}/ext/dataset</arg> <!-- <spark-opts>-->
<arg>--outputPath</arg><arg>${outputPath}</arg> <!-- &#45;&#45;executor-memory=${sparkExecutorMemory}-->
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg> <!-- &#45;&#45;executor-cores=${sparkExecutorCores}-->
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <!-- &#45;&#45;driver-memory=${sparkDriverMemory}-->
</spark> <!-- &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
<ok to="join_split"/> <!-- &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
<error to="Kill"/> <!-- &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
</action> <!-- &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
<action name="split_orp"> <!-- &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
<spark xmlns="uri:oozie:spark-action:0.2"> <!-- </spark-opts>-->
<master>yarn</master> <!-- <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/publication</arg>-->
<mode>cluster</mode> <!-- <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
<name>Split dumped result for community</name> <!-- <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Publication</arg>-->
<class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class> <!-- <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
<jar>dhp-graph-mapper-${projectVersion}.jar</jar> <!-- </spark>-->
<spark-opts> <!-- <ok to="join_split"/>-->
--executor-memory=${sparkExecutorMemory} <!-- <error to="Kill"/>-->
--executor-cores=${sparkExecutorCores} <!-- </action>-->
--driver-memory=${sparkDriverMemory}
--conf spark.extraListeners=${spark2ExtraListeners} <!-- <action name="split_dataset">-->
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} <!-- <spark xmlns="uri:oozie:spark-action:0.2">-->
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} <!-- <master>yarn</master>-->
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} <!-- <mode>cluster</mode>-->
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} <!-- <name>Split dumped result for community</name>-->
</spark-opts> <!-- <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
<arg>--sourcePath</arg><arg>${workingDir}/ext/orp</arg> <!-- <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
<arg>--outputPath</arg><arg>${outputPath}</arg> <!-- <spark-opts>-->
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg> <!-- &#45;&#45;executor-memory=${sparkExecutorMemory}-->
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <!-- &#45;&#45;executor-cores=${sparkExecutorCores}-->
</spark> <!-- &#45;&#45;driver-memory=${sparkDriverMemory}-->
<ok to="join_split"/> <!-- &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
<error to="Kill"/> <!-- &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
</action> <!-- &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
<action name="split_software"> <!-- &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
<spark xmlns="uri:oozie:spark-action:0.2"> <!-- &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
<master>yarn</master> <!-- </spark-opts>-->
<mode>cluster</mode> <!-- <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/dataset</arg>-->
<name>Split dumped result for community</name> <!-- <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
<class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class> <!-- <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Dataset</arg>-->
<jar>dhp-graph-mapper-${projectVersion}.jar</jar> <!-- <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
<spark-opts> <!-- </spark>-->
--executor-memory=${sparkExecutorMemory} <!-- <ok to="join_split"/>-->
--executor-cores=${sparkExecutorCores} <!-- <error to="Kill"/>-->
--driver-memory=${sparkDriverMemory} <!-- </action>-->
--conf spark.extraListeners=${spark2ExtraListeners} <!-- <action name="split_orp">-->
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} <!-- <spark xmlns="uri:oozie:spark-action:0.2">-->
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} <!-- <master>yarn</master>-->
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} <!-- <mode>cluster</mode>-->
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} <!-- <name>Split dumped result for community</name>-->
</spark-opts> <!-- <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
<arg>--sourcePath</arg><arg>${workingDir}/ext/software</arg> <!-- <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
<arg>--outputPath</arg><arg>${outputPath}</arg> <!-- <spark-opts>-->
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg> <!-- &#45;&#45;executor-memory=${sparkExecutorMemory}-->
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg> <!-- &#45;&#45;executor-cores=${sparkExecutorCores}-->
</spark> <!-- &#45;&#45;driver-memory=${sparkDriverMemory}-->
<ok to="join_split"/> <!-- &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
<error to="Kill"/> <!-- &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
</action> <!-- &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
<!-- &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
<!-- &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
<!-- </spark-opts>-->
<!-- <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/orp</arg>-->
<!-- <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
<!-- <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.OtherResearchProduct</arg>-->
<!-- <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
<!-- </spark>-->
<!-- <ok to="join_split"/>-->
<!-- <error to="Kill"/>-->
<!-- </action>-->
<!-- <action name="split_software">-->
<!-- <spark xmlns="uri:oozie:spark-action:0.2">-->
<!-- <master>yarn</master>-->
<!-- <mode>cluster</mode>-->
<!-- <name>Split dumped result for community</name>-->
<!-- <class>eu.dnetlib.dhp.oa.graph.dump.SparkSplitForCommunity</class>-->
<!-- <jar>dhp-graph-mapper-${projectVersion}.jar</jar>-->
<!-- <spark-opts>-->
<!-- &#45;&#45;executor-memory=${sparkExecutorMemory}-->
<!-- &#45;&#45;executor-cores=${sparkExecutorCores}-->
<!-- &#45;&#45;driver-memory=${sparkDriverMemory}-->
<!-- &#45;&#45;conf spark.extraListeners=${spark2ExtraListeners}-->
<!-- &#45;&#45;conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}-->
<!-- &#45;&#45;conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}-->
<!-- &#45;&#45;conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}-->
<!-- &#45;&#45;conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}-->
<!-- </spark-opts>-->
<!-- <arg>&#45;&#45;sourcePath</arg><arg>${workingDir}/ext/software</arg>-->
<!-- <arg>&#45;&#45;outputPath</arg><arg>${outputPath}</arg>-->
<!-- <arg>&#45;&#45;resultTableName</arg><arg>eu.dnetlib.dhp.schema.dump.oaf.Software</arg>-->
<!-- <arg>&#45;&#45;isLookUpUrl</arg><arg>${isLookUpUrl}</arg>-->
<!-- </spark>-->
<!-- <ok to="join_split"/>-->
<!-- <error to="Kill"/>-->
<!-- </action>-->
<!-- <join name="join_split" to="loadInZenodo"/>--> <!-- <join name="join_split" to="loadInZenodo"/>-->
<join name="join_split" to="End"/> <!-- <join name="join_split" to="End"/>-->
<!-- <action name="loadInZenodo">--> <!-- <action name="loadInZenodo">-->
<!-- <spark xmlns="uri:oozie:spark-action:0.2">--> <!-- <spark xmlns="uri:oozie:spark-action:0.2">-->

6
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/project_input_parameters.json
View File

@ -19,12 +19,6 @@
"paramDescription": "true if the spark session is managed, false otherwise", "paramDescription": "true if the spark session is managed, false otherwise",
"paramRequired": false "paramRequired": false
}, },
{
"paramName":"tn",
"paramLongName":"resultTableName",
"paramDescription": "the name of the result table we are currently working on",
"paramRequired": true
},
{ {
"paramName": "pip", "paramName": "pip",
"paramLongName": "preparedInfoPath", "paramLongName": "preparedInfoPath",

6
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json
View File

@ -24,12 +24,6 @@
"paramDescription": "true if the spark session is managed, false otherwise", "paramDescription": "true if the spark session is managed, false otherwise",
"paramRequired": false "paramRequired": false
}, },
{
"paramName":"tn",
"paramLongName":"resultTableName",
"paramDescription": "the name of the result table we are currently working on",
"paramRequired": true
},
{ {
"paramName":"map", "paramName":"map",
"paramLongName":"communityMap", "paramLongName":"communityMap",