miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

added the reader for the dump of crossref

This commit is contained in:
Miriam Baglioni 2021-06-21 09:22:29 +02:00
parent c07f820c21
commit 464ac6301c
2 changed files with 21 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref_dump_reader.json
View File

@ -0,0 +1,7 @@
[
{"paramName":"n", "paramLongName":"hdfsServerUri", "paramDescription": "the server uri", "paramRequired": true},
{"paramName":"w", "paramLongName":"workingPath", "paramDescription": "the default work path", "paramRequired": true},
{"paramName":"f", "paramLongName":"crossrefFileNameTarGz", "paramDescription": "the name of the activities orcid file", "paramRequired": true},
{"paramName":"issm", "paramLongName":"isSparkSessionManaged", "paramDescription": "the name of the activities orcid file", "paramRequired": false}
]

29
dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref_dump_reader/oozie_app/workflow.xml
View File

@ -1,9 +1,9 @@
<workflow-app name="read Crossref dump from HDFS" xmlns="uri:oozie:workflow:0.5"> <workflow-app name="read Crossref dump from HDFS" xmlns="uri:oozie:workflow:0.5">
<parameters> <parameters>
<property> <!-- <property>-->
<name>workingPath</name> <!-- <name>workingPath</name>-->
<description>the working dir base path</description> <!-- <description>the working dir base path</description>-->
</property> <!-- </property>-->
<property> <property>
<name>sparkDriverMemory</name> <name>sparkDriverMemory</name>
<description>memory for driver process</description> <description>memory for driver process</description>
@ -19,7 +19,7 @@
</parameters> </parameters>
<start to="SparkReadCrossRefDump"/> <start to="ReadCrossRefDump"/>
<kill name="Kill"> <kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
@ -29,11 +29,10 @@
<java> <java>
<job-tracker>${jobTracker}</job-tracker> <job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node> <name-node>${nameNode}</name-node>
<main-class>eu.dnetlib.doiboost.crossref.CrossrefDumpReader</main-class> <main-class>eu.dnetlib.doiboost.crossref.ExtractCrossrefRecords</main-class>
<arg>-n</arg><arg>${nameNode}</arg> <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
<arg>-w</arg><arg>/data/doiboost/crossref/</arg> <arg>--workingPath</arg><arg>/data/doiboost/crossref/</arg>
<arg>-f</arg><arg>crossref.tar.gz</arg> <arg>--crossrefFileNameTarGz</arg><arg>crossref.tar.gz</arg>
<arg>-o</arg><arg>/user/enrico.ottonello/crossref/</arg>
</java> </java>
<ok to="End"/> <ok to="End"/>
<error to="Kill"/> <error to="Kill"/>
@ -44,7 +43,7 @@
<master>yarn-cluster</master> <master>yarn-cluster</master>
<mode>cluster</mode> <mode>cluster</mode>
<name>SparkReadCrossRefDump</name> <name>SparkReadCrossRefDump</name>
<class>eu.dnetlib.doiboost.crossref.SparkExtractCrossrefRecords</class> <class>eu.dnetlib.doiboost.crossref.ExtractCrossrefRecords</class>
<jar>dhp-doiboost-${projectVersion}.jar</jar> <jar>dhp-doiboost-${projectVersion}.jar</jar>
<spark-opts> <spark-opts>
--conf spark.dynamicAllocation.enabled=true --conf spark.dynamicAllocation.enabled=true
@ -56,10 +55,10 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
</spark-opts> </spark-opts>
<arg>-n</arg><arg>${nameNode}</arg> <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
<arg>-w</arg><arg>/data/doiboost/crossref/</arg> <arg>--workingPath</arg><arg>/data/doiboost/crossref/</arg>
<arg>-f</arg><arg>crossref.tar.gz</arg> <arg>--crossrefFileNameTarGz</arg><arg>crossref.tar.gz</arg>
<arg>-o</arg><arg>/data/doiboost/crossref/</arg>
</spark> </spark>
<ok to="End"/> <ok to="End"/>
<error to="Kill"/> <error to="Kill"/>