miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

fixed issues on wf definition and proeprty name

This commit is contained in:
Miriam Baglioni 2020-11-30 12:08:33 +01:00
parent 02589717b0
commit a9cc70d3b0
4 changed files with 19 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/authorpids/MakeReportSparkJob.java
View File

@ -98,7 +98,7 @@ public class MakeReportSparkJob implements Serializable {
.orElse(Boolean.TRUE);
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
String outputPath = parser.get("reportOutputPath");
String outputPath = parser.get("outputPath");
log.info("outputPath: {}", outputPath);
String preparedInfoPath = parser.get("preparedInfoPath");

43
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml
View File

@ -311,12 +311,12 @@
<decision name="should_emend_orcid">
<switch>
<case to="perpare_result">${wf:conf('emend') eq true}</case>
<case to="prepare_result">${wf:conf('emend') eq true}</case>
<default to="End"/>
</switch>
</decision>
<fork name="perpare_result">
<fork name="prepare_result">
<path start="prepare_publication"/>
<path start="prepare_dataset"/>
<path start="prepare_software"/>
@ -343,7 +343,6 @@
<arg>--inputPath</arg><arg>${workingDir}/cleaned/publication</arg>
<arg>--outputPath</arg><arg>${workingDir}/prepared/publication</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
</spark>
<ok to="wait_prepare"/>
<error to="Kill"/>
@ -369,7 +368,6 @@
<arg>--inputPath</arg><arg>${workingDir}/cleaned/dataset</arg>
<arg>--outputPath</arg><arg>${workingDir}/prepared/dataset</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
</spark>
<ok to="wait_prepare"/>
<error to="Kill"/>
@ -395,7 +393,6 @@
<arg>--inputPath</arg><arg>${workingDir}/cleaned/software</arg>
<arg>--outputPath</arg><arg>${workingDir}/prepared/software</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
</spark>
<ok to="wait_prepare"/>
<error to="Kill"/>
@ -421,21 +418,13 @@
<arg>--inputPath</arg><arg>${workingDir}/cleaned/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${workingDir}/prepared/otherresearchproduct</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
</spark>
<ok to="wait_prepare"/>
<error to="Kill"/>
</action>
<join name="wait_prepare" to="cleanorreport"/>
<join name="wait_prepare" to="make_report"/>
<decision name="cleanorreport">
<switch>
<case to="make_report">${wf:conf('clean') eq false}</case>
<case to="clean_orcid_copy">${wf:conf('clean') eq true}</case>
<default to="make_report"/>
</switch>
</decision>
<fork name="make_report">
<path start="report_publication"/>
@ -461,12 +450,10 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--inputPath</arg><arg>${workingDir}/cleaned/publication</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/publication</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/publication</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
</spark>
<ok to="wait_report"/>
<error to="Kill"/>
@ -491,8 +478,6 @@
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/dataset</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/dataset</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--inputPath</arg><arg>${workingDir}/cleaned/dataset</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
</spark>
@ -518,9 +503,7 @@
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>$workingDir}/report/otherresearchproduct</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
<arg>--inputPath</arg><arg>${workingDir}/cleaned/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/otherresearchproduct</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
</spark>
@ -546,9 +529,7 @@
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/software</arg>
<arg>--outputPath</arg><arg>$workingDir}/report/software</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--inputPath</arg><arg>${workingDir}/cleaned/software</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/software</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
</spark>
@ -586,8 +567,7 @@
<arg>--inputPath</arg><arg>${workingDir}/cleaned/publication</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/publication</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/publication</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/publication</arg>
</spark>
<ok to="wait_clean"/>
<error to="Kill"/>
@ -610,11 +590,10 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/dataset</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/dataset</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/dataset</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--inputPath</arg><arg>${workingDir}/cleaned/dataset</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
</spark>
<ok to="wait_clean"/>
<error to="Kill"/>
@ -637,11 +616,10 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/otherresearchproduct</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/otherresearchproduct</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
<arg>--inputPath</arg><arg>${workingDir}/cleaned/otherresearchproduct</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
</spark>
<ok to="wait_clean"/>
<error to="Kill"/>
@ -664,11 +642,10 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/prepared/software</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/software</arg>
<arg>--outputPath</arg><arg>${graphOutputPath}/software</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--inputPath</arg><arg>${workingDir}/cleaned/software</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
</spark>
<ok to="wait_clean"/>
<error to="Kill"/>

16
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean_orcid/oozie_app/workflow.xml
View File

@ -222,7 +222,7 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--reportOutputPath</arg><arg>${reportOutputPath}/publication</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/publication</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
@ -249,7 +249,7 @@
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/dataset</arg>
<arg>--reportOutputPath</arg><arg>${reportOutputPath}/dataset</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/dataset</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
</spark>
@ -275,7 +275,7 @@
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/otherresearchproduct</arg>
<arg>--reportOutputPath</arg><arg>${reportOutputPath}/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/otherresearchproduct</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
</spark>
@ -301,7 +301,7 @@
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--preparedInfoPath</arg><arg>${workingDir}/software</arg>
<arg>--reportOutputPath</arg><arg>${reportOutputPath}/software</arg>
<arg>--outputPath</arg><arg>${workingDir}/report/software</arg>
<arg>--orcidInputPath</arg><arg>${orcidInputPath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
</spark>
@ -338,7 +338,7 @@
<arg>--inputPath</arg><arg>${inputPath}/publication</arg>
<arg>--outputPath</arg><arg>${outputPath}/publication</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--reportPath</arg><arg>${reportOutputPath}/publication</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/publication</arg>
</spark>
<ok to="wait_clean"/>
<error to="Kill"/>
@ -361,7 +361,7 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--reportPath</arg><arg>${reportOutputPath}/dataset</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/dataset</arg>
<arg>--outputPath</arg><arg>${outputPath}/dataset</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--inputPath</arg><arg>${inputPath}/dataset</arg>
@ -387,7 +387,7 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--reportPath</arg><arg>${reportOutputPath}/otherresearchproduct</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${outputPath}/otherresearchproduct</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
<arg>--inputPath</arg><arg>${inputPath}/otherresearchproduct</arg>
@ -413,7 +413,7 @@
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
--conf spark.sql.shuffle.partitions=7680
</spark-opts>
<arg>--reportPath</arg><arg>${reportOutputPath}/software</arg>
<arg>--reportPath</arg><arg>${workingDir}/report/software</arg>
<arg>--outputPath</arg><arg>${outputPath}/software</arg>
<arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--inputPath</arg><arg>${inputPath}/software</arg>

35
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/wrongassociation.json
View File

@ -1,35 +0,0 @@
{"orcid":"Alex Bullock" ,"result": "Gillian Farnie"}
{"orcid": "Luís Rocha", "result":"Pedro Relvas"}
{"orcid": "Prasanth Manohar", "result": "Nachimuthu Ramesh"}
{"orcid": "Zhiying Lin", "result":"Guanglong Huang"}
{"orcid":"Andrew Golnar","result":"Kim Pepin"}
{"orcid": "Gilles Marcou", "result":"Filippo Lunghini"}
{"orcid": "Philip Hahn", "result":"John Maron"}
{"orcid": "Kirsty Gibson", "result":"Kim R. Hardie"}
{"orcid": "Paula Lago", "result":"Shingo Takeda"}
{"orcid": "Paul Seidler", "result":"Dalziel J. Wilson"}
{"orcid": "Solomon Okunade", "result":"Rufus Adebayo Ajisafe"}
{"orcid": "Emi Arai", "result":"Masaru Hasegawa"}
{"orcid": "Dr Muhammad Yameen Sandhu", "result":"Nutapong Somjit"}
{"orcid": "Xianlei Cai", "result":"Weiming Yu"}
{"orcid": "Bing He", "result":"Chuan Xing"}
{"orcid": "JULIEN COURCHET", "result":"Franck Polleux"}
{"orcid": "Xiaoyun Pan", "result":"Liru Chen"}
{"orcid": "Marianne Okal", "result":"Brendan Hodge"}
{"orcid": "Michal Fereczkowski", "result":"Silje Grini Nielsen"}
{"orcid": "Nobuyuki Nakai", "result":"Tadafumi Kurogi"}
{"orcid": "Colin Daniel", "result":"Christine Cuyler"}
{"orcid": "Xavier Arnan", "result":"Anna Torné-Noguera"}
{"orcid": "Denita Hadziabdic", "result":"Meher Ony"}
{"orcid": "Kor de Jong", "result":"K. Koning"}
{"orcid": "Chaya Patel", "result":"David Leib"}
{"orcid": "Fagner Carniel", "result":"Adonai Lacruz"}
{"orcid": "Carrie Peltz", "result":"Erica Kornblith"}
{"orcid": "Kathryn Huyvaert", "result":"Larissa L. Bailey"}
{"orcid": "Christine Provost", "result":"Nathalie Sennéchael"}
{"orcid": "Nancy Pachana", "result":"Lisa DiNatale"}
{"orcid": "ARDESHIR BAYAT", "result":"P. Marcos Gorresen"}
{"orcid": "Paul Berkowitz", "result":"Silje Grini Nielsen"}
{"orcid": "Alice Laciny", "result":"Brian Metscher"}
{"orcid": "Octavio Rojas", "result":"Josie A. Griffin"}
{"orcid": "Carlo Sandroni", "result":"Riccardo Scattolini"}