antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

applying fix on the DOIBoost construction process that somehow wasn't part of the merge done in 83c90c7180

This commit is contained in:
Claudio Atzori 2021-10-14 14:33:56 +02:00
parent 4b1920f008
commit e15a1969a5
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala
View File

@ -206,9 +206,9 @@ object SparkGenerateDoiBoost {
(r.getSource, r) (r.getSource, r)
else if (r.getTarget.startsWith("unresolved")) else if (r.getTarget.startsWith("unresolved"))
(r.getTarget,r) (r.getTarget,r)
else else
("resolved", r) ("resolved", r)
}) })(Encoders.tuple(Encoders.STRING, mapEncoderRel))
val openaireOrganization:Dataset[(String,String)] = spark.read.text(openaireOrganizationPath).as[String].flatMap(s => extractIdGRID(s)).groupByKey(_._2).reduceGroups((x,y) => if (x != null) x else y ).map(_._2) val openaireOrganization:Dataset[(String,String)] = spark.read.text(openaireOrganizationPath).as[String].flatMap(s => extractIdGRID(s)).groupByKey(_._2).reduceGroups((x,y) => if (x != null) x else y ).map(_._2)
@ -221,8 +221,8 @@ object SparkGenerateDoiBoost {
currentRels.setSource(currentOrgs._1) currentRels.setSource(currentOrgs._1)
else else
currentRels.setTarget(currentOrgs._1) currentRels.setTarget(currentOrgs._1)
currentRels currentRels
}.write.save(s"$workingDirPath/doiBoostPublicationAffiliation") }.filter(r=> !r.getSource.startsWith("unresolved") && !r.getTarget.startsWith("unresolved")).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationAffiliation")
magPubs.joinWith(a,magPubs("_1").equalTo(a("PaperId"))).map( item => { magPubs.joinWith(a,magPubs("_1").equalTo(a("PaperId"))).map( item => {
val affiliation = item._2 val affiliation = item._2
@ -242,6 +242,6 @@ object SparkGenerateDoiBoost {
else else
null null
}).filter(o=> o!=null).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostOrganization") }).filter(o=> o!=null).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostOrganization")
} }
} }