|
|
|
@ -129,11 +129,9 @@ object SparkGenerateDoiBoost {
|
|
|
|
|
.map(DoiBoostMappingUtil.fixPublication)
|
|
|
|
|
.map(p => (p.getId, p))
|
|
|
|
|
.groupByKey(_._1)
|
|
|
|
|
.reduceGroups((left, right) =>
|
|
|
|
|
{
|
|
|
|
|
.reduceGroups((left, right) => {
|
|
|
|
|
//Check left is not null
|
|
|
|
|
if (left != null && left._1 != null)
|
|
|
|
|
{
|
|
|
|
|
if (left != null && left._1 != null) {
|
|
|
|
|
//If right is null then return left
|
|
|
|
|
if (right == null || right._2 == null)
|
|
|
|
|
left
|
|
|
|
@ -151,21 +149,17 @@ object SparkGenerateDoiBoost {
|
|
|
|
|
//Return publication Merged
|
|
|
|
|
(b1.getId, b1)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else {
|
|
|
|
|
} else {
|
|
|
|
|
// Left is Null so we return right
|
|
|
|
|
right
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
)
|
|
|
|
|
.filter(s => s!= null && s._2!=null)
|
|
|
|
|
})
|
|
|
|
|
.filter(s => s != null && s._2 != null)
|
|
|
|
|
.map(s => s._2._2)
|
|
|
|
|
.write
|
|
|
|
|
.mode(SaveMode.Overwrite)
|
|
|
|
|
.save(s"$workingDirPath/doiBoostPublicationFiltered")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val affiliationPath = parser.get("affiliationPath")
|
|
|
|
|
val paperAffiliationPath = parser.get("paperAffiliationPath")
|
|
|
|
|
|
|
|
|
@ -290,4 +284,4 @@ object SparkGenerateDoiBoost {
|
|
|
|
|
.save(s"$workingDirPath/doiBoostOrganization")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|