antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

moved filter on next phase

This commit is contained in:
Sandro La Bruzzo 2021-10-28 16:13:24 +02:00
parent 1be9aa0a5f
commit d9cbca83f7
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/resolution/SparkResolveRelation.scala
View File

@ -75,7 +75,7 @@ object SparkResolveRelation {
if (targetResolved != null && targetResolved._1.nonEmpty) if (targetResolved != null && targetResolved._1.nonEmpty)
currentRelation.setTarget(targetResolved._1) currentRelation.setTarget(targetResolved._1)
currentRelation currentRelation
}.filter(r => !r.getSource.startsWith("unresolved") && !r.getTarget.startsWith("unresolved")) }
.write .write
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.save(s"$workingPath/relation_resolved") .save(s"$workingPath/relation_resolved")
@ -88,6 +88,7 @@ object SparkResolveRelation {
fs.rename(new Path(s"$graphBasePath/relation"), new Path(s"$workingPath/relation")) fs.rename(new Path(s"$graphBasePath/relation"), new Path(s"$workingPath/relation"))
spark.read.load(s"$workingPath/relation_resolved").as[Relation] spark.read.load(s"$workingPath/relation_resolved").as[Relation]
.filter(r => !r.getSource.startsWith("unresolved") && !r.getTarget.startsWith("unresolved"))
.map(r => mapper.writeValueAsString(r)) .map(r => mapper.writeValueAsString(r))
.write .write
.option("compression", "gzip") .option("compression", "gzip")