sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

updated new version of scholexplorer Generation

This commit is contained in:
Sandro La Bruzzo 2024-04-29 09:00:30 +02:00
parent 052c6aac9d
commit 133ead1e3e
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala
View File

@ -107,9 +107,13 @@ class SparkCreateScholexplorerDump(propertyPath: String, args: Array[String], lo
.joinWith(resource, relations("source") === resource("dnetIdentifier"), "inner") .joinWith(resource, relations("source") === resource("dnetIdentifier"), "inner")
.map(res => ScholexplorerUtils.generateScholix(res._1, res._2)) .map(res => ScholexplorerUtils.generateScholix(res._1, res._2))
val resourceTarget = relations
.joinWith(resource, relations("target") === resource("dnetIdentifier"), "inner")
.map(res => (res._1.id, res._2))(Encoders.tuple(Encoders.STRING, Encoders.kryo(classOf[ScholixResource])))
scholix_one_verse scholix_one_verse
.joinWith(resource, scholix_one_verse("target.dnetIdentifier") === resource("dnetIdentifier"), "inner") .joinWith(resourceTarget, scholix_one_verse("identifier") === resourceTarget("_1"), "inner")
.map(k => ScholexplorerUtils.updateTarget(k._1, k._2)) .map(k => ScholexplorerUtils.updateTarget(k._1, k._2._2))
.write .write
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")