D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

hostedbymap #136

Merged
claudio.atzori merged 60 commits from hostedbymap into beta 2021-08-12 17:10:55 +02:00
Conversation 1 Commits 60 Files Changed 46 +4 -5
1 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 8ba8c77f92 - Show all commits

9
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToResult.scala
View File

@ -29,7 +29,6 @@ object SparkApplyHostedByMapToResult {
val i = p.getInstance().asScala
if (i.size == 1) {
val inst: Instance = i(0)
inst.getHostedby.setKey(ei.getHb_id)
inst.getHostedby.setValue(ei.getName)
if (ei.getOpenaccess) {
@ -60,7 +59,7 @@ object SparkApplyHostedByMapToResult {
val graphPath = parser.get("graphPath")
val outputPath = parser.get("outputPath")
val workingPath = parser.get("workingPath")
val preparedInfoPath = parser.get("preparedInfoPath")
implicit val formats = DefaultFormats
@ -70,15 +69,15 @@ object SparkApplyHostedByMapToResult {
implicit val mapEncoderEinfo: Encoder[EntityInfo] = Encoders.bean(classOf[EntityInfo])
val mapper = new ObjectMapper()
val pubs : Dataset[Publication] = spark.read.textFile("$graphPath/publication")
val pubs : Dataset[Publication] = spark.read.textFile(graphPath + "/publication")
.map(r => mapper.readValue(r, classOf[Publication]))
val pinfo : Dataset[EntityInfo] = spark.read.textFile("$workingPath/preparedInfo")
val pinfo : Dataset[EntityInfo] = spark.read.textFile(preparedInfoPath)
.map(ei => mapper.readValue(ei, classOf[EntityInfo]))
//a. publication join risultato del passo precedente su result id (left) setto la istanza (se piu' di una instance
// nel result => salto)con l'hosted by anche access right della instance se openaccess e' true
applyHBtoPubs(pinfo, pubs).write.mode(SaveMode.Overwrite).option("compression","gzip").json("$graphPath/publication")
applyHBtoPubs(pinfo, pubs).write.mode(SaveMode.Overwrite).option("compression","gzip").json(outputPath)