D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

WIP: dump of the OpenAIRE graph - Changes #103

Closed
miriam.baglioni wants to merge 77 commits from miriam.baglioni/dnet-hadoop:dump into master
pull from: miriam.baglioni/dnet-hadoop:dump
merge into: D-Net:master
Conversation 3 Commits 77 Files Changed 59 +8 -2
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 85bd6ab5d0 - Show all commits

10
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/funderresults/SparkResultLinkedToProject.java
View File

@ -98,7 +98,13 @@ public class SparkResultLinkedToProject implements Serializable {
"join project p " +
"on rel.target = p.id " +
"")
.as(Encoders.bean(inputClazz));
.as(Encoders.bean(inputClazz))
;
tmp.groupByKey(
(MapFunction< R, String>) value -> value
.getId(),
Encoders.STRING())
.mapGroups((MapGroupsFunction<String, R, R>) (k, it) -> it.next(), Encoders.bean(inputClazz))
//
// relations
@ -113,7 +119,7 @@ public class SparkResultLinkedToProject implements Serializable {
// .mapGroups((MapGroupsFunction<String, Tuple2<Relation, R>, R>) (k, it) -> {
// return it.next()._2();
// }, Encoders.bean(inputClazz))
tmp
//tmp
.write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")