D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

WIP: dump of the OpenAIRE graph - Changes #103

Closed
miriam.baglioni wants to merge 77 commits from miriam.baglioni/dnet-hadoop:dump into master
pull from: miriam.baglioni/dnet-hadoop:dump
merge into: D-Net:master
Conversation 3 Commits 77 Files Changed 59 +9 -15
1 changed files with 9 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 08f8dd9454 - Show all commits

24
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/CommunitySplit.java
View File

@ -57,16 +57,12 @@ public class CommunitySplit implements Serializable {
Dataset<CommunityResult> community_products = result Dataset<CommunityResult> community_products = result
.filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, c)); .filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, c));
// try {
// community_products.first(); community_products
community_products .write()
.write() .option("compression", "gzip")
.option("compression", "gzip") .mode(SaveMode.Overwrite)
.mode(SaveMode.Overwrite) .json(outputPath + "/" + c);
.json(outputPath + "/" + c);
// } catch (Exception e) {
//
// }
} }
@ -75,11 +71,9 @@ public class CommunitySplit implements Serializable {
return r return r
.getContext() .getContext()
.stream() .stream()
.map(con -> con.getCode()) .map(con -> con.getCode())
.collect(Collectors.toList()).contains(c); .collect(Collectors.toList())
// .filter(con -> con.getCode().equals(c)) .contains(c);
// .collect(Collectors.toList())
// .size() > 0;
} }
return false; return false;
} }