antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

added check to verify that dump contains at least one product

This commit is contained in:
Miriam Baglioni 2020-07-07 18:21:35 +02:00
parent d86adb82a7
commit 8a1b42ff21
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SparkSplitForCommunity.java
View File

@ -98,13 +98,17 @@ public class SparkSplitForCommunity implements Serializable {
}
private static void printResult(String c, Dataset<Result> result, String outputPath) {
result
.filter(r -> containsCommunity(r, c))
.repartition(1)
.write()
.option("compression", "gzip")
.mode(SaveMode.Append)
.json(outputPath + "/" + c);
Dataset<Result> community_products = result
.filter(r -> containsCommunity(r, c));
if(community_products.count() > 0){
community_products.repartition(1)
.write()
.option("compression", "gzip")
.mode(SaveMode.Overwrite)
.json(outputPath + "/" + c);
}
}
private static boolean containsCommunity(Result r, String c) {