changed logic to split for communities

This commit is contained in:
Miriam Baglioni 2021-07-13 15:15:27 +02:00
parent 6410ab71d8
commit 13c66e16be
1 changed files with 8 additions and 10 deletions

View File

@ -57,13 +57,11 @@ public class CommunitySplit implements Serializable {
Dataset<CommunityResult> community_products = result Dataset<CommunityResult> community_products = result
.filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, c)); .filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, c));
community_products
community_products .write()
.write() .option("compression", "gzip")
.option("compression", "gzip") .mode(SaveMode.Overwrite)
.mode(SaveMode.Overwrite) .json(outputPath + "/" + c);
.json(outputPath + "/" + c);
} }
@ -72,9 +70,9 @@ public class CommunitySplit implements Serializable {
return r return r
.getContext() .getContext()
.stream() .stream()
.map(con -> con.getCode()) .map(con -> con.getCode())
.collect(Collectors.toList()) .collect(Collectors.toList())
.contains(c); .contains(c);
} }
return false; return false;
} }