miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

added repartition to one before writing so as to have just one file for each community product

This commit is contained in:
Miriam Baglioni 2020-06-22 17:38:02 +02:00
parent e983d02c1c
commit 004bf225cb
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/SparkDumpRISISCatalogue.java
View File

@ -92,6 +92,7 @@ public class SparkDumpRISISCatalogue implements Serializable {
value -> execMap(value, communityName), value -> execMap(value, communityName),
Encoders.bean(eu.dnetlib.dhp.schema.dump.gcat.CatalogueEntry.class)) Encoders.bean(eu.dnetlib.dhp.schema.dump.gcat.CatalogueEntry.class))
.filter(Objects::nonNull) .filter(Objects::nonNull)
.repartition(1)
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")