michele.artini
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[broker] added coalesce(1) on the stats dataset before storing it on postgres

This commit is contained in:
Claudio Atzori 2021-07-09 15:47:51 +02:00
parent c30f3ce647
commit ae2b47b29d
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/GenerateStatsJob.java
View File

@ -80,6 +80,7 @@ public class GenerateStatsJob {
.map( .map(
(MapFunction<Tuple2<String, DatasourceStats>, DatasourceStats>) t -> t._2, (MapFunction<Tuple2<String, DatasourceStats>, DatasourceStats>) t -> t._2,
Encoders.bean(DatasourceStats.class)) Encoders.bean(DatasourceStats.class))
.coalesce(1)
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.jdbc(dbUrl, "oa_datasource_stats_temp", connectionProperties); .jdbc(dbUrl, "oa_datasource_stats_temp", connectionProperties);