added compression to output dataset

2020-11-13 12:45:31 +01:00 · 2020-11-13 12:45:31 +01:00 · 005f849674
parent 9a2fa9dc2f
commit 005f849674
1 changed files with 2 additions and 0 deletions
--- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/SparkGenEnrichedOrcidWorks.java
+++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/SparkGenEnrichedOrcidWorks.java
@ -128,6 +128,8 @@ public class SparkGenEnrichedOrcidWorks {
 						})
 					.filter(p -> p != null);

+				sc.hadoopConfiguration().set("mapreduce.output.fileoutputformat.compress", "true");
+
 				oafPublicationRDD
 					.mapToPair(
 						p -> new Tuple2<>(p.getClass().toString(),