avoid non necessary count operations over large spark datasets

2 years ago · c0750fb17c
parent bb5dca7979
commit c0750fb17c
1 changed files with 3 additions and 1 deletions
--- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java
@ -63,7 +63,9 @@ public class SparkCopyRelationsNoOpenorgs extends AbstractSparkAction {
 			.toJavaRDD()
 			.filter(x -> !isOpenorgs(x));

-		log.info("Number of non-Openorgs relations collected: {}", simRels.count());
+		if (log.isDebugEnabled()) {
+			log.debug("Number of non-Openorgs relations collected: {}", simRels.count());
+		}

 		spark
 			.createDataset(simRels.rdd(), Encoders.bean(Relation.class))