From 46972f8393a31d3ee1b149d7f8a69ad847cb68f4 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 20 Dec 2022 10:28:22 +0100 Subject: [PATCH] [orcid propagation] skip empty directory --- .../java/eu/dnetlib/dhp/PropagationConstant.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index 02fdcb09b..b01564d25 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import org.apache.hadoop.conf.Configuration; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; @@ -228,10 +229,15 @@ public class PropagationConstant { public static Dataset readPath( SparkSession spark, String inputPath, Class clazz) { - return spark - .read() - .textFile(inputPath) - .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + + if (HdfsSupport.exists(inputPath, spark.sparkContext().hadoopConfiguration())) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } else { + return spark.emptyDataset(Encoders.bean(clazz)); + } } }