1
0
Fork 0

[orcid propagation] skip empty directories

This commit is contained in:
Claudio Atzori 2022-12-20 14:15:46 +01:00
parent 9cf0a98699
commit 6aa91204a5
1 changed files with 9 additions and 4 deletions

View File

@ -228,10 +228,15 @@ public class PropagationConstant {
public static <R> Dataset<R> readPath(
SparkSession spark, String inputPath, Class<R> clazz) {
if (HdfsSupport.exists(inputPath, spark.sparkContext().hadoopConfiguration())) {
return spark
.read()
.textFile(inputPath)
.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
} else {
return spark.emptyDataset(Encoders.bean(clazz));
}
}
}