From ada063ce702373ffc8afc7e5fc41a3188c42a9ec Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 14 Jun 2021 12:04:47 +0200 Subject: [PATCH] fixed a problem with empty mdstore list (2) --- .../dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java index 31527283c..5109a0763 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java @@ -14,8 +14,6 @@ import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapred.SequenceFileOutputFormat; @@ -105,7 +103,10 @@ public class MigrateHdfsMdstoresApplication extends AbstractMigrationApplication // .coalesce(1) .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); } else { - FileSystem.get(sc.hadoopConfiguration()).createNewFile(new Path(outputPath)); + spark.emptyDataFrame() + .toJavaRDD() + .mapToPair(xml -> new Tuple2<>(new Text(), new Text())) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); } }