From 75ff902f9d96d0e8f6a65fd42ea44c550efb9a45 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 26 Jun 2023 16:50:59 +0200 Subject: [PATCH] WIP: various refactors --- .../main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java index e6e8c745c7..fa9bd39bed 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java @@ -81,6 +81,8 @@ public class SparkCreateSimRels extends AbstractSparkAction { final String outputPath = DedupUtility.createSimRelPath(workingPath, actionSetId, subEntity); removeOutputDir(spark, outputPath); + JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + SparkDedupConfig sparkConfig = new SparkDedupConfig(dedupConf, numPartitions); Dataset simRels = spark