From 991b06bd0b03c867bdd429b9b70adff8b1b6ab89 Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Tue, 5 Oct 2021 10:21:33 +0200
Subject: [PATCH] removed generation of EBI links from old dump, now EBI link
 dump is created by another wf

---
 .../eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala     | 5 -----
 .../eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml       | 2 +-
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala
index f14e5f264..1924d919e 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala
+++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala
@@ -32,14 +32,9 @@ object SparkEBILinksToOaf {
     import spark.implicits._
     implicit  val PMEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
 
-    val ebi_rdd:Dataset[EBILinkItem] = spark.createDataset(spark.sparkContext.textFile(sourcePath).map(s => BioDBToOAF.extractEBILinksFromDump(s))).as[EBILinkItem]
-
-    ebi_rdd.write.mode(SaveMode.Overwrite).save(s"${sourcePath}_dataset")
-
     val ebLinks:Dataset[EBILinkItem] = spark.read.load(s"${sourcePath}_dataset").as[EBILinkItem].filter(l => l.links!= null)
 
     ebLinks.flatMap(j =>BioDBToOAF.parse_ebi_links(j.links))
-      .repartition(4000)
       .filter(p => BioDBToOAF.EBITargetLinksFilter(p))
       .flatMap(p => BioDBToOAF.convertEBILinksToOaf(p))
       .write.mode(SaveMode.Overwrite).save(targetPath)
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml
index 3f442c5c6..1fa099ebc 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml
@@ -74,7 +74,7 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn-cluster</master>
             <mode>cluster</mode>
-            <name>Create Baselnie DataSet</name>
+            <name>Create Baseline DataSet</name>
 
             <class>eu.dnetlib.dhp.sx.ebi.SparkAddLinkUpdates</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>