From a4a54a3786762079ef7bae86d94b94545de551ca Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 8 Jul 2021 09:08:25 +0200 Subject: [PATCH] code refactor --- .../eu/dnetlib/dhp/sx/graph/SparkCreateScholix.scala | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/SparkCreateScholix.scala b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/SparkCreateScholix.scala index bb1c9438e..ba483bfb2 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/SparkCreateScholix.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/SparkCreateScholix.scala @@ -81,9 +81,6 @@ object SparkCreateScholix { val scholix_final:Dataset[Scholix] = spark.read.load(s"$targetPath/scholix").as[Scholix] - - - val stats:Dataset[(String,String,Long)]= scholix_final.map(s => (s.getSource.getDnetIdentifier, s.getTarget.getObjectType)).groupBy("_1", "_2").agg(count("_1")).as[(String,String,Long)] @@ -94,10 +91,7 @@ object SparkCreateScholix { .map(_._2) .write.mode(SaveMode.Overwrite).save(s"$targetPath/related_entities") - - - val relatedEntitiesDS:Dataset[RelatedEntities] = spark.read.load(s"$targetPath/related_entities").as[RelatedEntities].filter(r => r.relatedPublication>0 || r.relatedDataset > 0) - + val relatedEntitiesDS:Dataset[RelatedEntities] = spark.read.load(s"$targetPath/related_entities").as[RelatedEntities].filter(r => r.relatedPublication>0 || r.relatedDataset > 0) relatedEntitiesDS.joinWith(summaryDS, relatedEntitiesDS("id").equalTo(summaryDS("_1")), "inner").map{i => val re = i._1 @@ -109,8 +103,4 @@ object SparkCreateScholix { }.write.mode(SaveMode.Overwrite).save(s"${summaryPath}_filtered") } - - - - }