From faa1c38b29ddef33ab93605b19ca11ed9b7905bf Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 21 Oct 2022 11:07:16 +0200 Subject: [PATCH] all spark apps extends serializable --- src/main/java/com/sandro/app/SparkApp.scala | 2 +- src/main/java/eu/dnetlib/scholix/CheckRelation.scala | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/sandro/app/SparkApp.scala b/src/main/java/com/sandro/app/SparkApp.scala index d34cb9e..85aecc6 100644 --- a/src/main/java/com/sandro/app/SparkApp.scala +++ b/src/main/java/com/sandro/app/SparkApp.scala @@ -5,7 +5,7 @@ import org.apache.spark.sql.SparkSession import org.slf4j.Logger import scala.collection.mutable -trait SparkApp { +trait SparkApp extends Serializable { /** Utility to parse the arguments.properties using the * property json in the classpath identified from diff --git a/src/main/java/eu/dnetlib/scholix/CheckRelation.scala b/src/main/java/eu/dnetlib/scholix/CheckRelation.scala index 861a73b..65eab42 100644 --- a/src/main/java/eu/dnetlib/scholix/CheckRelation.scala +++ b/src/main/java/eu/dnetlib/scholix/CheckRelation.scala @@ -7,7 +7,7 @@ import org.slf4j.{Logger, LoggerFactory} import eu.dnetlib.dhp.schema.oaf.Relation import eu.dnetlib.scholix.CheckRelation.logger -class CheckRelation( args: Array[String], log: Logger) extends AbstractScalaApplication( args: Array[String], log: Logger) { +class CheckRelation( args: Array[String], log: Logger) extends AbstractScalaApplication( args: Array[String], log: Logger) { def filterRelations(r: Relation): Boolean = { val relClassFilter = List( @@ -58,11 +58,13 @@ class CheckRelation( args: Array[String], log: Logger) extends AbstractScalaAppl val total_rels_from_scholexplorer = df.map(s=> mapper.readValue(s, classOf[Relation])) .filter(r => r.getDataInfo != null && !r.getDataInfo.getDeletedbyinference) .filter(r => r.getSource.startsWith("50") && r.getTarget.startsWith("50")) -// .filter(r => filterRelations(r)) + .filter(r => filterRelations(r)) .count() logger.warn(s"Relation used by Scholexplorer $total_rels_from_scholexplorer") } + + }