sandro.labruzzo
/
DHP-Explorer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

all spark apps extends serializable

This commit is contained in:
Sandro La Bruzzo 2022-10-21 11:07:16 +02:00
parent 11634d1cc5
commit faa1c38b29
2 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/sandro/app/SparkApp.scala
View File

@ -5,7 +5,7 @@ import org.apache.spark.sql.SparkSession
import org.slf4j.Logger import org.slf4j.Logger
import scala.collection.mutable import scala.collection.mutable
trait SparkApp { trait SparkApp extends Serializable {
/** Utility to parse the arguments.properties using the /** Utility to parse the arguments.properties using the
* property json in the classpath identified from * property json in the classpath identified from

4
src/main/java/eu/dnetlib/scholix/CheckRelation.scala
View File

@ -58,11 +58,13 @@ class CheckRelation( args: Array[String], log: Logger) extends AbstractScalaAppl
val total_rels_from_scholexplorer = df.map(s=> mapper.readValue(s, classOf[Relation])) val total_rels_from_scholexplorer = df.map(s=> mapper.readValue(s, classOf[Relation]))
.filter(r => r.getDataInfo != null && !r.getDataInfo.getDeletedbyinference) .filter(r => r.getDataInfo != null && !r.getDataInfo.getDeletedbyinference)
.filter(r => r.getSource.startsWith("50") && r.getTarget.startsWith("50")) .filter(r => r.getSource.startsWith("50") && r.getTarget.startsWith("50"))
// .filter(r => filterRelations(r)) .filter(r => filterRelations(r))
.count() .count()
logger.warn(s"Relation used by Scholexplorer $total_rels_from_scholexplorer") logger.warn(s"Relation used by Scholexplorer $total_rels_from_scholexplorer")
} }
} }