sandro.labruzzo
/
DHP-Explorer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

all spark apps extends serializable

This commit is contained in:
Sandro La Bruzzo 2022-10-21 11:07:16 +02:00
parent 11634d1cc5
commit faa1c38b29
2 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/sandro/app/SparkApp.scala
View File

@ -5,7 +5,7 @@ import org.apache.spark.sql.SparkSession
import org.slf4j.Logger
import scala.collection.mutable
trait SparkApp {
trait SparkApp extends Serializable {
/** Utility to parse the arguments.properties using the
* property json in the classpath identified from

6
src/main/java/eu/dnetlib/scholix/CheckRelation.scala
View File

@ -7,7 +7,7 @@ import org.slf4j.{Logger, LoggerFactory}
import eu.dnetlib.dhp.schema.oaf.Relation
import eu.dnetlib.scholix.CheckRelation.logger
class CheckRelation( args: Array[String], log: Logger) extends AbstractScalaApplication( args: Array[String], log: Logger) {
class CheckRelation( args: Array[String], log: Logger) extends AbstractScalaApplication( args: Array[String], log: Logger) {
def filterRelations(r: Relation): Boolean = {
val relClassFilter = List(
@ -58,11 +58,13 @@ class CheckRelation( args: Array[String], log: Logger) extends AbstractScalaAppl
val total_rels_from_scholexplorer = df.map(s=> mapper.readValue(s, classOf[Relation]))
.filter(r => r.getDataInfo != null && !r.getDataInfo.getDeletedbyinference)
.filter(r => r.getSource.startsWith("50") && r.getTarget.startsWith("50"))
// .filter(r => filterRelations(r))
.filter(r => filterRelations(r))
.count()
logger.warn(s"Relation used by Scholexplorer $total_rels_from_scholexplorer")
}
}