|
|
|
@ -42,6 +42,7 @@ object SparkCreateScholix {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val relationDS: Dataset[(String, Relation)] = spark.read.load(relationPath).as[Relation]
|
|
|
|
|
.filter(r => (r.getDataInfo== null || r.getDataInfo.getDeletedbyinference == false) && !r.getRelClass.toLowerCase.contains("merge"))
|
|
|
|
|
.map(r => (r.getSource, r))(Encoders.tuple(Encoders.STRING, relEncoder))
|
|
|
|
|
|
|
|
|
|
val summaryDS: Dataset[(String, ScholixSummary)] = spark.read.load(summaryPath).as[ScholixSummary]
|
|
|
|
|