package eu.dnetlib.dhp.provision

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.{coalesce, col, count, lit}

object DatasetJoiner {

  def startJoin(spark: SparkSession, relPath:String, targetPath:String) {
    val relation = spark.read.load(relPath)

    val relatedPublication = relation.where("target like '50%'").groupBy("source").agg(count("target").as("publication")).select(col("source"). alias("p_source"), col("publication"))
    val relatedDataset = relation.where("target like '60%'").groupBy("source").agg(count("target").as("dataset")).select(col("source"). alias("d_source"), col("dataset"))
    val relatedUnknown = relation.where("target like '70%'").groupBy("source").agg(count("target").as("unknown")).select(col("source"). alias("u_source"), col("unknown"))
    val firstJoin = relatedPublication
        .join(relatedDataset,col("p_source").equalTo(col("d_source")),"full")
      .select(coalesce(col("p_source"), col("d_source")).alias("id"),
              col("publication"),
              col("dataset"))
      .join(relatedUnknown, col("u_source").equalTo(col("id")),"full")
      .select(coalesce(col("u_source"), col("id")).alias("source"),
        coalesce(col("publication"),lit(0)).alias("relatedPublication"),
        coalesce(col("dataset"),lit(0)).alias("relatedDataset"),
        coalesce(col("unknown"),lit(0)).alias("relatedUnknown")
      )
    firstJoin.write.mode("overwrite").save(targetPath)

  }

}