dnet-hadoop/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/TestPreprocess.scala

package eu.dnetlib.dhp.oa.graph.hostedbymap

import eu.dnetlib.dhp.schema.oaf.Datasource
import org.apache.spark.SparkConf
import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
import org.json4s.DefaultFormats
import org.json4s.jackson.Serialization.write
import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.Test

class TestPreprocess extends java.io.Serializable {

  implicit val mapEncoderDats: Encoder[Datasource] = Encoders.kryo[Datasource]
  implicit val schema = Encoders.product[HostedByInfo]

  def toHBIString(hbi: HostedByItemType): String = {
    implicit val formats = DefaultFormats

    write(hbi)
  }

  @Test
  def readDatasource(): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local[*]")
    conf.set("spark.driver.host", "localhost")
    val spark: SparkSession =
      SparkSession
        .builder()
        .appName(getClass.getSimpleName)
        .config(conf)
        .getOrCreate()
    val path = getClass.getResource("datasource.json").getPath

    val ds: Dataset[HostedByItemType] = SparkProduceHostedByMap.oaHostedByDataset(spark, path)

    assertEquals(9, ds.count)

    assertEquals(8, ds.filter(hbi => !hbi.issn.equals("")).count)
    assertEquals(5, ds.filter(hbi => !hbi.eissn.equals("")).count)
    assertEquals(0, ds.filter(hbi => !hbi.lissn.equals("")).count)

    assertEquals(
      0,
      ds.filter(hbi => hbi.issn.equals("") && hbi.eissn.equals("") && hbi.lissn.equals("")).count
    )

    assertTrue(ds.filter(hbi => hbi.issn.equals("0212-8365")).count == 1)
    assertTrue(ds.filter(hbi => hbi.eissn.equals("2253-900X")).count == 1)
    assertTrue(
      ds.filter(hbi => hbi.issn.equals("0212-8365") && hbi.eissn.equals("2253-900X")).count == 1
    )
    assertTrue(
      ds.filter(hbi => hbi.issn.equals("0212-8365") && hbi.officialname.equals("Thémata")).count == 1
    )
    assertTrue(
      ds.filter(hbi =>
        hbi.issn.equals("0212-8365") && hbi.id
          .equals("10|doajarticles::abbc9265bea9ff62776a1c39785af00c")
      ).count == 1
    )
    ds.foreach(hbi => assertTrue(hbi.id.startsWith("10|")))
    ds.foreach(hbi => println(toHBIString(hbi)))
    spark.close()
  }

  @Test
  def readGold(): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local[*]")
    conf.set("spark.driver.host", "localhost")
    val spark: SparkSession =
      SparkSession
        .builder()
        .appName(getClass.getSimpleName)
        .config(conf)
        .getOrCreate()
    val path = getClass.getResource("unibi_transformed.json").getPath

    val ds: Dataset[HostedByItemType] = SparkProduceHostedByMap.goldHostedByDataset(spark, path)

    assertEquals(29, ds.count)

    assertEquals(29, ds.filter(hbi => !hbi.issn.equals("")).count)
    assertEquals(0, ds.filter(hbi => !hbi.eissn.equals("")).count)
    assertEquals(29, ds.filter(hbi => !hbi.lissn.equals("")).count)

    assertEquals(
      0,
      ds.filter(hbi => hbi.issn.equals("") && hbi.eissn.equals("") && hbi.lissn.equals("")).count
    )

    assertTrue(
      ds.filter(hbi => hbi.issn.equals("2239-6101"))
        .first()
        .officialname
        .equals("European journal of sustainable development.")
    )
    assertTrue(ds.filter(hbi => hbi.issn.equals("2239-6101")).first().lissn.equals("2239-5938"))
    assertTrue(ds.filter(hbi => hbi.issn.equals("2239-6101")).count == 1)
    ds.foreach(hbi => assertTrue(hbi.id.equals(Constants.UNIBI)))
    ds.foreach(hbi => println(toHBIString(hbi)))

    spark.close()
  }

  @Test
  def readDoaj(): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local[*]")
    conf.set("spark.driver.host", "localhost")
    val spark: SparkSession =
      SparkSession
        .builder()
        .appName(getClass.getSimpleName)
        .config(conf)
        .getOrCreate()
    val path = getClass.getResource("doaj_transformed.json").getPath

    val ds: Dataset[HostedByItemType] = SparkProduceHostedByMap.doajHostedByDataset(spark, path)

    assertEquals(25, ds.count)

    assertEquals(14, ds.filter(hbi => !hbi.issn.equals("")).count)
    assertEquals(21, ds.filter(hbi => !hbi.eissn.equals("")).count)
    assertEquals(0, ds.filter(hbi => !hbi.lissn.equals("")).count)

    assertEquals(
      0,
      ds.filter(hbi => hbi.issn.equals("") && hbi.eissn.equals("") && hbi.lissn.equals("")).count
    )

    assertTrue(
      ds.filter(hbi => hbi.issn.equals("2077-3099"))
        .first()
        .officialname
        .equals("Journal of Space Technology")
    )
    assertTrue(ds.filter(hbi => hbi.issn.equals("2077-3099")).first().eissn.equals("2411-5029"))
    assertTrue(ds.filter(hbi => hbi.issn.equals("2077-3099")).count == 1)
    assertTrue(ds.filter(hbi => hbi.eissn.equals("2077-2955")).first().issn.equals(""))
    ds.foreach(hbi => assertTrue(hbi.id.equals(Constants.DOAJ)))
    ds.foreach(hbi => println(toHBIString(hbi)))

    spark.close()
  }

  @Test
  def testAggregator(): Unit = {

    val conf = new SparkConf()
    conf.setMaster("local[*]")
    conf.set("spark.driver.host", "localhost")
    val spark: SparkSession =
      SparkSession
        .builder()
        .appName(getClass.getSimpleName)
        .config(conf)
        .getOrCreate()

    val tmp = SparkProduceHostedByMap
      .oaHostedByDataset(spark, getClass.getResource("datasource.json").getPath)
      .union(
        SparkProduceHostedByMap
          .goldHostedByDataset(spark, getClass.getResource("unibi_transformed.json").getPath)
      )
      .union(
        SparkProduceHostedByMap
          .doajHostedByDataset(spark, getClass.getResource("doaj_transformed.json").getPath)
      )
      .flatMap(hbi => SparkProduceHostedByMap.toList(hbi))(
        Encoders.tuple(Encoders.STRING, Encoders.product[HostedByItemType])
      )

    assertEquals(106, tmp.count)
    assertEquals(82, tmp.map(i => i._1)(Encoders.STRING).distinct().count)

    val ds: Dataset[(String, HostedByItemType)] = Aggregators.explodeHostedByItemType(
      SparkProduceHostedByMap
        .oaHostedByDataset(spark, getClass.getResource("datasource.json").getPath)
        .union(
          SparkProduceHostedByMap
            .goldHostedByDataset(spark, getClass.getResource("unibi_transformed.json").getPath)
        )
        .union(
          SparkProduceHostedByMap
            .doajHostedByDataset(spark, getClass.getResource("doaj_transformed.json").getPath)
        )
        .flatMap(hbi => SparkProduceHostedByMap.toList(hbi))(
          Encoders.tuple(Encoders.STRING, Encoders.product[HostedByItemType])
        )
    )

    assertEquals(82, ds.count)

    assertEquals(13, ds.filter(i => i._2.id.startsWith("10|")).count)

    assertTrue(ds.filter(i => i._1.equals("2077-3757")).first()._2.id.startsWith("10|"))
    assertTrue(ds.filter(i => i._1.equals("2077-3757")).first()._2.openAccess)
    assertEquals(1, ds.filter(i => i._1.equals("2077-3757")).count)

    val hbmap: Dataset[String] = ds
      .filter(hbi => hbi._2.id.startsWith("10|"))
      .map(SparkProduceHostedByMap.toHostedByMap)(Encoders.STRING)

    hbmap.foreach(entry => println(entry))
    spark.close()

  }

}