package eu.dnetlib.dhp.sx.graph

import eu.dnetlib.dhp.application.AbstractScalaApplication
import eu.dnetlib.dhp.collection.CollectionUtils.fixRelations
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
import eu.dnetlib.dhp.datacite.{DataciteToOAFTransformation, DataciteType}
import eu.dnetlib.dhp.schema.oaf.{Oaf, Relation, Result}
import eu.dnetlib.dhp.schema.sx.scholix.{Scholix, ScholixResource}
import eu.dnetlib.dhp.schema.sx.summary.ScholixSummary
import eu.dnetlib.dhp.sx.graph.scholix.ScholixUtils
import eu.dnetlib.dhp.utils.{DHPUtils, ISLookupClientFactory}
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.sql.functions.max
import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.JavaConverters._
import java.text.SimpleDateFormat

class SparkRetrieveDataciteDelta(propertyPath: String, args: Array[String], log: Logger)
    extends AbstractScalaApplication(propertyPath, args, log: Logger) {

  val ISO_DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ssZ"
  val simpleFormatter = new SimpleDateFormat(ISO_DATE_PATTERN)

  val SCHOLIX_RESOURCE_PATH_NAME = "scholixResource"
  val DATACITE_OAF_PATH_NAME = "dataciteOAFUpdate"
  val PID_MAP_PATH_NAME = "pidMap"
  val RESOLVED_REL_PATH_NAME = "resolvedRelation"
  val SCHOLIX_PATH_NAME = "scholix"

  def scholixResourcePath(workingPath: String) = s"$workingPath/$SCHOLIX_RESOURCE_PATH_NAME"
  def dataciteOAFPath(workingPath: String) = s"$workingPath/$DATACITE_OAF_PATH_NAME"
  def pidMapPath(workingPath: String) = s"$workingPath/$PID_MAP_PATH_NAME"
  def resolvedRelationPath(workingPath: String) = s"$workingPath/$RESOLVED_REL_PATH_NAME"
  def scholixPath(workingPath: String) = s"$workingPath/$SCHOLIX_PATH_NAME"

  /** Utility to parse Date in ISO8601 to epochMillis
    * @param inputDate The String represents an input date in ISO8601
    * @return The relative epochMillis of parsed date
    */
  def ISO8601toEpochMillis(inputDate: String): Long = {
    simpleFormatter.parse(inputDate).getTime
  }

  /** This method tries to retrieve the last collection date from all datacite
    * records in HDFS.
    * This method should be called before indexing scholexplorer to retrieve
    * the delta of Datacite record to download, since from the generation of
    * raw graph to the generation of Scholexplorer sometimes it takes 20 days
    * @param spark
    * @param entitiesPath
    * @return the last collection date from the current scholexplorer Graph of the datacite records
    */
  def retrieveLastCollectedFrom(spark: SparkSession, entitiesPath: String): Long = {
    log.info("Retrieve last entities collected From")

    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
    implicit val resultEncoder: Encoder[Result] = Encoders.kryo[Result]
    import spark.implicits._

    val entitiesDS = spark.read
      .load(s"$entitiesPath/*")
      .as[Oaf]
      .filter(o => o.isInstanceOf[Result])
      .map(r => r.asInstanceOf[Result])

    val date = entitiesDS
      .filter(r => r.getDateofcollection != null)
      .map(_.getDateofcollection)
      .select(max("value"))
      .first
      .getString(0)

    ISO8601toEpochMillis(date) / 1000
  }

  /** The method of update Datacite relationships on Scholexplorer
    * needs some utilities data structures
    * One is the scholixResource DS that stores all the nodes in the Scholix Graph
    * in format ScholixResource
    * @param summaryPath the path of the summary in Scholix
    * @param workingPath the working path
    * @param spark the spark session
    */
  def generateScholixResource(
    summaryPath: String,
    workingPath: String,
    spark: SparkSession
  ): Unit = {
    implicit val summaryEncoder: Encoder[ScholixSummary] = Encoders.kryo[ScholixSummary]
    implicit val scholixResourceEncoder: Encoder[ScholixResource] = Encoders.kryo[ScholixResource]

    log.info("Convert All summary to ScholixResource")
    spark.read
      .load(summaryPath)
      .as[ScholixSummary]
      .map(ScholixUtils.generateScholixResourceFromSummary)(scholixResourceEncoder)
      .filter(r => r.getIdentifier != null && r.getIdentifier.size > 0)
      .write
      .mode(SaveMode.Overwrite)
      .save(s"${scholixResourcePath(workingPath)}_native")
  }

  /** This method convert the new Datacite Resource into Scholix Resource
    * Needed to fill the source and the type of Scholix Relationships
    * @param workingPath the Working Path
    * @param spark The spark Session
    */
  def addMissingScholixResource(workingPath: String, spark: SparkSession): Unit = {
    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
    implicit val scholixResourceEncoder: Encoder[ScholixResource] = Encoders.kryo[ScholixResource]
    implicit val resultEncoder: Encoder[Result] = Encoders.kryo[Result]
    import spark.implicits._

    spark.read
      .load(dataciteOAFPath(workingPath))
      .as[Oaf]
      .filter(_.isInstanceOf[Result])
      .map(_.asInstanceOf[Result])
      .map(ScholixUtils.generateScholixResourceFromResult)
      .filter(r => r.getIdentifier != null && r.getIdentifier.size > 0)
      .write
      .mode(SaveMode.Overwrite)
      .save(s"${scholixResourcePath(workingPath)}_update")

    val update = spark.read.load(s"${scholixResourcePath(workingPath)}_update").as[ScholixResource]
    val native = spark.read.load(s"${scholixResourcePath(workingPath)}_native").as[ScholixResource]
    val graph = update
      .union(native)
      .groupByKey(_.getDnetIdentifier)
      .reduceGroups((a, b) => if (a != null && a.getDnetIdentifier != null) a else b)
      .map(_._2)
    graph.write.mode(SaveMode.Overwrite).save(s"${scholixResourcePath(workingPath)}_graph")
  }

  /** This method get and Transform only datacite records with
    * timestamp greater than timestamp
    * @param datacitePath the datacite input Path
    * @param timestamp the timestamp
    * @param workingPath the working path where save the generated Dataset
    * @param spark SparkSession
    * @param vocabularies Vocabularies needed for transformation
    */

  def getDataciteUpdate(
    datacitePath: String,
    timestamp: Long,
    workingPath: String,
    spark: SparkSession,
    vocabularies: VocabularyGroup
  ): Long = {
    import spark.implicits._
    val ds = spark.read.load(datacitePath).as[DataciteType]
    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
    val total = ds.filter(_.timestamp >= timestamp).count()
    if (total > 0) {
      ds.filter(_.timestamp >= timestamp)
        .flatMap(d =>
          DataciteToOAFTransformation
            .generateOAF(d.json, d.timestamp, d.timestamp, vocabularies, exportLinks = true)
        )
        .flatMap(i => fixRelations(i))
        .filter(i => i != null)
        .write
        .mode(SaveMode.Overwrite)
        .save(dataciteOAFPath(workingPath))
    }
    total
  }

  /** After added the new ScholixResource, we need to update the scholix Pid Map
    * to intersected with the new Datacite Relations
    *
    * @param workingPath The working Path starting from save the new Map
    * @param spark the spark session
    */
  def generatePidMap(workingPath: String, spark: SparkSession): Unit = {
    implicit val scholixResourceEncoder: Encoder[ScholixResource] = Encoders.kryo[ScholixResource]
    import spark.implicits._
    spark.read
      .load(s"${scholixResourcePath(workingPath)}_graph")
      .as[ScholixResource]
      .flatMap(r =>
        r.getIdentifier.asScala
          .map(i => DHPUtils.generateUnresolvedIdentifier(i.getIdentifier, i.getSchema))
          .map(t => (t, r.getDnetIdentifier))
      )(Encoders.tuple(Encoders.STRING, Encoders.STRING))
      .groupByKey(_._1)
      .reduceGroups((a, b) => if (a != null && a._2 != null) a else b)
      .map(_._2)(Encoders.tuple(Encoders.STRING, Encoders.STRING))
      .write
      .mode(SaveMode.Overwrite)
      .save(pidMapPath(workingPath))
  }

  /** This method resolve the datacite relation and filter the resolved
    * relation
    * @param workingPath the working path
    * @param spark the spark session
    */

  def resolveUpdateRelation(workingPath: String, spark: SparkSession): Unit = {
    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
    implicit val relationEncoder: Encoder[Relation] = Encoders.kryo[Relation]
    import spark.implicits._

    val pidMap = spark.read.load(pidMapPath(workingPath)).as[(String, String)]

    val unresolvedRelations: Dataset[(String, Relation)] = spark.read
      .load(dataciteOAFPath(workingPath))
      .as[Oaf]
      .filter(_.isInstanceOf[Relation])
      .map(_.asInstanceOf[Relation])
      .map { r =>
        if (r.getSource.startsWith("unresolved"))
          (r.getSource, r)
        else
          (r.getTarget, r)
      }(Encoders.tuple(Encoders.STRING, relationEncoder))

    unresolvedRelations
      .joinWith(pidMap, unresolvedRelations("_1").equalTo(pidMap("_1")))
      .map(t => {
        val r = t._1._2
        val resolvedIdentifier = t._2._2
        if (r.getSource.startsWith("unresolved"))
          r.setSource(resolvedIdentifier)
        else
          r.setTarget(resolvedIdentifier)
        r
      })(relationEncoder)
      .filter(r => !(r.getSource.startsWith("unresolved") || r.getTarget.startsWith("unresolved")))
      .write
      .mode(SaveMode.Overwrite)
      .save(resolvedRelationPath(workingPath))
  }

  /** This method generate scholix starting from resolved relation
    *
    * @param workingPath
    * @param spark
    */
  def generateScholixUpdate(workingPath: String, spark: SparkSession): Unit = {
    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
    implicit val scholixEncoder: Encoder[Scholix] = Encoders.kryo[Scholix]
    implicit val scholixResourceEncoder: Encoder[ScholixResource] = Encoders.kryo[ScholixResource]
    implicit val relationEncoder: Encoder[Relation] = Encoders.kryo[Relation]
    implicit val intermediateEncoder: Encoder[(String, Scholix)] =
      Encoders.tuple(Encoders.STRING, scholixEncoder)

    val relations: Dataset[(String, Relation)] = spark.read
      .load(resolvedRelationPath(workingPath))
      .as[Relation]
      .map(r => (r.getSource, r))(Encoders.tuple(Encoders.STRING, relationEncoder))

    val id_summary: Dataset[(String, ScholixResource)] = spark.read
      .load(s"${scholixResourcePath(workingPath)}_graph")
      .as[ScholixResource]
      .map(r => (r.getDnetIdentifier, r))(Encoders.tuple(Encoders.STRING, scholixResourceEncoder))

    id_summary.cache()

    relations
      .joinWith(id_summary, relations("_1").equalTo(id_summary("_1")), "inner")
      .map(t => (t._1._2.getTarget, ScholixUtils.scholixFromSource(t._1._2, t._2._2)))
      .write
      .mode(SaveMode.Overwrite)
      .save(s"$workingPath/scholix_one_verse")

    val source_scholix: Dataset[(String, Scholix)] =
      spark.read.load(s"$workingPath/scholix_one_verse").as[(String, Scholix)]

    source_scholix
      .joinWith(id_summary, source_scholix("_1").equalTo(id_summary("_1")), "inner")
      .map(t => {
        val target: ScholixResource = t._2._2
        val scholix: Scholix = t._1._2
        ScholixUtils.generateCompleteScholix(scholix, target)
      })(scholixEncoder)
      .write
      .mode(SaveMode.Overwrite)
      .save(s"$workingPath/scholix")
  }

  /** Here all the spark applications runs this method
    * where the whole logic of the spark node is defined
    */
  override def run(): Unit = {
    val sourcePath = parser.get("sourcePath")
    log.info(s"SourcePath is '$sourcePath'")

    val datacitePath = parser.get("datacitePath")
    log.info(s"DatacitePath is '$datacitePath'")

    val workingPath = parser.get("workingSupportPath")
    log.info(s"workingPath is '$workingPath'")

    val isLookupUrl: String = parser.get("isLookupUrl")
    log.info("isLookupUrl: {}", isLookupUrl)

    val isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl)
    val vocabularies = VocabularyGroup.loadVocsFromIS(isLookupService)
    require(vocabularies != null)

    val updateDS: Boolean = "true".equalsIgnoreCase(parser.get("updateDS"))
    log.info(s"updateDS is '$updateDS'")

    var lastCollectionDate = 0L
    if (updateDS) {
      generateScholixResource(s"$sourcePath/provision/summaries", workingPath, spark)
      log.info("Retrieve last entities collected From starting from scholix Graph")
      lastCollectionDate = retrieveLastCollectedFrom(spark, s"$sourcePath/entities")
    } else {
      val fs = FileSystem.get(spark.sparkContext.hadoopConfiguration)
      fs.delete(new Path(s"${scholixResourcePath(workingPath)}_native"), true)
      fs.rename(
        new Path(s"${scholixResourcePath(workingPath)}_graph"),
        new Path(s"${scholixResourcePath(workingPath)}_native")
      )
      lastCollectionDate = retrieveLastCollectedFrom(spark, dataciteOAFPath(workingPath))
    }

    val numRecords =
      getDataciteUpdate(datacitePath, lastCollectionDate, workingPath, spark, vocabularies)
    if (numRecords > 0) {
      addMissingScholixResource(workingPath, spark)
      generatePidMap(workingPath, spark)
      resolveUpdateRelation(workingPath, spark)
      generateScholixUpdate(workingPath, spark)
    }
  }
}

object SparkRetrieveDataciteDelta {
  val log: Logger = LoggerFactory.getLogger(SparkRetrieveDataciteDelta.getClass)

  def main(args: Array[String]): Unit = {
    new SparkRetrieveDataciteDelta(
      "/eu/dnetlib/dhp/sx/graph/retrieve_datacite_delta_params.json",
      args,
      log
    ).initialize().run()
  }
}