[scala-refactor] Module dhp-graph-mapper:

Moved all scala source into src/main/scala and src/test/scala

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/Aggregators.scala

@@ -1,8 +1,8 @@
 package eu.dnetlib.dhp.oa.graph.hostedbymap
 
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, TypedColumn}
 import org.apache.spark.sql.expressions.Aggregator
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, TypedColumn}
 
 
 case class HostedByItemType(id: String, officialname: String, issn: String, eissn: String, lissn: String, openAccess: Boolean) {}

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToDatasource.scala

@@ -2,13 +2,12 @@ package eu.dnetlib.dhp.oa.graph.hostedbymap
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.oa.graph.hostedbymap.SparkApplyHostedByMapToResult.{applyHBtoPubs, getClass}
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
 import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.{Datasource, Publication}
+import eu.dnetlib.dhp.schema.oaf.Datasource
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
 
@@ -52,18 +51,18 @@ object SparkApplyHostedByMapToDatasource {
 
     val mapper = new ObjectMapper()
 
-    val dats : Dataset[Datasource] = spark.read.textFile(graphPath + "/datasource")
+    val dats: Dataset[Datasource] = spark.read.textFile(graphPath + "/datasource")
       .map(r => mapper.readValue(r, classOf[Datasource]))
 
-    val pinfo : Dataset[EntityInfo] = Aggregators.datasourceToSingleId( spark.read.textFile(preparedInfoPath)
+    val pinfo: Dataset[EntityInfo] = Aggregators.datasourceToSingleId(spark.read.textFile(preparedInfoPath)
       .map(ei => mapper.readValue(ei, classOf[EntityInfo])))
 
-    applyHBtoDats(pinfo, dats).write.mode(SaveMode.Overwrite).option("compression","gzip").json(outputPath)
+    applyHBtoDats(pinfo, dats).write.mode(SaveMode.Overwrite).option("compression", "gzip").json(outputPath)
 
     spark.read.textFile(outputPath)
       .write
       .mode(SaveMode.Overwrite)
-      .option("compression","gzip")
+      .option("compression", "gzip")
       .text(graphPath + "/datasource")
   }
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToResult.scala

@@ -5,16 +5,14 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils
-import eu.dnetlib.dhp.schema.oaf.{Datasource, Instance, OpenAccessRoute, Publication}
+import eu.dnetlib.dhp.schema.oaf.{Instance, OpenAccessRoute, Publication}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
-
 import scala.collection.JavaConverters._
 
-
 object SparkApplyHostedByMapToResult {
 
   def applyHBtoPubs(join: Dataset[EntityInfo], pubs: Dataset[Publication]) = {
@@ -25,7 +23,7 @@ object SparkApplyHostedByMapToResult {
           val ei: EntityInfo = t2._2
           val i = p.getInstance().asScala
           if (i.size == 1) {
-            val inst: Instance = i(0)
+            val inst: Instance = i.head
             inst.getHostedby.setKey(ei.getHostedById)
             inst.getHostedby.setValue(ei.getName)
             if (ei.getOpenAccess) {
@@ -39,6 +37,7 @@ object SparkApplyHostedByMapToResult {
         p
       })(Encoders.bean(classOf[Publication]))
   }
+
   def main(args: Array[String]): Unit = {
 
 
@@ -67,18 +66,18 @@ object SparkApplyHostedByMapToResult {
     implicit val mapEncoderEinfo: Encoder[EntityInfo] = Encoders.bean(classOf[EntityInfo])
     val mapper = new ObjectMapper()
 
-    val pubs : Dataset[Publication] = spark.read.textFile(graphPath + "/publication")
+    val pubs: Dataset[Publication] = spark.read.textFile(graphPath + "/publication")
       .map(r => mapper.readValue(r, classOf[Publication]))
 
-    val pinfo : Dataset[EntityInfo] = spark.read.textFile(preparedInfoPath)
+    val pinfo: Dataset[EntityInfo] = spark.read.textFile(preparedInfoPath)
-        .map(ei => mapper.readValue(ei, classOf[EntityInfo]))
+      .map(ei => mapper.readValue(ei, classOf[EntityInfo]))
 
-    applyHBtoPubs(pinfo, pubs).write.mode(SaveMode.Overwrite).option("compression","gzip").json(outputPath)
+    applyHBtoPubs(pinfo, pubs).write.mode(SaveMode.Overwrite).option("compression", "gzip").json(outputPath)
 
     spark.read.textFile(outputPath)
       .write
       .mode(SaveMode.Overwrite)
-      .option("compression","gzip")
+      .option("compression", "gzip")
       .text(graphPath + "/publication")
   }
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkPrepareHostedByInfoToApply.scala

@@ -3,61 +3,58 @@ package eu.dnetlib.dhp.oa.graph.hostedbymap
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
-
 import eu.dnetlib.dhp.schema.oaf.{Journal, Publication}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
 import org.slf4j.{Logger, LoggerFactory}
 
-
-
 object SparkPrepareHostedByInfoToApply {
 
   implicit val mapEncoderPInfo: Encoder[EntityInfo] = Encoders.bean(classOf[EntityInfo])
 
-  def getList(id: String, j: Journal, name: String ) : List[EntityInfo] = {
+  def getList(id: String, j: Journal, name: String): List[EntityInfo] = {
-    var lst:List[EntityInfo] = List()
+    var lst: List[EntityInfo] = List()
 
 
-    if (j.getIssnLinking != null && !j.getIssnLinking.equals("")){
+    if (j.getIssnLinking != null && !j.getIssnLinking.equals("")) {
       lst = EntityInfo.newInstance(id, j.getIssnLinking, name) :: lst
     }
-    if (j.getIssnOnline != null && !j.getIssnOnline.equals("")){
+    if (j.getIssnOnline != null && !j.getIssnOnline.equals("")) {
       lst = EntityInfo.newInstance(id, j.getIssnOnline, name) :: lst
     }
-    if (j.getIssnPrinted != null && !j.getIssnPrinted.equals("")){
+    if (j.getIssnPrinted != null && !j.getIssnPrinted.equals("")) {
       lst = EntityInfo.newInstance(id, j.getIssnPrinted, name) :: lst
     }
     lst
   }
 
-  def prepareResultInfo(spark:SparkSession, publicationPath:String) : Dataset[EntityInfo] = {
+  def prepareResultInfo(spark: SparkSession, publicationPath: String): Dataset[EntityInfo] = {
     implicit val mapEncoderPubs: Encoder[Publication] = Encoders.bean(classOf[Publication])
 
     val mapper = new ObjectMapper()
 
-    val dd : Dataset[Publication] = spark.read.textFile(publicationPath)
+    val dd: Dataset[Publication] = spark.read.textFile(publicationPath)
       .map(r => mapper.readValue(r, classOf[Publication]))
 
-    dd.filter(p => p.getJournal != null ).flatMap(p => getList(p.getId, p.getJournal, ""))
+    dd.filter(p => p.getJournal != null).flatMap(p => getList(p.getId, p.getJournal, ""))
 
   }
 
 
-  def toEntityInfo(input:String): EntityInfo = {
+  def toEntityInfo(input: String): EntityInfo = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
 
     lazy val json: json4s.JValue = parse(input)
-    val c :Map[String,HostedByItemType] = json.extract[Map[String, HostedByItemType]]
+    val c: Map[String, HostedByItemType] = json.extract[Map[String, HostedByItemType]]
     toEntityItem(c.keys.head, c.values.head)
   }
 
 
-  def toEntityItem(journal_id: String , hbi: HostedByItemType): EntityInfo = {
+  def toEntityItem(journal_id: String, hbi: HostedByItemType): EntityInfo = {
 
     EntityInfo.newInstance(hbi.id, journal_id, hbi.officialname, hbi.openAccess)
 
@@ -67,7 +64,7 @@ object SparkPrepareHostedByInfoToApply {
     Aggregators.resultToSingleId(res.joinWith(hbm, res.col("journalId").equalTo(hbm.col("journalId")), "left")
       .map(t2 => {
         val res: EntityInfo = t2._1
-        if(t2._2 != null ){
+        if (t2._2 != null) {
           val ds = t2._2
           res.setHostedById(ds.getId)
           res.setOpenAccess(ds.getOpenAccess)
@@ -107,10 +104,10 @@ object SparkPrepareHostedByInfoToApply {
 
 
     //STEP1: read the hostedbymap and transform it in EntityInfo
-    val hostedByInfo:Dataset[EntityInfo] = spark.createDataset(spark.sparkContext.textFile(hostedByMapPath)).map(toEntityInfo)
+    val hostedByInfo: Dataset[EntityInfo] = spark.createDataset(spark.sparkContext.textFile(hostedByMapPath)).map(toEntityInfo)
 
-   //STEP2: create association (publication, issn), (publication, eissn), (publication, lissn)
+    //STEP2: create association (publication, issn), (publication, eissn), (publication, lissn)
-    val resultInfoDataset:Dataset[EntityInfo] = prepareResultInfo(spark, graphPath + "/publication")
+    val resultInfoDataset: Dataset[EntityInfo] = prepareResultInfo(spark, graphPath + "/publication")
 
     //STEP3: left join resultInfo with hostedByInfo on journal_id. Reduction of all the results with the same id in just
     //one entry (one result could be associated to issn and eissn and so possivly matching more than once against the map)

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkProduceHostedByMap.scala

@@ -1,41 +1,39 @@
 package eu.dnetlib.dhp.oa.graph.hostedbymap
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.{DOAJModel, UnibiGoldModel}
 import eu.dnetlib.dhp.schema.oaf.Datasource
 import org.apache.commons.io.IOUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
-import com.fasterxml.jackson.databind.ObjectMapper
+
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.Path
 import java.io.PrintWriter
 
-import org.apache.hadoop.io.compress.GzipCodec
-
-
 object SparkProduceHostedByMap {
 
 
   implicit val tupleForJoinEncoder: Encoder[(String, HostedByItemType)] = Encoders.tuple(Encoders.STRING, Encoders.product[HostedByItemType])
 
 
-  def toHostedByItemType(input: ((HostedByInfo, HostedByInfo), HostedByInfo)) : HostedByItemType = {
+  def toHostedByItemType(input: ((HostedByInfo, HostedByInfo), HostedByInfo)): HostedByItemType = {
     val openaire: HostedByInfo = input._1._1
     val doaj: HostedByInfo = input._1._2
     val gold: HostedByInfo = input._2
     val isOpenAccess: Boolean = doaj == null && gold == null
 
     openaire.journal_id match {
-      case Constants.ISSN =>  HostedByItemType(openaire.id, openaire.officialname, openaire.journal_id, "", "", isOpenAccess)
+      case Constants.ISSN => HostedByItemType(openaire.id, openaire.officialname, openaire.journal_id, "", "", isOpenAccess)
-      case Constants.EISSN =>  HostedByItemType(openaire.id, openaire.officialname, "", openaire.journal_id, "", isOpenAccess)
+      case Constants.EISSN => HostedByItemType(openaire.id, openaire.officialname, "", openaire.journal_id, "", isOpenAccess)
-      case Constants.ISSNL =>  HostedByItemType(openaire.id, openaire.officialname, "", "", openaire.journal_id, isOpenAccess)
+      case Constants.ISSNL => HostedByItemType(openaire.id, openaire.officialname, "", "", openaire.journal_id, isOpenAccess)
 
       // catch the default with a variable so you can print it
-      case whoa =>  null
+      case whoa => null
     }
   }
 
@@ -44,7 +42,7 @@ object SparkProduceHostedByMap {
 
     implicit val formats = org.json4s.DefaultFormats
 
-    val map: Map [String, HostedByItemType] = Map (input._1 -> input._2 )
+    val map: Map[String, HostedByItemType] = Map(input._1 -> input._2)
 
     Serialization.write(map)
 
@@ -52,34 +50,33 @@ object SparkProduceHostedByMap {
   }
 
 
-
+  def getHostedByItemType(id: String, officialname: String, issn: String, eissn: String, issnl: String, oa: Boolean): HostedByItemType = {
-  def getHostedByItemType(id:String, officialname: String, issn:String, eissn:String, issnl:String, oa:Boolean): HostedByItemType = {
+    if (issn != null) {
-    if(issn != null){
+      if (eissn != null) {
-      if(eissn != null){
+        if (issnl != null) {
-        if(issnl != null){
+          HostedByItemType(id, officialname, issn, eissn, issnl, oa)
-          HostedByItemType(id, officialname, issn, eissn, issnl  , oa)
+        } else {
-        }else{
+          HostedByItemType(id, officialname, issn, eissn, "", oa)
-          HostedByItemType(id, officialname, issn, eissn, ""  , oa)
         }
-      }else{
+      } else {
-        if(issnl != null){
+        if (issnl != null) {
-          HostedByItemType(id, officialname, issn, "", issnl  , oa)
+          HostedByItemType(id, officialname, issn, "", issnl, oa)
-        }else{
+        } else {
-          HostedByItemType(id, officialname, issn, "", ""  , oa)
+          HostedByItemType(id, officialname, issn, "", "", oa)
         }
       }
-    }else{
+    } else {
-      if(eissn != null){
+      if (eissn != null) {
-        if(issnl != null){
+        if (issnl != null) {
-          HostedByItemType(id, officialname, "", eissn, issnl  , oa)
+          HostedByItemType(id, officialname, "", eissn, issnl, oa)
-        }else{
+        } else {
-          HostedByItemType(id, officialname, "", eissn, ""  , oa)
+          HostedByItemType(id, officialname, "", eissn, "", oa)
         }
-      }else{
+      } else {
-        if(issnl != null){
+        if (issnl != null) {
-          HostedByItemType(id, officialname, "", "", issnl  , oa)
+          HostedByItemType(id, officialname, "", "", issnl, oa)
-        }else{
+        } else {
-          HostedByItemType("", "", "", "", ""  , oa)
+          HostedByItemType("", "", "", "", "", oa)
         }
       }
     }
@@ -90,10 +87,10 @@ object SparkProduceHostedByMap {
 
       return getHostedByItemType(dats.getId, dats.getOfficialname.getValue, dats.getJournal.getIssnPrinted, dats.getJournal.getIssnOnline, dats.getJournal.getIssnLinking, false)
     }
-    HostedByItemType("","","","","",false)
+    HostedByItemType("", "", "", "", "", false)
   }
 
-  def oaHostedByDataset(spark:SparkSession, datasourcePath : String) : Dataset[HostedByItemType] = {
+  def oaHostedByDataset(spark: SparkSession, datasourcePath: String): Dataset[HostedByItemType] = {
 
     import spark.implicits._
 
@@ -102,10 +99,10 @@ object SparkProduceHostedByMap {
 
     implicit var encoderD = Encoders.kryo[Datasource]
 
-    val dd : Dataset[Datasource] = spark.read.textFile(datasourcePath)
+    val dd: Dataset[Datasource] = spark.read.textFile(datasourcePath)
       .map(r => mapper.readValue(r, classOf[Datasource]))
 
-    dd.map{ddt => oaToHostedbyItemType(ddt)}.filter(hb => !(hb.id.equals("")))
+    dd.map { ddt => oaToHostedbyItemType(ddt) }.filter(hb => !(hb.id.equals("")))
 
   }
 
@@ -115,17 +112,17 @@ object SparkProduceHostedByMap {
   }
 
 
-  def goldHostedByDataset(spark:SparkSession, datasourcePath:String) : Dataset[HostedByItemType] = {
+  def goldHostedByDataset(spark: SparkSession, datasourcePath: String): Dataset[HostedByItemType] = {
     import spark.implicits._
 
     implicit val mapEncoderUnibi: Encoder[UnibiGoldModel] = Encoders.kryo[UnibiGoldModel]
 
     val mapper = new ObjectMapper()
 
-    val dd : Dataset[UnibiGoldModel] = spark.read.textFile(datasourcePath)
+    val dd: Dataset[UnibiGoldModel] = spark.read.textFile(datasourcePath)
       .map(r => mapper.readValue(r, classOf[UnibiGoldModel]))
 
-    dd.map{ddt => goldToHostedbyItemType(ddt)}.filter(hb => !(hb.id.equals("")))
+    dd.map { ddt => goldToHostedbyItemType(ddt) }.filter(hb => !(hb.id.equals("")))
 
   }
 
@@ -134,41 +131,40 @@ object SparkProduceHostedByMap {
     return getHostedByItemType(Constants.DOAJ, doaj.getJournalTitle, doaj.getIssn, doaj.getEissn, "", true)
   }
 
-  def doajHostedByDataset(spark:SparkSession, datasourcePath:String) : Dataset[HostedByItemType] = {
+  def doajHostedByDataset(spark: SparkSession, datasourcePath: String): Dataset[HostedByItemType] = {
     import spark.implicits._
 
     implicit val mapEncoderDOAJ: Encoder[DOAJModel] = Encoders.kryo[DOAJModel]
 
     val mapper = new ObjectMapper()
 
-    val dd : Dataset[DOAJModel] = spark.read.textFile(datasourcePath)
+    val dd: Dataset[DOAJModel] = spark.read.textFile(datasourcePath)
       .map(r => mapper.readValue(r, classOf[DOAJModel]))
 
-    dd.map{ddt => doajToHostedbyItemType(ddt)}.filter(hb => !(hb.id.equals("")))
+    dd.map { ddt => doajToHostedbyItemType(ddt) }.filter(hb => !(hb.id.equals("")))
 
   }
 
   def toList(input: HostedByItemType): List[(String, HostedByItemType)] = {
-    var lst : List[(String, HostedByItemType)] = List()
+    var lst: List[(String, HostedByItemType)] = List()
-    if(!input.issn.equals("")){
+    if (!input.issn.equals("")) {
       lst = (input.issn, input) :: lst
     }
-    if(!input.eissn.equals("")){
+    if (!input.eissn.equals("")) {
       lst = (input.eissn, input) :: lst
     }
-    if(!input.lissn.equals("")){
+    if (!input.lissn.equals("")) {
       lst = (input.lissn, input) :: lst
     }
     lst
   }
 
 
-
+  def writeToHDFS(input: Array[String], outputPath: String, hdfsNameNode: String): Unit = {
-  def writeToHDFS(input: Array[String], outputPath: String, hdfsNameNode : String):Unit = {
     val conf = new Configuration()
 
     conf.set("fs.defaultFS", hdfsNameNode)
-    val fs= FileSystem.get(conf)
+    val fs = FileSystem.get(conf)
     val output = fs.create(new Path(outputPath))
     val writer = new PrintWriter(output)
     try {
@@ -182,7 +178,6 @@ object SparkProduceHostedByMap {
   }
 
 
-
   def main(args: Array[String]): Unit = {
 
     val logger: Logger = LoggerFactory.getLogger(getClass)
@@ -213,7 +208,7 @@ object SparkProduceHostedByMap {
       .union(doajHostedByDataset(spark, workingDirPath + "/doaj.json"))
       .flatMap(hbi => toList(hbi))).filter(hbi => hbi._2.id.startsWith("10|"))
       .map(hbi => toHostedByMap(hbi))(Encoders.STRING)
-      .rdd.saveAsTextFile(outputPath , classOf[GzipCodec])
+      .rdd.saveAsTextFile(outputPath, classOf[GzipCodec])
 
 
   }

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/raw/CopyHdfsOafSparkApplication.scala

@@ -4,17 +4,11 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.common.HdfsSupport
 import eu.dnetlib.dhp.schema.common.ModelSupport
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo
 import eu.dnetlib.dhp.schema.oaf.Oaf
 import eu.dnetlib.dhp.utils.DHPUtils
-import org.apache.commons.io.IOUtils
-import org.apache.commons.lang3.StringUtils
-import org.apache.http.client.methods.HttpGet
-import org.apache.http.impl.client.HttpClients
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
 import org.apache.spark.{SparkConf, SparkContext}
 import org.slf4j.LoggerFactory
-
 import scala.collection.JavaConverters._
 import scala.io.Source
 
@@ -59,7 +53,7 @@ object CopyHdfsOafSparkApplication {
     if (validPaths.nonEmpty) {
       val oaf = spark.read.load(validPaths: _*).as[Oaf]
       val mapper = new ObjectMapper()
-      val l =ModelSupport.oafTypes.entrySet.asScala.map(e => e.getKey).toList
+      val l = ModelSupport.oafTypes.entrySet.asScala.map(e => e.getKey).toList
       l.foreach(
         e =>
           oaf.filter(o => o.getClass.getSimpleName.equalsIgnoreCase(e))

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/resolution/SparkResolveEntities.scala

@@ -2,9 +2,8 @@ package eu.dnetlib.dhp.oa.graph.resolution
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.common.HdfsSupport
 import eu.dnetlib.dhp.schema.common.EntityType
-import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{Dataset => OafDataset,_}
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.spark.SparkConf
@@ -75,7 +74,7 @@ object SparkResolveEntities {
     }
   }
 
-  def generateResolvedEntities(spark: SparkSession, workingPath: String, graphBasePath: String, targetPath:String) = {
+  def generateResolvedEntities(spark: SparkSession, workingPath: String, graphBasePath: String, targetPath: String) = {
 
     implicit val resEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
     import spark.implicits._

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/resolution/SparkResolveRelation.scala

@@ -3,7 +3,7 @@ package eu.dnetlib.dhp.oa.graph.resolution
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.common.HdfsSupport
-import eu.dnetlib.dhp.schema.oaf.{Relation, Result}
+import eu.dnetlib.dhp.schema.oaf.Relation
 import eu.dnetlib.dhp.utils.DHPUtils
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.fs.{FileSystem, Path}

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/sx/graphimport/SparkDataciteToOAF.scala

@@ -18,7 +18,6 @@ object SparkDataciteToOAF {
         .config(conf)
         .appName(getClass.getSimpleName)
         .master(parser.get("master")).getOrCreate()
-    import spark.implicits._
 
 
     val sc = spark.sparkContext

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkConvertDatasetToJsonRDD.scala

@@ -2,7 +2,7 @@ package eu.dnetlib.dhp.sx.graph
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Oaf, OtherResearchProduct, Publication, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.Result
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.SparkConf
@@ -29,13 +29,13 @@ object SparkConvertDatasetToJsonRDD {
     val targetPath = parser.get("targetPath")
     log.info(s"targetPath  -> $targetPath")
 
-    val resultObject = List("publication","dataset","software", "otherResearchProduct")
+    val resultObject = List("publication", "dataset", "software", "otherResearchProduct")
     val mapper = new ObjectMapper()
-    implicit  val oafEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
+    implicit val oafEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
 
 
-    resultObject.foreach{item =>
+    resultObject.foreach { item =>
-      spark.read.load(s"$sourcePath/$item").as[Result].map(r=> mapper.writeValueAsString(r))(Encoders.STRING).rdd.saveAsTextFile(s"$targetPath/${item.toLowerCase}", classOf[GzipCodec])
+      spark.read.load(s"$sourcePath/$item").as[Result].map(r => mapper.writeValueAsString(r))(Encoders.STRING).rdd.saveAsTextFile(s"$targetPath/${item.toLowerCase}", classOf[GzipCodec])
     }
   }
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkConvertObjectToJson.scala

@@ -5,10 +5,10 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.sx.scholix.Scholix
 import eu.dnetlib.dhp.schema.sx.summary.ScholixSummary
 import org.apache.commons.io.IOUtils
+import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
-import org.apache.hadoop.io.compress._
 
 object SparkConvertObjectToJson {
 
@@ -32,8 +32,8 @@ object SparkConvertObjectToJson {
     log.info(s"objectType  -> $objectType")
 
 
-    implicit val scholixEncoder :Encoder[Scholix]= Encoders.kryo[Scholix]
+    implicit val scholixEncoder: Encoder[Scholix] = Encoders.kryo[Scholix]
-    implicit val summaryEncoder :Encoder[ScholixSummary]= Encoders.kryo[ScholixSummary]
+    implicit val summaryEncoder: Encoder[ScholixSummary] = Encoders.kryo[ScholixSummary]
 
 
     val mapper = new ObjectMapper
@@ -42,11 +42,11 @@ object SparkConvertObjectToJson {
       case "scholix" =>
         log.info("Serialize Scholix")
         val d: Dataset[Scholix] = spark.read.load(sourcePath).as[Scholix]
-        d.map(s => mapper.writeValueAsString(s))(Encoders.STRING).rdd.repartition(6000).saveAsTextFile(targetPath,  classOf[GzipCodec])
+        d.map(s => mapper.writeValueAsString(s))(Encoders.STRING).rdd.repartition(6000).saveAsTextFile(targetPath, classOf[GzipCodec])
       case "summary" =>
         log.info("Serialize Summary")
         val d: Dataset[ScholixSummary] = spark.read.load(sourcePath).as[ScholixSummary]
-        d.map(s => mapper.writeValueAsString(s))(Encoders.STRING).rdd.repartition(1000).saveAsTextFile(targetPath,  classOf[GzipCodec])
+        d.map(s => mapper.writeValueAsString(s))(Encoders.STRING).rdd.repartition(1000).saveAsTextFile(targetPath, classOf[GzipCodec])
     }
   }
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkConvertRDDtoDataset.scala

@@ -2,11 +2,12 @@ package eu.dnetlib.dhp.sx.graph
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Relation, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Relation, Software,Dataset => OafDataset}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
+
 object SparkConvertRDDtoDataset {
 
   def main(args: Array[String]): Unit = {
@@ -31,39 +32,39 @@ object SparkConvertRDDtoDataset {
     val entityPath = s"$t/entities"
     val relPath = s"$t/relation"
     val mapper = new ObjectMapper()
-    implicit  val datasetEncoder: Encoder[OafDataset] = Encoders.kryo(classOf[OafDataset])
+    implicit val datasetEncoder: Encoder[OafDataset] = Encoders.kryo(classOf[OafDataset])
-    implicit  val publicationEncoder: Encoder[Publication] = Encoders.kryo(classOf[Publication])
+    implicit val publicationEncoder: Encoder[Publication] = Encoders.kryo(classOf[Publication])
-    implicit  val relationEncoder: Encoder[Relation] = Encoders.kryo(classOf[Relation])
+    implicit val relationEncoder: Encoder[Relation] = Encoders.kryo(classOf[Relation])
-    implicit  val orpEncoder: Encoder[OtherResearchProduct] = Encoders.kryo(classOf[OtherResearchProduct])
+    implicit val orpEncoder: Encoder[OtherResearchProduct] = Encoders.kryo(classOf[OtherResearchProduct])
-    implicit  val softwareEncoder: Encoder[Software] = Encoders.kryo(classOf[Software])
+    implicit val softwareEncoder: Encoder[Software] = Encoders.kryo(classOf[Software])
 
 
     log.info("Converting dataset")
-    val rddDataset =spark.sparkContext.textFile(s"$sourcePath/dataset").map(s => mapper.readValue(s, classOf[OafDataset]))
+    val rddDataset = spark.sparkContext.textFile(s"$sourcePath/dataset").map(s => mapper.readValue(s, classOf[OafDataset]))
     spark.createDataset(rddDataset).as[OafDataset].write.mode(SaveMode.Overwrite).save(s"$entityPath/dataset")
 
 
     log.info("Converting publication")
-    val rddPublication =spark.sparkContext.textFile(s"$sourcePath/publication").map(s => mapper.readValue(s, classOf[Publication]))
+    val rddPublication = spark.sparkContext.textFile(s"$sourcePath/publication").map(s => mapper.readValue(s, classOf[Publication]))
     spark.createDataset(rddPublication).as[Publication].write.mode(SaveMode.Overwrite).save(s"$entityPath/publication")
 
     log.info("Converting software")
-    val rddSoftware =spark.sparkContext.textFile(s"$sourcePath/software").map(s => mapper.readValue(s, classOf[Software]))
+    val rddSoftware = spark.sparkContext.textFile(s"$sourcePath/software").map(s => mapper.readValue(s, classOf[Software]))
     spark.createDataset(rddSoftware).as[Software].write.mode(SaveMode.Overwrite).save(s"$entityPath/software")
 
     log.info("Converting otherresearchproduct")
-    val rddOtherResearchProduct =spark.sparkContext.textFile(s"$sourcePath/otherresearchproduct").map(s => mapper.readValue(s, classOf[OtherResearchProduct]))
+    val rddOtherResearchProduct = spark.sparkContext.textFile(s"$sourcePath/otherresearchproduct").map(s => mapper.readValue(s, classOf[OtherResearchProduct]))
     spark.createDataset(rddOtherResearchProduct).as[OtherResearchProduct].write.mode(SaveMode.Overwrite).save(s"$entityPath/otherresearchproduct")
 
 
     log.info("Converting Relation")
 
 
-    val relationSemanticFilter = List("cites", "iscitedby","merges", "ismergedin")
+    val relationSemanticFilter = List("cites", "iscitedby", "merges", "ismergedin")
 
-    val rddRelation =spark.sparkContext.textFile(s"$sourcePath/relation")
+    val rddRelation = spark.sparkContext.textFile(s"$sourcePath/relation")
       .map(s => mapper.readValue(s, classOf[Relation]))
-      .filter(r=> r.getSource.startsWith("50") && r.getTarget.startsWith("50"))
+      .filter(r => r.getSource.startsWith("50") && r.getTarget.startsWith("50"))
       .filter(r => !relationSemanticFilter.exists(k => k.equalsIgnoreCase(r.getRelClass)))
     spark.createDataset(rddRelation).as[Relation].write.mode(SaveMode.Overwrite).save(s"$relPath")
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateInputGraph.scala

@@ -1,14 +1,12 @@
 package eu.dnetlib.dhp.sx.graph
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Oaf, OtherResearchProduct, Publication, Relation, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{Dataset => OafDataset,_}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
-
-
 object SparkCreateInputGraph {
 
   def main(args: Array[String]): Unit = {
@@ -33,7 +31,7 @@ object SparkCreateInputGraph {
 
     )
 
-    implicit  val oafEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
+    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
     implicit val publicationEncoder: Encoder[Publication] = Encoders.kryo(classOf[Publication])
     implicit val datasetEncoder: Encoder[OafDataset] = Encoders.kryo(classOf[OafDataset])
     implicit val softwareEncoder: Encoder[Software] = Encoders.kryo(classOf[Software])
@@ -41,16 +39,13 @@ object SparkCreateInputGraph {
     implicit val relEncoder: Encoder[Relation] = Encoders.kryo(classOf[Relation])
 
 
-
-
-
     val sourcePath = parser.get("sourcePath")
     log.info(s"sourcePath  -> $sourcePath")
     val targetPath = parser.get("targetPath")
     log.info(s"targetPath  -> $targetPath")
 
 
-    val oafDs:Dataset[Oaf] = spark.read.load(s"$sourcePath/*").as[Oaf]
+    val oafDs: Dataset[Oaf] = spark.read.load(s"$sourcePath/*").as[Oaf]
 
 
     log.info("Extract Publication")
@@ -70,27 +65,27 @@ object SparkCreateInputGraph {
 
     resultObject.foreach { r =>
       log.info(s"Make ${r._1} unique")
-      makeDatasetUnique(s"$targetPath/extracted/${r._1}",s"$targetPath/preprocess/${r._1}",spark, r._2)
+      makeDatasetUnique(s"$targetPath/extracted/${r._1}", s"$targetPath/preprocess/${r._1}", spark, r._2)
     }
   }
 
 
-  def extractEntities[T <: Oaf ](oafDs:Dataset[Oaf], targetPath:String, clazz:Class[T], log:Logger) :Unit = {
+  def extractEntities[T <: Oaf](oafDs: Dataset[Oaf], targetPath: String, clazz: Class[T], log: Logger): Unit = {
 
-    implicit  val resEncoder: Encoder[T] = Encoders.kryo(clazz)
+    implicit val resEncoder: Encoder[T] = Encoders.kryo(clazz)
     log.info(s"Extract ${clazz.getSimpleName}")
     oafDs.filter(o => o.isInstanceOf[T]).map(p => p.asInstanceOf[T]).write.mode(SaveMode.Overwrite).save(targetPath)
   }
 
 
-  def makeDatasetUnique[T <: Result ](sourcePath:String, targetPath:String, spark:SparkSession, clazz:Class[T]) :Unit = {
+  def makeDatasetUnique[T <: Result](sourcePath: String, targetPath: String, spark: SparkSession, clazz: Class[T]): Unit = {
     import spark.implicits._
 
-    implicit  val resEncoder: Encoder[T] = Encoders.kryo(clazz)
+    implicit val resEncoder: Encoder[T] = Encoders.kryo(clazz)
 
-    val ds:Dataset[T] = spark.read.load(sourcePath).as[T]
+    val ds: Dataset[T] = spark.read.load(sourcePath).as[T]
 
-    ds.groupByKey(_.getId).reduceGroups{(x,y) =>
+    ds.groupByKey(_.getId).reduceGroups { (x, y) =>
       x.mergeFrom(y)
       x
     }.map(_._2).write.mode(SaveMode.Overwrite).save(targetPath)

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholix.scala

@@ -9,7 +9,7 @@ import eu.dnetlib.dhp.sx.graph.scholix.ScholixUtils.RelatedEntities
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.functions.count
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkCreateScholix {
@@ -42,7 +42,7 @@ object SparkCreateScholix {
 
 
     val relationDS: Dataset[(String, Relation)] = spark.read.load(relationPath).as[Relation]
-      .filter(r => (r.getDataInfo== null || r.getDataInfo.getDeletedbyinference == false) && !r.getRelClass.toLowerCase.contains("merge"))
+      .filter(r => (r.getDataInfo == null || r.getDataInfo.getDeletedbyinference == false) && !r.getRelClass.toLowerCase.contains("merge"))
       .map(r => (r.getSource, r))(Encoders.tuple(Encoders.STRING, relEncoder))
 
     val summaryDS: Dataset[(String, ScholixSummary)] = spark.read.load(summaryPath).as[ScholixSummary]
@@ -51,54 +51,54 @@ object SparkCreateScholix {
 
     relationDS.joinWith(summaryDS, relationDS("_1").equalTo(summaryDS("_1")), "left")
       .map { input: ((String, Relation), (String, ScholixSummary)) =>
-        if (input._1!= null && input._2!= null) {
+        if (input._1 != null && input._2 != null) {
           val rel: Relation = input._1._2
           val source: ScholixSummary = input._2._2
           (rel.getTarget, ScholixUtils.scholixFromSource(rel, source))
         }
-        else  null
+        else null
       }(Encoders.tuple(Encoders.STRING, scholixEncoder))
-      .filter(r => r!= null)
+      .filter(r => r != null)
       .write.mode(SaveMode.Overwrite).save(s"$targetPath/scholix_from_source")
 
     val scholixSource: Dataset[(String, Scholix)] = spark.read.load(s"$targetPath/scholix_from_source").as[(String, Scholix)](Encoders.tuple(Encoders.STRING, scholixEncoder))
 
     scholixSource.joinWith(summaryDS, scholixSource("_1").equalTo(summaryDS("_1")), "left")
       .map { input: ((String, Scholix), (String, ScholixSummary)) =>
-        if (input._2== null) {
+        if (input._2 == null) {
           null
         } else {
           val s: Scholix = input._1._2
           val target: ScholixSummary = input._2._2
           ScholixUtils.generateCompleteScholix(s, target)
         }
-      }.filter(s => s!= null).write.mode(SaveMode.Overwrite).save(s"$targetPath/scholix_one_verse")
+      }.filter(s => s != null).write.mode(SaveMode.Overwrite).save(s"$targetPath/scholix_one_verse")
 
 
     val scholix_o_v: Dataset[Scholix] = spark.read.load(s"$targetPath/scholix_one_verse").as[Scholix]
 
     scholix_o_v.flatMap(s => List(s, ScholixUtils.createInverseScholixRelation(s))).as[Scholix]
-      .map(s=> (s.getIdentifier,s))(Encoders.tuple(Encoders.STRING, scholixEncoder))
+      .map(s => (s.getIdentifier, s))(Encoders.tuple(Encoders.STRING, scholixEncoder))
       .groupByKey(_._1)
       .agg(ScholixUtils.scholixAggregator.toColumn)
       .map(s => s._2)
       .write.mode(SaveMode.Overwrite).save(s"$targetPath/scholix")
 
-    val scholix_final:Dataset[Scholix] = spark.read.load(s"$targetPath/scholix").as[Scholix]
+    val scholix_final: Dataset[Scholix] = spark.read.load(s"$targetPath/scholix").as[Scholix]
 
-    val stats:Dataset[(String,String,Long)]= scholix_final.map(s => (s.getSource.getDnetIdentifier, s.getTarget.getObjectType)).groupBy("_1", "_2").agg(count("_1")).as[(String,String,Long)]
+    val stats: Dataset[(String, String, Long)] = scholix_final.map(s => (s.getSource.getDnetIdentifier, s.getTarget.getObjectType)).groupBy("_1", "_2").agg(count("_1")).as[(String, String, Long)]
 
 
     stats
-      .map(s => RelatedEntities(s._1, if ("dataset".equalsIgnoreCase(s._2)) s._3 else  0, if ("publication".equalsIgnoreCase(s._2)) s._3 else  0 ))
+      .map(s => RelatedEntities(s._1, if ("dataset".equalsIgnoreCase(s._2)) s._3 else 0, if ("publication".equalsIgnoreCase(s._2)) s._3 else 0))
       .groupByKey(_.id)
-      .reduceGroups((a, b) => RelatedEntities(a.id, a.relatedDataset+b.relatedDataset, a.relatedPublication+b.relatedPublication))
+      .reduceGroups((a, b) => RelatedEntities(a.id, a.relatedDataset + b.relatedDataset, a.relatedPublication + b.relatedPublication))
       .map(_._2)
       .write.mode(SaveMode.Overwrite).save(s"$targetPath/related_entities")
 
-    val relatedEntitiesDS:Dataset[RelatedEntities] = spark.read.load(s"$targetPath/related_entities").as[RelatedEntities].filter(r => r.relatedPublication>0 || r.relatedDataset > 0)
+    val relatedEntitiesDS: Dataset[RelatedEntities] = spark.read.load(s"$targetPath/related_entities").as[RelatedEntities].filter(r => r.relatedPublication > 0 || r.relatedDataset > 0)
 
-    relatedEntitiesDS.joinWith(summaryDS, relatedEntitiesDS("id").equalTo(summaryDS("_1")), "inner").map{i =>
+    relatedEntitiesDS.joinWith(summaryDS, relatedEntitiesDS("id").equalTo(summaryDS("_1")), "inner").map { i =>
       val re = i._1
       val sum = i._2._2
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateSummaryObject.scala

@@ -6,7 +6,7 @@ import eu.dnetlib.dhp.schema.sx.summary.ScholixSummary
 import eu.dnetlib.dhp.sx.graph.scholix.ScholixUtils
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkCreateSummaryObject {
@@ -28,15 +28,15 @@ object SparkCreateSummaryObject {
     val targetPath = parser.get("targetPath")
     log.info(s"targetPath  -> $targetPath")
 
-    implicit val resultEncoder:Encoder[Result] = Encoders.kryo[Result]
+    implicit val resultEncoder: Encoder[Result] = Encoders.kryo[Result]
-    implicit val oafEncoder:Encoder[Oaf] = Encoders.kryo[Oaf]
+    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
 
-    implicit val summaryEncoder:Encoder[ScholixSummary] = Encoders.kryo[ScholixSummary]
+    implicit val summaryEncoder: Encoder[ScholixSummary] = Encoders.kryo[ScholixSummary]
 
 
-    val ds:Dataset[Result] = spark.read.load(s"$sourcePath/*").as[Result].filter(r=>r.getDataInfo== null ||  r.getDataInfo.getDeletedbyinference== false)
+    val ds: Dataset[Result] = spark.read.load(s"$sourcePath/*").as[Result].filter(r => r.getDataInfo == null || r.getDataInfo.getDeletedbyinference == false)
 
-    ds.repartition(6000).map(r => ScholixUtils.resultToSummary(r)).filter(s => s!= null).write.mode(SaveMode.Overwrite).save(targetPath)
+    ds.repartition(6000).map(r => ScholixUtils.resultToSummary(r)).filter(s => s != null).write.mode(SaveMode.Overwrite).save(targetPath)
 
   }
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/pangaea/PangaeaUtils.scala

@@ -5,6 +5,7 @@ import org.apache.spark.sql.{Encoder, Encoders}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
+
 import java.util.regex.Pattern
 import scala.language.postfixOps
 import scala.xml.{Elem, Node, XML}

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/pangaea/SparkGeneratePanagaeaDataset.scala

@@ -2,11 +2,11 @@ package eu.dnetlib.dhp.sx.graph.pangaea
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.spark.rdd.RDD
-import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.{SparkConf, SparkContext}
 import org.slf4j.{Logger, LoggerFactory}
-
 import scala.collection.JavaConverters._
+
 import scala.io.Source
 
 object SparkGeneratePanagaeaDataset {
@@ -28,17 +28,17 @@ object SparkGeneratePanagaeaDataset {
 
     parser.getObjectMap.asScala.foreach(s => logger.info(s"${s._1} -> ${s._2}"))
     logger.info("Converting sequential file into Dataset")
-    val sc:SparkContext = spark.sparkContext
+    val sc: SparkContext = spark.sparkContext
 
-    val workingPath:String = parser.get("workingPath")
+    val workingPath: String = parser.get("workingPath")
 
     implicit val pangaeaEncoders: Encoder[PangaeaDataModel] = Encoders.kryo[PangaeaDataModel]
 
-    val inputRDD:RDD[PangaeaDataModel] =  sc.textFile(s"$workingPath/update").map(s => PangaeaUtils.toDataset(s))
+    val inputRDD: RDD[PangaeaDataModel] = sc.textFile(s"$workingPath/update").map(s => PangaeaUtils.toDataset(s))
 
     spark.createDataset(inputRDD).as[PangaeaDataModel]
-      .map(s => (s.identifier,s))(Encoders.tuple(Encoders.STRING, pangaeaEncoders))
+      .map(s => (s.identifier, s))(Encoders.tuple(Encoders.STRING, pangaeaEncoders))
-       .groupByKey(_._1)(Encoders.STRING)
+      .groupByKey(_._1)(Encoders.STRING)
       .agg(PangaeaUtils.getDatasetAggregator().toColumn)
       .map(s => s._2)
       .write.mode(SaveMode.Overwrite).save(s"$workingPath/dataset")
@@ -46,7 +46,4 @@ object SparkGeneratePanagaeaDataset {
   }
 
 
-
-
-
 }

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala

@@ -1,6 +1,5 @@
 package eu.dnetlib.dhp.sx.graph.scholix
 
-
 import eu.dnetlib.dhp.schema.oaf.{Publication, Relation, Result, StructuredProperty}
 import eu.dnetlib.dhp.schema.sx.scholix._
 import eu.dnetlib.dhp.schema.sx.summary.{CollectedFromType, SchemeValue, ScholixSummary, Typology}
@@ -10,23 +9,22 @@ import org.apache.spark.sql.{Encoder, Encoders}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
-
 import scala.collection.JavaConverters._
 import scala.io.Source
-import scala.language.postfixOps
 
 object ScholixUtils {
 
 
   val DNET_IDENTIFIER_SCHEMA: String = "DNET Identifier"
 
-  val DATE_RELATION_KEY:String = "RelationDate"
+  val DATE_RELATION_KEY: String = "RelationDate"
-  case class RelationVocabulary(original:String, inverse:String){}
 
-  case class RelatedEntities(id:String, relatedDataset:Long, relatedPublication:Long){}
+  case class RelationVocabulary(original: String, inverse: String) {}
 
-  val relations:Map[String, RelationVocabulary] = {
+  case class RelatedEntities(id: String, relatedDataset: Long, relatedPublication: Long) {}
-    val input =Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/relations.json")).mkString
+
+  val relations: Map[String, RelationVocabulary] = {
+    val input = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/relations.json")).mkString
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
 
     lazy val json: json4s.JValue = parse(input)
@@ -35,12 +33,12 @@ object ScholixUtils {
   }
 
 
-  def extractRelationDate(relation: Relation):String = {
+  def extractRelationDate(relation: Relation): String = {
 
-    if (relation.getProperties== null || !relation.getProperties.isEmpty)
+    if (relation.getProperties == null || !relation.getProperties.isEmpty)
       null
     else {
-      val date =relation.getProperties.asScala.find(p => DATE_RELATION_KEY.equalsIgnoreCase(p.getKey)).map(p => p.getValue)
+      val date = relation.getProperties.asScala.find(p => DATE_RELATION_KEY.equalsIgnoreCase(p.getKey)).map(p => p.getValue)
       if (date.isDefined)
         date.get
       else
@@ -48,9 +46,9 @@ object ScholixUtils {
     }
   }
 
-  def extractRelationDate(summary: ScholixSummary):String = {
+  def extractRelationDate(summary: ScholixSummary): String = {
 
-    if(summary.getDate== null || summary.getDate.isEmpty)
+    if (summary.getDate == null || summary.getDate.isEmpty)
       null
     else {
       summary.getDate.get(0)
@@ -59,15 +57,14 @@ object ScholixUtils {
 
   }
 
-  def inverseRelationShip(rel:ScholixRelationship):ScholixRelationship = {
+  def inverseRelationShip(rel: ScholixRelationship): ScholixRelationship = {
     new ScholixRelationship(rel.getInverse, rel.getSchema, rel.getName)
 
 
   }
 
 
-
+  val statsAggregator: Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] = new Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] with Serializable {
-  val statsAggregator:Aggregator[(String,String, Long), RelatedEntities, RelatedEntities] = new Aggregator[(String,String, Long), RelatedEntities, RelatedEntities] with  Serializable {
     override def zero: RelatedEntities = null
 
     override def reduce(b: RelatedEntities, a: (String, String, Long)): RelatedEntities = {
@@ -78,17 +75,16 @@ object ScholixUtils {
       if (b == null)
         RelatedEntities(a._1, relatedDataset, relatedPublication)
       else
-        RelatedEntities(a._1,b.relatedDataset+ relatedDataset, b.relatedPublication+ relatedPublication )
+        RelatedEntities(a._1, b.relatedDataset + relatedDataset, b.relatedPublication + relatedPublication)
     }
 
     override def merge(b1: RelatedEntities, b2: RelatedEntities): RelatedEntities = {
-      if (b1!= null && b2!= null)
+      if (b1 != null && b2 != null)
-        RelatedEntities(b1.id, b1.relatedDataset+ b2.relatedDataset, b1.relatedPublication+ b2.relatedPublication)
+        RelatedEntities(b1.id, b1.relatedDataset + b2.relatedDataset, b1.relatedPublication + b2.relatedPublication)
 
+      else if (b1 != null)
+        b1
       else
-        if (b1!= null)
-          b1
-        else
         b2
     }
 
@@ -104,12 +100,12 @@ object ScholixUtils {
     override def zero: Scholix = null
 
 
-    def scholix_complete(s:Scholix):Boolean ={
+    def scholix_complete(s: Scholix): Boolean = {
-      if (s== null || s.getIdentifier==null) {
+      if (s == null || s.getIdentifier == null) {
         false
       } else if (s.getSource == null || s.getTarget == null) {
-          false
+        false
-        }
+      }
       else if (s.getLinkprovider == null || s.getLinkprovider.isEmpty)
         false
       else
@@ -121,7 +117,7 @@ object ScholixUtils {
     }
 
     override def merge(b1: Scholix, b2: Scholix): Scholix = {
-      if (scholix_complete(b1)) b1 else  b2
+      if (scholix_complete(b1)) b1 else b2
     }
 
     override def finish(reduction: Scholix): Scholix = reduction
@@ -132,7 +128,7 @@ object ScholixUtils {
   }
 
 
-  def createInverseScholixRelation(scholix: Scholix):Scholix = {
+  def createInverseScholixRelation(scholix: Scholix): Scholix = {
     val s = new Scholix
     s.setPublicationDate(scholix.getPublicationDate)
     s.setPublisher(scholix.getPublisher)
@@ -144,34 +140,33 @@ object ScholixUtils {
     s
 
 
-
   }
 
 
-  def extractCollectedFrom(summary:ScholixSummary): List[ScholixEntityId] = {
+  def extractCollectedFrom(summary: ScholixSummary): List[ScholixEntityId] = {
-    if (summary.getDatasources!= null && !summary.getDatasources.isEmpty) {
+    if (summary.getDatasources != null && !summary.getDatasources.isEmpty) {
-      val l: List[ScholixEntityId] = summary.getDatasources.asScala.map{
+      val l: List[ScholixEntityId] = summary.getDatasources.asScala.map {
         d => new ScholixEntityId(d.getDatasourceName, List(new ScholixIdentifier(d.getDatasourceId, "DNET Identifier", null)).asJava)
       }(collection.breakOut)
-       l
+      l
     } else List()
   }
 
-  def extractCollectedFrom(relation: Relation) : List[ScholixEntityId] = {
+  def extractCollectedFrom(relation: Relation): List[ScholixEntityId] = {
     if (relation.getCollectedfrom != null && !relation.getCollectedfrom.isEmpty) {
 
 
       val l: List[ScholixEntityId] = relation.getCollectedfrom.asScala.map {
         c =>
 
-          new ScholixEntityId(c.getValue, List(new ScholixIdentifier(c.getKey, DNET_IDENTIFIER_SCHEMA,null)).asJava)
+          new ScholixEntityId(c.getValue, List(new ScholixIdentifier(c.getKey, DNET_IDENTIFIER_SCHEMA, null)).asJava)
       }(collection breakOut)
       l
     } else List()
   }
 
 
-  def generateCompleteScholix(scholix: Scholix, target:ScholixSummary): Scholix = {
+  def generateCompleteScholix(scholix: Scholix, target: ScholixSummary): Scholix = {
     val s = new Scholix
     s.setPublicationDate(scholix.getPublicationDate)
     s.setPublisher(scholix.getPublisher)
@@ -192,29 +187,28 @@ object ScholixUtils {
     r.setObjectType(summaryObject.getTypology.toString)
     r.setObjectSubType(summaryObject.getSubType)
 
-    if (summaryObject.getTitle!= null && !summaryObject.getTitle.isEmpty)
+    if (summaryObject.getTitle != null && !summaryObject.getTitle.isEmpty)
-        r.setTitle(summaryObject.getTitle.get(0))
+      r.setTitle(summaryObject.getTitle.get(0))
 
-    if (summaryObject.getAuthor!= null && !summaryObject.getAuthor.isEmpty){
+    if (summaryObject.getAuthor != null && !summaryObject.getAuthor.isEmpty) {
-      val l:List[ScholixEntityId] = summaryObject.getAuthor.asScala.map(a => new ScholixEntityId(a,null)).toList
+      val l: List[ScholixEntityId] = summaryObject.getAuthor.asScala.map(a => new ScholixEntityId(a, null)).toList
       if (l.nonEmpty)
         r.setCreator(l.asJava)
     }
 
-    if (summaryObject.getDate!= null && !summaryObject.getDate.isEmpty)
+    if (summaryObject.getDate != null && !summaryObject.getDate.isEmpty)
       r.setPublicationDate(summaryObject.getDate.get(0))
-    if (summaryObject.getPublisher!= null && !summaryObject.getPublisher.isEmpty)
+    if (summaryObject.getPublisher != null && !summaryObject.getPublisher.isEmpty) {
-    {
+      val plist: List[ScholixEntityId] = summaryObject.getPublisher.asScala.map(p => new ScholixEntityId(p, null)).toList
-      val plist:List[ScholixEntityId] =summaryObject.getPublisher.asScala.map(p => new ScholixEntityId(p, null)).toList
 
       if (plist.nonEmpty)
         r.setPublisher(plist.asJava)
     }
 
 
-    if (summaryObject.getDatasources!= null && !summaryObject.getDatasources.isEmpty) {
+    if (summaryObject.getDatasources != null && !summaryObject.getDatasources.isEmpty) {
 
-      val l:List[ScholixCollectedFrom] = summaryObject.getDatasources.asScala.map(c => new ScholixCollectedFrom(
+      val l: List[ScholixCollectedFrom] = summaryObject.getDatasources.asScala.map(c => new ScholixCollectedFrom(
         new ScholixEntityId(c.getDatasourceName, List(new ScholixIdentifier(c.getDatasourceId, DNET_IDENTIFIER_SCHEMA, null)).asJava)
         , "collected", "complete"
 
@@ -228,12 +222,9 @@ object ScholixUtils {
   }
 
 
+  def scholixFromSource(relation: Relation, source: ScholixSummary): Scholix = {
 
-
+    if (relation == null || source == null)
-
-  def scholixFromSource(relation:Relation, source:ScholixSummary):Scholix = {
-
-    if (relation== null || source== null)
       return null
 
     val s = new Scholix
@@ -253,9 +244,9 @@ object ScholixUtils {
     s.setPublicationDate(d)
 
 
-    if (source.getPublisher!= null && !source.getPublisher.isEmpty) {
+    if (source.getPublisher != null && !source.getPublisher.isEmpty) {
       val l: List[ScholixEntityId] = source.getPublisher.asScala
-        .map{
+        .map {
           p =>
             new ScholixEntityId(p, null)
         }(collection.breakOut)
@@ -265,7 +256,7 @@ object ScholixUtils {
     }
 
     val semanticRelation = relations.getOrElse(relation.getRelClass.toLowerCase, null)
-    if (semanticRelation== null)
+    if (semanticRelation == null)
       return null
     s.setRelationship(new ScholixRelationship(semanticRelation.original, "datacite", semanticRelation.inverse))
     s.setSource(generateScholixResourceFromSummary(source))
@@ -274,8 +265,8 @@ object ScholixUtils {
   }
 
 
-  def findURLForPID(pidValue:List[StructuredProperty], urls:List[String]):List[(StructuredProperty, String)] = {
+  def findURLForPID(pidValue: List[StructuredProperty], urls: List[String]): List[(StructuredProperty, String)] = {
-    pidValue.map{
+    pidValue.map {
       p =>
         val pv = p.getValue
 
@@ -285,67 +276,67 @@ object ScholixUtils {
   }
 
 
-  def extractTypedIdentifierFromInstance(r:Result):List[ScholixIdentifier] = {
+  def extractTypedIdentifierFromInstance(r: Result): List[ScholixIdentifier] = {
     if (r.getInstance() == null || r.getInstance().isEmpty)
       return List()
-    r.getInstance().asScala.filter(i => i.getUrl!= null && !i.getUrl.isEmpty)
+    r.getInstance().asScala.filter(i => i.getUrl != null && !i.getUrl.isEmpty)
-      .filter(i => i.getPid!= null && i.getUrl != null)
+      .filter(i => i.getPid != null && i.getUrl != null)
       .flatMap(i => findURLForPID(i.getPid.asScala.toList, i.getUrl.asScala.toList))
       .map(i => new ScholixIdentifier(i._1.getValue, i._1.getQualifier.getClassid, i._2)).distinct.toList
   }
 
-  def resultToSummary(r:Result):ScholixSummary = {
+  def resultToSummary(r: Result): ScholixSummary = {
     val s = new ScholixSummary
     s.setId(r.getId)
     if (r.getPid == null || r.getPid.isEmpty)
       return null
 
-    val persistentIdentifiers:List[ScholixIdentifier] =  extractTypedIdentifierFromInstance(r)
+    val persistentIdentifiers: List[ScholixIdentifier] = extractTypedIdentifierFromInstance(r)
     if (persistentIdentifiers.isEmpty)
       return null
     s.setLocalIdentifier(persistentIdentifiers.asJava)
-    if (r.isInstanceOf[Publication] )
+    if (r.isInstanceOf[Publication])
       s.setTypology(Typology.publication)
     else
       s.setTypology(Typology.dataset)
 
     s.setSubType(r.getInstance().get(0).getInstancetype.getClassname)
 
-    if (r.getTitle!= null && r.getTitle.asScala.nonEmpty) {
+    if (r.getTitle != null && r.getTitle.asScala.nonEmpty) {
-      val titles:List[String] =r.getTitle.asScala.map(t => t.getValue)(collection breakOut)
+      val titles: List[String] = r.getTitle.asScala.map(t => t.getValue)(collection breakOut)
       if (titles.nonEmpty)
         s.setTitle(titles.asJava)
       else
-        return  null
+        return null
     }
 
-    if(r.getAuthor!= null && !r.getAuthor.isEmpty) {
+    if (r.getAuthor != null && !r.getAuthor.isEmpty) {
-      val authors:List[String] = r.getAuthor.asScala.map(a=> a.getFullname)(collection breakOut)
+      val authors: List[String] = r.getAuthor.asScala.map(a => a.getFullname)(collection breakOut)
       if (authors nonEmpty)
         s.setAuthor(authors.asJava)
     }
     if (r.getInstance() != null) {
-      val dt:List[String] = r.getInstance().asScala.filter(i => i.getDateofacceptance != null).map(i => i.getDateofacceptance.getValue)(collection.breakOut)
+      val dt: List[String] = r.getInstance().asScala.filter(i => i.getDateofacceptance != null).map(i => i.getDateofacceptance.getValue)(collection.breakOut)
       if (dt.nonEmpty)
         s.setDate(dt.distinct.asJava)
     }
-    if (r.getDescription!= null && !r.getDescription.isEmpty) {
+    if (r.getDescription != null && !r.getDescription.isEmpty) {
-      val d = r.getDescription.asScala.find(f => f!= null && f.getValue!=null)
+      val d = r.getDescription.asScala.find(f => f != null && f.getValue != null)
       if (d.isDefined)
         s.setDescription(d.get.getValue)
     }
 
-    if (r.getSubject!= null && !r.getSubject.isEmpty) {
+    if (r.getSubject != null && !r.getSubject.isEmpty) {
-      val subjects:List[SchemeValue] =r.getSubject.asScala.map(s => new SchemeValue(s.getQualifier.getClassname, s.getValue))(collection breakOut)
+      val subjects: List[SchemeValue] = r.getSubject.asScala.map(s => new SchemeValue(s.getQualifier.getClassname, s.getValue))(collection breakOut)
       if (subjects.nonEmpty)
         s.setSubject(subjects.asJava)
     }
 
-    if (r.getPublisher!= null)
+    if (r.getPublisher != null)
       s.setPublisher(List(r.getPublisher.getValue).asJava)
 
-    if (r.getCollectedfrom!= null && !r.getCollectedfrom.isEmpty) {
+    if (r.getCollectedfrom != null && !r.getCollectedfrom.isEmpty) {
-      val cf:List[CollectedFromType] = r.getCollectedfrom.asScala.map(c => new CollectedFromType(c.getValue, c.getKey, "complete"))(collection breakOut)
+      val cf: List[CollectedFromType] = r.getCollectedfrom.asScala.map(c => new CollectedFromType(c.getValue, c.getKey, "complete"))(collection breakOut)
       if (cf.nonEmpty)
         s.setDatasources(cf.distinct.asJava)
     }

dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/TestPrepare.scala

@@ -3,13 +3,9 @@ package eu.dnetlib.dhp.oa.graph.hostedbymap
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.oa.graph.hostedbymap.SparkPrepareHostedByInfoToApply.{joinResHBM, prepareResultInfo, toEntityInfo}
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
-import eu.dnetlib.dhp.schema.oaf.{Datasource, OpenAccessRoute, Publication}
-import javax.management.openmbean.OpenMBeanAttributeInfo
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
-import org.json4s
 import org.json4s.DefaultFormats
-import eu.dnetlib.dhp.schema.common.ModelConstants
 import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue}
 import org.junit.jupiter.api.Test
 

dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/TestPreprocess.scala

@@ -4,10 +4,9 @@ import eu.dnetlib.dhp.schema.oaf.Datasource
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.json4s.DefaultFormats
-import org.junit.jupiter.api.Assertions.{assertNotNull, assertTrue}
-import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.Assertions._
 import org.json4s.jackson.Serialization.write
+import org.junit.jupiter.api.Assertions._
+import org.junit.jupiter.api.Test
 
 class TestPreprocess extends java.io.Serializable{