Merge pull request 'scala_refactor' (#169) from scala_refactor into beta

Reviewed-on: D-Net/dnet-hadoop#169

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/scholix/SparkCreateActionset.scala

@@ -1,69 +0,0 @@
-package eu.dnetlib.dhp.actionmanager.scholix
-
-import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Oaf, Relation, Result}
-import org.apache.spark.SparkConf
-import org.apache.spark.sql._
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.io.Source
-
-object SparkCreateActionset {
-
-  def main(args: Array[String]): Unit = {
-    val log: Logger = LoggerFactory.getLogger(getClass)
-    val conf: SparkConf = new SparkConf()
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/actionset/generate_actionset.json")).mkString)
-    parser.parseArgument(args)
-
-
-    val spark: SparkSession =
-      SparkSession
-        .builder()
-        .config(conf)
-        .appName(getClass.getSimpleName)
-        .master(parser.get("master")).getOrCreate()
-
-
-    val sourcePath = parser.get("sourcePath")
-    log.info(s"sourcePath  -> $sourcePath")
-
-    val targetPath = parser.get("targetPath")
-    log.info(s"targetPath  -> $targetPath")
-
-    val workingDirFolder = parser.get("workingDirFolder")
-    log.info(s"workingDirFolder  -> $workingDirFolder")
-
-    implicit val oafEncoders: Encoder[Oaf] = Encoders.kryo[Oaf]
-    implicit val resultEncoders: Encoder[Result] = Encoders.kryo[Result]
-    implicit val relationEncoders: Encoder[Relation] = Encoders.kryo[Relation]
-
-    import spark.implicits._
-
-    val relation = spark.read.load(s"$sourcePath/relation").as[Relation]
-
-    relation.filter(r => (r.getDataInfo == null || r.getDataInfo.getDeletedbyinference == false) && !r.getRelClass.toLowerCase.contains("merge"))
-      .flatMap(r => List(r.getSource, r.getTarget)).distinct().write.mode(SaveMode.Overwrite).save(s"$workingDirFolder/id_relation")
-
-
-    val idRelation = spark.read.load(s"$workingDirFolder/id_relation").as[String]
-
-    log.info("extract source and target Identifier involved in relations")
-
-
-    log.info("save relation filtered")
-
-    relation.filter(r => (r.getDataInfo == null || r.getDataInfo.getDeletedbyinference == false) && !r.getRelClass.toLowerCase.contains("merge"))
-      .write.mode(SaveMode.Overwrite).save(s"$workingDirFolder/actionSetOaf")
-
-    log.info("saving entities")
-
-    val entities: Dataset[(String, Result)] = spark.read.load(s"$sourcePath/entities/*").as[Result].map(p => (p.getId, p))(Encoders.tuple(Encoders.STRING, resultEncoders))
-
-    entities
-      .joinWith(idRelation, entities("_1").equalTo(idRelation("value")))
-      .map(p => p._1._2)
-      .write.mode(SaveMode.Append).save(s"$workingDirFolder/actionSetOaf")
-  }
-
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/scholix/SparkSaveActionSet.scala

@@ -1,86 +0,0 @@
-package eu.dnetlib.dhp.actionmanager.scholix
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.action.AtomicAction
-import eu.dnetlib.dhp.schema.oaf.{Oaf, Dataset => OafDataset,Publication, Software, OtherResearchProduct, Relation}
-import org.apache.hadoop.io.Text
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.hadoop.mapred.SequenceFileOutputFormat
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Encoder, Encoders, SparkSession}
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.io.Source
-
-object SparkSaveActionSet {
-
-
-  def toActionSet(item: Oaf): (String, String) = {
-    val mapper = new ObjectMapper()
-
-    item match {
-      case dataset: OafDataset =>
-        val a: AtomicAction[OafDataset] = new AtomicAction[OafDataset]
-        a.setClazz(classOf[OafDataset])
-        a.setPayload(dataset)
-        (dataset.getClass.getCanonicalName, mapper.writeValueAsString(a))
-      case publication: Publication =>
-        val a: AtomicAction[Publication] = new AtomicAction[Publication]
-        a.setClazz(classOf[Publication])
-        a.setPayload(publication)
-        (publication.getClass.getCanonicalName, mapper.writeValueAsString(a))
-      case software: Software =>
-        val a: AtomicAction[Software] = new AtomicAction[Software]
-        a.setClazz(classOf[Software])
-        a.setPayload(software)
-        (software.getClass.getCanonicalName, mapper.writeValueAsString(a))
-      case orp: OtherResearchProduct =>
-        val a: AtomicAction[OtherResearchProduct] = new AtomicAction[OtherResearchProduct]
-        a.setClazz(classOf[OtherResearchProduct])
-        a.setPayload(orp)
-        (orp.getClass.getCanonicalName, mapper.writeValueAsString(a))
-
-      case relation: Relation =>
-        val a: AtomicAction[Relation] = new AtomicAction[Relation]
-        a.setClazz(classOf[Relation])
-        a.setPayload(relation)
-        (relation.getClass.getCanonicalName, mapper.writeValueAsString(a))
-      case _ =>
-        null
-    }
-
-  }
-
-  def main(args: Array[String]): Unit = {
-    val log: Logger = LoggerFactory.getLogger(getClass)
-    val conf: SparkConf = new SparkConf()
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/actionset/save_actionset.json")).mkString)
-    parser.parseArgument(args)
-
-
-    val spark: SparkSession =
-      SparkSession
-        .builder()
-        .config(conf)
-        .appName(getClass.getSimpleName)
-        .master(parser.get("master")).getOrCreate()
-
-
-    val sourcePath = parser.get("sourcePath")
-    log.info(s"sourcePath  -> $sourcePath")
-
-    val targetPath = parser.get("targetPath")
-    log.info(s"targetPath  -> $targetPath")
-
-    implicit val oafEncoders: Encoder[Oaf] = Encoders.kryo[Oaf]
-    implicit val tEncoder: Encoder[(String, String)] = Encoders.tuple(Encoders.STRING, Encoders.STRING)
-
-    spark.read.load(sourcePath).as[Oaf]
-      .map(o => toActionSet(o))
-      .filter(o => o != null)
-      .rdd.map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$targetPath", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text, Text]], classOf[GzipCodec])
-
-  }
-
-}

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteModelConstants.scala

@@ -1,8 +1,8 @@
 package eu.dnetlib.dhp.datacite
 
 import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.{DataInfo, KeyValue}
 import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils
+import eu.dnetlib.dhp.schema.oaf.{DataInfo, KeyValue}
 
 import java.io.InputStream
 import java.time.format.DateTimeFormatter

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala

@@ -6,7 +6,7 @@ import eu.dnetlib.dhp.datacite.DataciteModelConstants._
 import eu.dnetlib.dhp.schema.action.AtomicAction
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, OafMapperUtils}
-import eu.dnetlib.dhp.schema.oaf.{AccessRight, Author, DataInfo, Instance, KeyValue, Oaf, OtherResearchProduct, Publication, Qualifier, Relation, Result, Software, StructuredProperty, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{Dataset => OafDataset, _}
 import eu.dnetlib.dhp.utils.DHPUtils
 import org.apache.commons.lang3.StringUtils
 import org.json4s.DefaultFormats
@@ -29,6 +29,7 @@ object DataciteToOAFTransformation {
   /**
    * This method should skip record if json contains invalid text
    * defined in gile datacite_filter
+   *
    * @param json
    * @return True if the record should be skipped
    */

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala

@@ -7,6 +7,7 @@ import org.json4s.DefaultFormats
 import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.{compact, parse, render}
 import collection.JavaConverters._
+
 object BioDBToOAF {
 
   case class EBILinkItem(id: Long, links: String) {}

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/SparkTransformBioDatabaseToOAF.scala

@@ -1,9 +1,9 @@
 package eu.dnetlib.dhp.sx.bio
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.Oaf
-import BioDBToOAF.ScholixResolved
 import eu.dnetlib.dhp.collection.CollectionUtils
+import eu.dnetlib.dhp.schema.oaf.Oaf
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF.ScholixResolved
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala

@@ -3,7 +3,7 @@ package eu.dnetlib.dhp.sx.bio.ebi
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
 import eu.dnetlib.dhp.schema.oaf.Result
-import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal, PMParser, PubMedToOaf}
+import eu.dnetlib.dhp.sx.bio.pubmed._
 import eu.dnetlib.dhp.utils.ISLookupClientFactory
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.conf.Configuration

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkDownloadEBILinks.scala

@@ -1,9 +1,8 @@
 package eu.dnetlib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
 import eu.dnetlib.dhp.sx.bio.BioDBToOAF.EBILinkItem
-import eu.dnetlib.dhp.sx.bio.pubmed.PMJournal
+import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
 import org.apache.commons.io.IOUtils
 import org.apache.http.client.config.RequestConfig
 import org.apache.http.client.methods.HttpGet

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkEBILinksToOaf.scala

@@ -1,11 +1,10 @@
 package eu.dnetlib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.collection.CollectionUtils
 import eu.dnetlib.dhp.schema.oaf.Oaf
 import eu.dnetlib.dhp.sx.bio.BioDBToOAF
 import eu.dnetlib.dhp.sx.bio.BioDBToOAF.EBILinkItem
-import BioDBToOAF.EBILinkItem
-import eu.dnetlib.dhp.collection.CollectionUtils
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql._

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala

@@ -4,7 +4,7 @@ import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.{GraphCleaningFunctions, IdentifierFactory, OafMapperUtils, PidType}
 import eu.dnetlib.dhp.schema.oaf._
-import scala.collection.JavaConverters._
+import collection.JavaConverters._
 
 import java.util.regex.Pattern
 
@@ -22,10 +22,10 @@ object PubMedToOaf {
   val collectedFrom: KeyValue = OafMapperUtils.keyValue(ModelConstants.EUROPE_PUBMED_CENTRAL_ID, "Europe PubMed Central")
 
 
-
   /**
    * Cleaning the DOI Applying regex in order to
    * remove doi starting with URL
+   *
    * @param doi input DOI
    * @return cleaned DOI
    */
@@ -93,7 +93,6 @@ object PubMedToOaf {
    * @param vocabularyName the input vocabulary name
    * @param vocabularies   all the vocabularies
    * @param term           the term to find
-   *
    * @return the cleaned term value
    */
   def getVocabularyTerm(vocabularyName: String, vocabularies: VocabularyGroup, term: String): Qualifier = {
@@ -106,7 +105,6 @@ object PubMedToOaf {
   /**
    * Map the Pubmed Article into the OAF instance
    *
-   *
    * @param article      the pubmed articles
    * @param vocabularies the vocabularies
    * @return The OAF instance if the mapping did not fail
@@ -185,7 +183,6 @@ object PubMedToOaf {
     //--------------------------------------------------------------------------------------
 
 
-
     // RESULT MAPPING
     //--------------------------------------------------------------------------------------
     result.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(article.getDate), dataInfo))

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateOpenCitationsASTest.java

@@ -89,13 +89,13 @@ public class CreateOpenCitationsASTest {
 					"-inputPath",
 					inputPath,
 					"-outputPath",
-					workingDir.toString() + "/actionSet"
+					workingDir.toString() + "/actionSet1"
 				});
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
+			.sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class)
 			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
 			.map(aa -> ((Relation) aa.getPayload()));
 
@@ -121,13 +121,13 @@ public class CreateOpenCitationsASTest {
 					"-inputPath",
 					inputPath,
 					"-outputPath",
-					workingDir.toString() + "/actionSet"
+					workingDir.toString() + "/actionSet2"
 				});
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
+			.sequenceFile(workingDir.toString() + "/actionSet2", Text.class, Text.class)
 			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
 			.map(aa -> ((Relation) aa.getPayload()));
 
@@ -153,13 +153,13 @@ public class CreateOpenCitationsASTest {
 					"-inputPath",
 					inputPath,
 					"-outputPath",
-					workingDir.toString() + "/actionSet"
+					workingDir.toString() + "/actionSet3"
 				});
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
+			.sequenceFile(workingDir.toString() + "/actionSet3", Text.class, Text.class)
 			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
 			.map(aa -> ((Relation) aa.getPayload()));
 
@@ -186,13 +186,13 @@ public class CreateOpenCitationsASTest {
 					"-inputPath",
 					inputPath,
 					"-outputPath",
-					workingDir.toString() + "/actionSet"
+					workingDir.toString() + "/actionSet4"
 				});
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
+			.sequenceFile(workingDir.toString() + "/actionSet4", Text.class, Text.class)
 			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
 			.map(aa -> ((Relation) aa.getPayload()));
 
@@ -226,13 +226,13 @@ public class CreateOpenCitationsASTest {
 					"-inputPath",
 					inputPath,
 					"-outputPath",
-					workingDir.toString() + "/actionSet"
+					workingDir.toString() + "/actionSet5"
 				});
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
+			.sequenceFile(workingDir.toString() + "/actionSet5", Text.class, Text.class)
 			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
 			.map(aa -> ((Relation) aa.getPayload()));
 
@@ -261,13 +261,13 @@ public class CreateOpenCitationsASTest {
 					"-inputPath",
 					inputPath,
 					"-outputPath",
-					workingDir.toString() + "/actionSet"
+					workingDir.toString() + "/actionSet6"
 				});
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
+			.sequenceFile(workingDir.toString() + "/actionSet6", Text.class, Text.class)
 			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
 			.map(aa -> ((Relation) aa.getPayload()));
 
@@ -306,13 +306,13 @@ public class CreateOpenCitationsASTest {
 					"-inputPath",
 					inputPath,
 					"-outputPath",
-					workingDir.toString() + "/actionSet"
+					workingDir.toString() + "/actionSet7"
 				});
 
 		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
 
 		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet", Text.class, Text.class)
+			.sequenceFile(workingDir.toString() + "/actionSet7", Text.class, Text.class)
 			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
 			.map(aa -> ((Relation) aa.getPayload()));
 

dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTest.scala

@@ -8,6 +8,7 @@ import org.apache.commons.io.FileUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.functions.{col, count}
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
+import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.extension.ExtendWith
 import org.junit.jupiter.api.{AfterEach, BeforeEach, Test}
 import org.mockito.junit.jupiter.MockitoExtension
@@ -17,7 +18,6 @@ import java.nio.file.{Files, Path}
 import java.text.SimpleDateFormat
 import java.util.Locale
 import scala.io.Source
-import org.junit.jupiter.api.Assertions._
 
 @ExtendWith(Array(classOf[MockitoExtension]))
 class DataciteToOAFTest extends  AbstractVocabularyTest{

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala

@@ -1,21 +1,19 @@
 package eu.dnetlib.doiboost
 
-import java.time.LocalDate
+import com.fasterxml.jackson.databind.ObjectMapper
-import java.time.format.DateTimeFormatter
-
 import eu.dnetlib.dhp.schema.action.AtomicAction
-import eu.dnetlib.dhp.schema.oaf.{AccessRight, DataInfo, Dataset, Field, Instance, KeyValue, Oaf, OpenAccessRoute, Organization, Publication, Qualifier, Relation, Result, StructuredProperty}
+import eu.dnetlib.dhp.schema.common.ModelConstants
+import eu.dnetlib.dhp.schema.oaf._
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils
 import eu.dnetlib.dhp.utils.DHPUtils
 import org.apache.commons.lang3.StringUtils
-import com.fasterxml.jackson.databind.ObjectMapper
-import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils
-import eu.dnetlib.doiboost.DoiBoostMappingUtil.{getClosedAccessQualifier, getEmbargoedAccessQualifier, getUnknownQualifier}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
 import org.slf4j.{Logger, LoggerFactory}
 
+import java.time.LocalDate
+import java.time.format.DateTimeFormatter
 import scala.collection.JavaConverters._
 
 

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDOIBoostActionSet.scala

@@ -8,11 +8,12 @@ import org.apache.hadoop.io.Text
 import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.hadoop.mapred.SequenceFileOutputFormat
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkGenerateDOIBoostActionSet {
   val logger: Logger = LoggerFactory.getLogger(getClass)
+
   def main(args: Array[String]): Unit = {
 
     val conf: SparkConf = new SparkConf()
@@ -55,7 +56,6 @@ object SparkGenerateDOIBoostActionSet {
       .map(d => DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING))
 
 
-
     val asCRelation = spark.read.load(crossRefRelation).as[Relation]
       .filter(r => r != null && r.getSource != null && r.getTarget != null)
       .map(d => DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING))
@@ -65,23 +65,12 @@ object SparkGenerateDOIBoostActionSet {
       .map(d => DoiBoostMappingUtil.toActionSet(d))(Encoders.tuple(Encoders.STRING, Encoders.STRING))
 
 
-
-
-
     val d: Dataset[(String, String)] = asDataset.union(asPublication).union(asOrganization).union(asCRelation).union(asRelAffiliation)
 
 
-
     d.rdd.repartition(6000).map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$sequenceFilePath", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text, Text]], classOf[GzipCodec])
 
 
-
-
-
-
-
-
-
   }
 
 }

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala

@@ -9,14 +9,12 @@ import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.expressions.Aggregator
 import org.apache.spark.sql.functions.col
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.collection.JavaConverters._
 import org.json4s.DefaultFormats
-import org.json4s.JsonAST.{JField, JObject, JString,JArray}
+import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.parse
-
+import org.slf4j.{Logger, LoggerFactory}
+import scala.collection.JavaConverters._
 object SparkGenerateDoiBoost {
 
 
@@ -38,7 +36,6 @@ object SparkGenerateDoiBoost {
   }
 
 
-
   def main(args: Array[String]): Unit = {
 
     val logger: Logger = LoggerFactory.getLogger(getClass)
@@ -118,8 +115,7 @@ object SparkGenerateDoiBoost {
     val crossrefPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/crossrefPublication").as[Publication].map(p => (p.getId, p))
     val uwPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/uwPublication").as[Publication].map(p => (p.getId, p))
 
-    def applyMerge(item:((String, Publication), (String, Publication))) : Publication =
+    def applyMerge(item: ((String, Publication), (String, Publication))): Publication = {
-    {
       val crossrefPub = item._1._2
       if (item._2 != null) {
         val otherPub = item._2._2
@@ -130,6 +126,7 @@ object SparkGenerateDoiBoost {
       }
       crossrefPub
     }
+
     crossrefPublication.joinWith(uwPublication, crossrefPublication("_1").equalTo(uwPublication("_1")), "left").map(applyMerge).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/firstJoin")
     logger.info("Phase 3) Join Result with ORCID")
     val fj: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/firstJoin").as[Publication].map(p => (p.getId, p))
@@ -169,7 +166,6 @@ object SparkGenerateDoiBoost {
       .select(col("_1.PaperId"), col("_2.AffiliationId"), col("_2.GridId"), col("_2.OfficialPage"), col("_2.DisplayName")).as[DoiBoostAffiliation]
 
 
-
     val magPubs: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/doiBoostPublicationFiltered").as[Publication]
       .map(p => (ConversionUtil.extractMagIdentifier(p.getOriginalId.asScala), p))(tupleForJoinEncoder).filter(s => s._1 != null)
 
@@ -198,8 +194,6 @@ object SparkGenerateDoiBoost {
     })(mapEncoderRel).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationAffiliation_unresolved")
 
 
-
-
     val unresolvedRels: Dataset[(String, Relation)] = spark.read.load(s"$workingDirPath/doiBoostPublicationAffiliation_unresolved").as[Relation].map(r => {
 
       if (r.getSource.startsWith("unresolved"))

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala

@@ -4,20 +4,19 @@ import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf._
 import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, OafMapperUtils}
 import eu.dnetlib.dhp.utils.DHPUtils
-import eu.dnetlib.doiboost.DoiBoostMappingUtil.{decideAccessRight, _}
+import eu.dnetlib.doiboost.DoiBoostMappingUtil
+import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 import org.apache.commons.lang.StringUtils
 import org.json4s
 import org.json4s.DefaultFormats
-import org.json4s.JsonAST.{JValue, _}
+import org.json4s.JsonAST._
 import org.json4s.jackson.JsonMethods._
 import org.slf4j.{Logger, LoggerFactory}
 
+import java.util
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.util.matching.Regex
-import java.util
-
-import eu.dnetlib.doiboost.DoiBoostMappingUtil
 
 case class CrossrefDT(doi: String, json:String, timestamp: Long) {}
 

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/CrossrefDataset.scala

@@ -6,7 +6,7 @@ import org.apache.commons.io.IOUtils
 import org.apache.hadoop.io.{IntWritable, Text}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.expressions.Aggregator
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql.{Dataset, Encoder, SaveMode, SparkSession}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
@@ -30,7 +30,6 @@ object CrossrefDataset {
   def main(args: Array[String]): Unit = {
 
 
-
     val conf: SparkConf = new SparkConf()
     val parser = new ArgumentApplicationParser(IOUtils.toString(CrossrefDataset.getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref_to_dataset_params.json")))
     parser.parseArgument(args)

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/GenerateCrossrefDataset.scala

@@ -2,17 +2,12 @@ package eu.dnetlib.doiboost.crossref
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
-import eu.dnetlib.doiboost.crossref.CrossrefDataset.to_item
-import eu.dnetlib.doiboost.crossref.UnpackCrtossrefEntries.getClass
-import org.apache.hadoop.io.{IntWritable, Text}
-import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.rdd.RDD
-import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.{SparkConf, SparkContext}
 import org.json4s
 import org.json4s.DefaultFormats
-import org.json4s.JsonAST.JArray
+import org.json4s.jackson.JsonMethods.parse
-import org.json4s.jackson.JsonMethods.{compact, parse, render}
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.io.Source
@@ -24,7 +19,6 @@ object GenerateCrossrefDataset {
   implicit val mrEncoder: Encoder[CrossrefDT] = Encoders.kryo[CrossrefDT]
 
 
-
   def crossrefElement(meta: String): CrossrefDT = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
     lazy val json: json4s.JValue = parse(meta)

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/SparkMapDumpIntoOAF.scala

@@ -4,10 +4,8 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.oaf
 import eu.dnetlib.dhp.schema.oaf.{Oaf, Publication, Relation, Dataset => OafDataset}
 import org.apache.commons.io.IOUtils
-
 import org.apache.spark.SparkConf
-
+import org.apache.spark.sql._
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
 
 

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/UnpackCrtossrefEntries.scala

@@ -2,8 +2,8 @@ package eu.dnetlib.doiboost.crossref
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.hadoop.io.compress.GzipCodec
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.JArray
@@ -17,8 +17,6 @@ object UnpackCrtossrefEntries {
   val log: Logger = LoggerFactory.getLogger(UnpackCrtossrefEntries.getClass)
 
 
-
-
   def extractDump(input: String): List[String] = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
     lazy val json: json4s.JValue = parse(input)
@@ -30,7 +28,6 @@ object UnpackCrtossrefEntries {
   }
 
 
-
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf
     val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref_dump_reader/generate_dataset_params.json")).mkString)

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/mag/MagDataModel.scala

@@ -5,10 +5,10 @@ import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
 import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, StructuredProperty}
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
+import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
-import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/mag/SparkImportMagIntoDataset.scala

@@ -3,8 +3,8 @@ package eu.dnetlib.doiboost.mag
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkImportMagIntoDataset {
@@ -75,7 +75,6 @@ object SparkImportMagIntoDataset {
         .master(parser.get("master")).getOrCreate()
 
 
-
     stream.foreach { case (k, v) =>
       val s: StructType = getSchema(k)
       val df = spark.read

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/mag/SparkProcessMAG.scala

@@ -5,13 +5,10 @@ import eu.dnetlib.dhp.schema.oaf.Publication
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.functions.{col, collect_list, struct}
-import org.apache.spark.sql.functions._
 import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
-
 import scala.collection.JavaConverters._
-
 object SparkProcessMAG {
 
   def getDistinctResults(d: Dataset[MagPapers]): Dataset[MagPapers] = {
@@ -20,7 +17,7 @@ object SparkProcessMAG {
       .reduceGroups((p1: MagPapers, p2: MagPapers) => ConversionUtil.choiceLatestMagArtitcle(p1, p2))
       .map(_._2)(Encoders.product[MagPapers])
       .map(mp => {
-        new MagPapers(mp.PaperId, mp.Rank, DoiBoostMappingUtil.normalizeDoi(mp.Doi),
+        MagPapers(mp.PaperId, mp.Rank, DoiBoostMappingUtil.normalizeDoi(mp.Doi),
           mp.DocType, mp.PaperTitle, mp.OriginalTitle,
           mp.BookTitle, mp.Year, mp.Date, mp.Publisher: String,
           mp.JournalId, mp.ConferenceSeriesId, mp.ConferenceInstanceId,
@@ -153,6 +150,5 @@ object SparkProcessMAG {
       .write.mode(SaveMode.Overwrite).save(s"$targetPath/magPublication")
 
 
-
   }
 }

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala

@@ -4,17 +4,16 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
 import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Publication}
-import eu.dnetlib.dhp.schema.orcid.{AuthorData, OrcidDOI}
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
 import eu.dnetlib.doiboost.DoiBoostMappingUtil.{createSP, generateDataInfo}
 import org.apache.commons.lang.StringUtils
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.collection.JavaConverters._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST._
 import org.json4s.jackson.JsonMethods._
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.collection.JavaConverters._
 
 
 case class ORCIDItem(doi:String, authors:List[OrcidAuthor]){}

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/orcid/SparkPreprocessORCID.scala

@@ -1,15 +1,12 @@
 package eu.dnetlib.doiboost.orcid
 
-import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.oa.merge.AuthorMerger
 import eu.dnetlib.dhp.schema.oaf.Publication
-import eu.dnetlib.dhp.schema.orcid.OrcidDOI
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.functions._
+import org.apache.spark.sql.functions.{col, collect_list}
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkPreprocessORCID {

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/uw/SparkMapUnpayWallToOAF.scala

@@ -1,16 +1,14 @@
 package eu.dnetlib.doiboost.uw
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-
 import eu.dnetlib.dhp.schema.oaf.Publication
 import eu.dnetlib.doiboost.crossref.SparkMapDumpIntoOAF
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
-
 object SparkMapUnpayWallToOAF {
 
   def main(args: Array[String]): Unit = {

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala

@@ -4,14 +4,13 @@ import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
 import eu.dnetlib.dhp.schema.oaf.{AccessRight, Instance, OpenAccessRoute, Publication}
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
+import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.JavaConverters._
-import eu.dnetlib.doiboost.DoiBoostMappingUtil._
-import eu.dnetlib.doiboost.uw.UnpayWallToOAF.get_unpaywall_color
 
 
 

dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/dhp/doiboost/DoiBoostHostedByMapTest.scala

@@ -1,70 +0,0 @@
-package eu.dnetlib.dhp.doiboost
-
-import eu.dnetlib.dhp.schema.oaf.{Publication, Dataset => OafDataset}
-import eu.dnetlib.doiboost.{DoiBoostMappingUtil, HostedByItemType}
-import eu.dnetlib.doiboost.SparkGenerateDoiBoost.getClass
-import eu.dnetlib.doiboost.mag.ConversionUtil
-import eu.dnetlib.doiboost.orcid.ORCIDElement
-import org.apache.spark.SparkConf
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
-import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig}
-import org.junit.jupiter.api.Test
-
-import scala.io.Source
-
-class DoiBoostHostedByMapTest {
-
-
-
-//  @Test
-//  def testMerge():Unit = {
-//    val conf: SparkConf = new SparkConf()
-//    val spark: SparkSession =
-//      SparkSession
-//        .builder()
-//        .config(conf)
-//        .appName(getClass.getSimpleName)
-//        .master("local[*]").getOrCreate()
-//
-//
-//
-//    implicit val mapEncoderPub: Encoder[Publication] = Encoders.kryo[Publication]
-//    implicit val mapEncoderDataset: Encoder[OafDataset] = Encoders.kryo[OafDataset]
-//    implicit val tupleForJoinEncoder: Encoder[(String, Publication)] = Encoders.tuple(Encoders.STRING, mapEncoderPub)
-//
-//
-//    import spark.implicits._
-//    val dataset:RDD[String]= spark.sparkContext.textFile("/home/sandro/Downloads/hbMap.gz")
-//
-//
-//    val hbMap:Dataset[(String, HostedByItemType)] =spark.createDataset(dataset.map(DoiBoostMappingUtil.toHostedByItem))
-//
-//
-//    hbMap.show()
-//
-//
-//
-//
-//
-//
-//
-//
-//
-//
-//  }
-
-
-  @Test
-  def idDSGeneration():Unit = {
-    val s ="doajarticles::0066-782X"
-
-
-
-    println(DoiBoostMappingUtil.generateDSId(s))
-
-
-  }
-
-
-}

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/DoiBoostHostedByMapTest.scala

@@ -0,0 +1,20 @@
+package eu.dnetlib.dhp.doiboost
+
+import eu.dnetlib.doiboost.DoiBoostMappingUtil
+import org.junit.jupiter.api.Test
+
+class DoiBoostHostedByMapTest {
+
+  @Test
+  def idDSGeneration():Unit = {
+    val s ="doajarticles::0066-782X"
+
+
+
+    println(DoiBoostMappingUtil.generateDSId(s))
+
+
+  }
+
+
+}

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala

@@ -1,7 +1,8 @@
-package eu.dnetlib.doiboost.crossref
+package eu.dnetlib.dhp.doiboost.crossref
 
 import eu.dnetlib.dhp.schema.oaf._
 import eu.dnetlib.dhp.utils.DHPUtils
+import eu.dnetlib.doiboost.crossref.Crossref2Oaf
 import org.codehaus.jackson.map.{ObjectMapper, SerializationConfig}
 import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.Test
@@ -21,9 +22,9 @@ class CrossrefMappingTest {
 
   @Test
   def testFunderRelationshipsMapping(): Unit = {
-    val template = Source.fromInputStream(getClass.getResourceAsStream("article_funder_template.json")).mkString
+    val template = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/article_funder_template.json")).mkString
-    val funder_doi = Source.fromInputStream(getClass.getResourceAsStream("funder_doi")).mkString
+    val funder_doi = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/funder_doi")).mkString
-    val funder_name = Source.fromInputStream(getClass.getResourceAsStream("funder_doi")).mkString
+    val funder_name = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/funder_doi")).mkString
 
 
     for (line <- funder_doi.lines) {
@@ -72,7 +73,7 @@ class CrossrefMappingTest {
 
   @Test
   def testOrcidID() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("orcid_data.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/orcid_data.json")).mkString
 
 
     assertNotNull(json)
@@ -93,7 +94,7 @@ class CrossrefMappingTest {
 
   @Test
   def testEmptyTitle() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("empty_title.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/empty_title.json")).mkString
 
 
     assertNotNull(json)
@@ -115,7 +116,7 @@ class CrossrefMappingTest {
 
   @Test
   def testPeerReviewed(): Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("prwTest.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/prwTest.json")).mkString
     mapper.getSerializationConfig.enable(SerializationConfig.Feature.INDENT_OUTPUT)
 
     assertNotNull(json)
@@ -156,7 +157,7 @@ class CrossrefMappingTest {
 
   @Test
   def testJournalRelation(): Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("awardTest.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/awardTest.json")).mkString
     assertNotNull(json)
 
     assertFalse(json.isEmpty)
@@ -177,7 +178,7 @@ class CrossrefMappingTest {
 
   @Test
   def testConvertBookFromCrossRef2Oaf(): Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("book.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/book.json")).mkString
     assertNotNull(json)
 
     assertFalse(json.isEmpty);
@@ -233,7 +234,7 @@ class CrossrefMappingTest {
 
   @Test
   def testConvertPreprintFromCrossRef2Oaf(): Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("preprint.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/preprint.json")).mkString
     assertNotNull(json)
 
     assertFalse(json.isEmpty);
@@ -291,7 +292,7 @@ class CrossrefMappingTest {
 
   @Test
   def testConvertDatasetFromCrossRef2Oaf(): Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("dataset.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/dataset.json")).mkString
     assertNotNull(json)
 
     assertFalse(json.isEmpty);
@@ -332,7 +333,7 @@ class CrossrefMappingTest {
 
   @Test
   def testConvertArticleFromCrossRef2Oaf(): Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("article.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/article.json")).mkString
     assertNotNull(json)
 
     assertFalse(json.isEmpty);
@@ -400,7 +401,7 @@ class CrossrefMappingTest {
   @Test
   def testSetDateOfAcceptanceCrossRef2Oaf(): Unit = {
 
-    val json = Source.fromInputStream(getClass.getResourceAsStream("dump_file.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/dump_file.json")).mkString
     assertNotNull(json)
 
     assertFalse(json.isEmpty);
@@ -415,55 +416,12 @@ class CrossrefMappingTest {
     assert(items.size == 1)
     val result: Result = items.head.asInstanceOf[Publication]
     assertNotNull(result)
-
     logger.info(mapper.writeValueAsString(result));
-
-//    assertNotNull(result.getDataInfo, "Datainfo test not null Failed");
-//    assertNotNull(
-//      result.getDataInfo.getProvenanceaction,
-//      "DataInfo/Provenance test not null Failed");
-//    assertFalse(
-//      result.getDataInfo.getProvenanceaction.getClassid.isEmpty,
-//      "DataInfo/Provenance/classId test not null Failed");
-//    assertFalse(
-//      result.getDataInfo.getProvenanceaction.getClassname.isEmpty,
-//      "DataInfo/Provenance/className test not null Failed");
-//    assertFalse(
-//      result.getDataInfo.getProvenanceaction.getSchemeid.isEmpty,
-//      "DataInfo/Provenance/SchemeId test not null Failed");
-//    assertFalse(
-//      result.getDataInfo.getProvenanceaction.getSchemename.isEmpty,
-//      "DataInfo/Provenance/SchemeName test not null Failed");
-//
-//    assertNotNull(result.getCollectedfrom, "CollectedFrom test not null Failed");
-//    assertFalse(result.getCollectedfrom.isEmpty);
-//
-//    val collectedFromList = result.getCollectedfrom.asScala
-//    assert(collectedFromList.exists(c => c.getKey.equalsIgnoreCase("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2")), "Wrong collected from assertion")
-//
-//    assert(collectedFromList.exists(c => c.getValue.equalsIgnoreCase("crossref")), "Wrong collected from assertion")
-//
-//
-//    val relevantDates = result.getRelevantdate.asScala
-//
-//    assert(relevantDates.exists(d => d.getQualifier.getClassid.equalsIgnoreCase("created")), "Missing relevant date of type created")
-//
-//    val rels = resultList.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]]
-//    assertFalse(rels.isEmpty)
-//    rels.foreach(relation => {
-//      assertNotNull(relation)
-//      assertFalse(relation.getSource.isEmpty)
-//      assertFalse(relation.getTarget.isEmpty)
-//      assertFalse(relation.getRelClass.isEmpty)
-//      assertFalse(relation.getRelType.isEmpty)
-//      assertFalse(relation.getSubRelType.isEmpty)
-//
-//    })
   }
 
   @Test
   def testNormalizeDOI(): Unit = {
-    val template = Source.fromInputStream(getClass.getResourceAsStream("article_funder_template.json")).mkString
+    val template = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/article_funder_template.json")).mkString
     val line :String = "\"funder\": [{\"name\": \"Wellcome Trust Masters Fellowship\",\"award\": [\"090633\"]}],"
     val json = template.replace("%s", line)
     val resultList: List[Oaf] = Crossref2Oaf.convert(json)
@@ -479,7 +437,7 @@ class CrossrefMappingTest {
 
   @Test
   def testNormalizeDOI2(): Unit = {
-    val template = Source.fromInputStream(getClass.getResourceAsStream("article.json")).mkString
+    val template = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/article.json")).mkString
 
     val resultList: List[Oaf] = Crossref2Oaf.convert(template)
     assertTrue(resultList.nonEmpty)
@@ -494,7 +452,7 @@ class CrossrefMappingTest {
 
   @Test
   def testLicenseVorClosed() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("publication_license_vor.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/publication_license_vor.json")).mkString
 
 
     assertNotNull(json)
@@ -521,7 +479,7 @@ class CrossrefMappingTest {
 
   @Test
   def testLicenseOpen() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("publication_license_open.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/publication_license_open.json")).mkString
 
 
     assertNotNull(json)
@@ -544,7 +502,7 @@ class CrossrefMappingTest {
 
   @Test
   def testLicenseEmbargoOpen() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("publication_license_embargo_open.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/publication_license_embargo_open.json")).mkString
 
 
     assertNotNull(json)
@@ -567,7 +525,7 @@ class CrossrefMappingTest {
 
   @Test
   def testLicenseEmbargo() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("publication_license_embargo.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/publication_license_embargo.json")).mkString
 
 
     assertNotNull(json)
@@ -591,7 +549,7 @@ class CrossrefMappingTest {
 
   @Test
   def testLicenseEmbargoDateTime() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("publication_license_embargo_datetime.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/publication_license_embargo_datetime.json")).mkString
 
 
     assertNotNull(json)
@@ -614,7 +572,7 @@ class CrossrefMappingTest {
 
   @Test
   def testMultipleURLs() :Unit = {
-    val json = Source.fromInputStream(getClass.getResourceAsStream("multiple_urls.json")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/multiple_urls.json")).mkString
 
 
     assertNotNull(json)

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/mag/MAGMappingTest.scala

@@ -1,11 +1,12 @@
-package eu.dnetlib.doiboost.mag
+package eu.dnetlib.dhp.doiboost.mag
 
+import eu.dnetlib.doiboost.mag.{ConversionUtil, MagPapers, SparkProcessMAG}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, SparkSession}
 import org.codehaus.jackson.map.ObjectMapper
+import org.json4s.DefaultFormats
 import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.Test
-import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
 
 import java.sql.Timestamp
@@ -47,7 +48,7 @@ class MAGMappingTest {
 
   @Test
   def buildInvertedIndexTest(): Unit = {
-    val json_input = Source.fromInputStream(getClass.getResourceAsStream("invertedIndex.json")).mkString
+    val json_input = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/mag/invertedIndex.json")).mkString
     val description = ConversionUtil.convertInvertedIndexString(json_input)
     assertNotNull(description)
     assertTrue(description.nonEmpty)
@@ -71,7 +72,7 @@ class MAGMappingTest {
         .appName(getClass.getSimpleName)
         .config(conf)
         .getOrCreate()
-    val path = getClass.getResource("magPapers.json").getPath
+    val path = getClass.getResource("/eu/dnetlib/doiboost/mag/magPapers.json").getPath
 
     import org.apache.spark.sql.Encoders
     val schema = Encoders.product[MagPapers].schema
@@ -101,7 +102,7 @@ class MAGMappingTest {
         .appName(getClass.getSimpleName)
         .config(conf)
         .getOrCreate()
-    val path = getClass.getResource("duplicatedMagPapers.json").getPath
+    val path = getClass.getResource("/eu/dnetlib/doiboost/mag/duplicatedMagPapers.json").getPath
 
     import org.apache.spark.sql.Encoders
     val schema = Encoders.product[MagPapers].schema

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/orcid/MappingORCIDToOAFTest.scala

@@ -1,7 +1,8 @@
-package eu.dnetlib.doiboost.orcid
+package eu.dnetlib.dhp.doiboost.orcid
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.schema.oaf.Publication
+import eu.dnetlib.doiboost.orcid._
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.junit.jupiter.api.Assertions._
@@ -10,9 +11,8 @@ import org.junit.jupiter.api.io.TempDir
 import org.slf4j.{Logger, LoggerFactory}
 
 import java.nio.file.Path
-import scala.io.Source
-
 import scala.collection.JavaConversions._
+import scala.io.Source
 
 class MappingORCIDToOAFTest {
   val logger: Logger = LoggerFactory.getLogger(ORCIDToOAF.getClass)
@@ -20,7 +20,7 @@ class MappingORCIDToOAFTest {
 
   @Test
   def testExtractData():Unit ={
-    val json = Source.fromInputStream(getClass.getResourceAsStream("dataOutput")).mkString
+    val json = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/orcid/dataOutput")).mkString
     assertNotNull(json)
     assertFalse(json.isEmpty)
     json.lines.foreach(s => {

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/uw/UnpayWallMappingTest.scala

@@ -1,13 +1,13 @@
-package eu.dnetlib.doiboost.uw
+package eu.dnetlib.dhp.doiboost.uw
-
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.schema.oaf.OpenAccessRoute
+import eu.dnetlib.doiboost.uw.UnpayWallToOAF
+import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.Test
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.io.Source
-import org.junit.jupiter.api.Assertions._
-import org.slf4j.{Logger, LoggerFactory}
 
 class UnpayWallMappingTest {
 
@@ -18,7 +18,7 @@ class UnpayWallMappingTest {
   @Test
   def testMappingToOAF():Unit ={
 
-    val Ilist = Source.fromInputStream(getClass.getResourceAsStream("input.json")).mkString
+    val Ilist = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/uw/input.json")).mkString
 
     var i:Int = 0
     for (line <-Ilist.lines) {

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/Aggregators.scala

@@ -1,8 +1,8 @@
 package eu.dnetlib.dhp.oa.graph.hostedbymap
 
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, TypedColumn}
 import org.apache.spark.sql.expressions.Aggregator
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, TypedColumn}
 
 
 case class HostedByItemType(id: String, officialname: String, issn: String, eissn: String, lissn: String, openAccess: Boolean) {}

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToDatasource.scala

@@ -2,13 +2,12 @@ package eu.dnetlib.dhp.oa.graph.hostedbymap
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.oa.graph.hostedbymap.SparkApplyHostedByMapToResult.{applyHBtoPubs, getClass}
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
 import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.{Datasource, Publication}
+import eu.dnetlib.dhp.schema.oaf.Datasource
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToResult.scala

@@ -5,16 +5,14 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils
-import eu.dnetlib.dhp.schema.oaf.{Datasource, Instance, OpenAccessRoute, Publication}
+import eu.dnetlib.dhp.schema.oaf.{Instance, OpenAccessRoute, Publication}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
-
 import scala.collection.JavaConverters._
 
-
 object SparkApplyHostedByMapToResult {
 
   def applyHBtoPubs(join: Dataset[EntityInfo], pubs: Dataset[Publication]) = {
@@ -25,7 +23,7 @@ object SparkApplyHostedByMapToResult {
           val ei: EntityInfo = t2._2
           val i = p.getInstance().asScala
           if (i.size == 1) {
-            val inst: Instance = i(0)
+            val inst: Instance = i.head
             inst.getHostedby.setKey(ei.getHostedById)
             inst.getHostedby.setValue(ei.getName)
             if (ei.getOpenAccess) {
@@ -39,6 +37,7 @@ object SparkApplyHostedByMapToResult {
         p
       })(Encoders.bean(classOf[Publication]))
   }
+
   def main(args: Array[String]): Unit = {
 
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkPrepareHostedByInfoToApply.scala

@@ -3,18 +3,15 @@ package eu.dnetlib.dhp.oa.graph.hostedbymap
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
-
 import eu.dnetlib.dhp.schema.oaf.{Journal, Publication}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
 import org.slf4j.{Logger, LoggerFactory}
 
-
-
 object SparkPrepareHostedByInfoToApply {
 
   implicit val mapEncoderPInfo: Encoder[EntityInfo] = Encoders.bean(classOf[EntityInfo])

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkProduceHostedByMap.scala

@@ -1,22 +1,20 @@
 package eu.dnetlib.dhp.oa.graph.hostedbymap
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.{DOAJModel, UnibiGoldModel}
 import eu.dnetlib.dhp.schema.oaf.Datasource
 import org.apache.commons.io.IOUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
-import com.fasterxml.jackson.databind.ObjectMapper
+
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.Path
 import java.io.PrintWriter
 
-import org.apache.hadoop.io.compress.GzipCodec
-
-
 object SparkProduceHostedByMap {
 
 
@@ -52,7 +50,6 @@ object SparkProduceHostedByMap {
   }
 
 
-
   def getHostedByItemType(id: String, officialname: String, issn: String, eissn: String, issnl: String, oa: Boolean): HostedByItemType = {
     if (issn != null) {
       if (eissn != null) {
@@ -163,7 +160,6 @@ object SparkProduceHostedByMap {
   }
 
 
-
   def writeToHDFS(input: Array[String], outputPath: String, hdfsNameNode: String): Unit = {
     val conf = new Configuration()
 
@@ -182,7 +178,6 @@ object SparkProduceHostedByMap {
   }
 
 
-
   def main(args: Array[String]): Unit = {
 
     val logger: Logger = LoggerFactory.getLogger(getClass)

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/raw/CopyHdfsOafSparkApplication.scala

@@ -4,17 +4,11 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.common.HdfsSupport
 import eu.dnetlib.dhp.schema.common.ModelSupport
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo
 import eu.dnetlib.dhp.schema.oaf.Oaf
 import eu.dnetlib.dhp.utils.DHPUtils
-import org.apache.commons.io.IOUtils
-import org.apache.commons.lang3.StringUtils
-import org.apache.http.client.methods.HttpGet
-import org.apache.http.impl.client.HttpClients
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
 import org.apache.spark.{SparkConf, SparkContext}
 import org.slf4j.LoggerFactory
-
 import scala.collection.JavaConverters._
 import scala.io.Source
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/resolution/SparkResolveEntities.scala

@@ -2,9 +2,8 @@ package eu.dnetlib.dhp.oa.graph.resolution
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.common.HdfsSupport
 import eu.dnetlib.dhp.schema.common.EntityType
-import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{Dataset => OafDataset,_}
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.spark.SparkConf

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/resolution/SparkResolveRelation.scala

@@ -3,7 +3,7 @@ package eu.dnetlib.dhp.oa.graph.resolution
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.common.HdfsSupport
-import eu.dnetlib.dhp.schema.oaf.{Relation, Result}
+import eu.dnetlib.dhp.schema.oaf.Relation
 import eu.dnetlib.dhp.utils.DHPUtils
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.fs.{FileSystem, Path}

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/sx/graphimport/SparkDataciteToOAF.scala

@@ -18,7 +18,6 @@ object SparkDataciteToOAF {
         .config(conf)
         .appName(getClass.getSimpleName)
         .master(parser.get("master")).getOrCreate()
-    import spark.implicits._
 
 
     val sc = spark.sparkContext

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkConvertDatasetToJsonRDD.scala

@@ -2,7 +2,7 @@ package eu.dnetlib.dhp.sx.graph
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Oaf, OtherResearchProduct, Publication, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.Result
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.SparkConf

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkConvertObjectToJson.scala

@@ -5,10 +5,10 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.sx.scholix.Scholix
 import eu.dnetlib.dhp.schema.sx.summary.ScholixSummary
 import org.apache.commons.io.IOUtils
+import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
-import org.apache.hadoop.io.compress._
 
 object SparkConvertObjectToJson {
 

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkConvertRDDtoDataset.scala

@@ -2,11 +2,12 @@ package eu.dnetlib.dhp.sx.graph
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Relation, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Relation, Software,Dataset => OafDataset}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
+
 object SparkConvertRDDtoDataset {
 
   def main(args: Array[String]): Unit = {

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateInputGraph.scala

@@ -1,14 +1,12 @@
 package eu.dnetlib.dhp.sx.graph
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Oaf, OtherResearchProduct, Publication, Relation, Result, Software, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{Dataset => OafDataset,_}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
-
-
 object SparkCreateInputGraph {
 
   def main(args: Array[String]): Unit = {
@@ -41,9 +39,6 @@ object SparkCreateInputGraph {
     implicit val relEncoder: Encoder[Relation] = Encoders.kryo(classOf[Relation])
 
 
-
-
-
     val sourcePath = parser.get("sourcePath")
     log.info(s"sourcePath  -> $sourcePath")
     val targetPath = parser.get("targetPath")

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholix.scala

@@ -9,7 +9,7 @@ import eu.dnetlib.dhp.sx.graph.scholix.ScholixUtils.RelatedEntities
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.functions.count
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkCreateScholix {

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateSummaryObject.scala

@@ -6,7 +6,7 @@ import eu.dnetlib.dhp.schema.sx.summary.ScholixSummary
 import eu.dnetlib.dhp.sx.graph.scholix.ScholixUtils
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkCreateSummaryObject {

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/pangaea/PangaeaUtils.scala

@@ -5,6 +5,7 @@ import org.apache.spark.sql.{Encoder, Encoders}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
+
 import java.util.regex.Pattern
 import scala.language.postfixOps
 import scala.xml.{Elem, Node, XML}

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/pangaea/SparkGeneratePanagaeaDataset.scala

@@ -2,11 +2,11 @@ package eu.dnetlib.dhp.sx.graph.pangaea
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.spark.rdd.RDD
-import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.{SparkConf, SparkContext}
 import org.slf4j.{Logger, LoggerFactory}
-
 import scala.collection.JavaConverters._
+
 import scala.io.Source
 
 object SparkGeneratePanagaeaDataset {
@@ -46,7 +46,4 @@ object SparkGeneratePanagaeaDataset {
   }
 
 
-
-
-
 }

dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala

@@ -1,6 +1,5 @@
 package eu.dnetlib.dhp.sx.graph.scholix
 
-
 import eu.dnetlib.dhp.schema.oaf.{Publication, Relation, Result, StructuredProperty}
 import eu.dnetlib.dhp.schema.sx.scholix._
 import eu.dnetlib.dhp.schema.sx.summary.{CollectedFromType, SchemeValue, ScholixSummary, Typology}
@@ -10,10 +9,8 @@ import org.apache.spark.sql.{Encoder, Encoders}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
-
 import scala.collection.JavaConverters._
 import scala.io.Source
-import scala.language.postfixOps
 
 object ScholixUtils {
 
@@ -21,6 +18,7 @@ object ScholixUtils {
   val DNET_IDENTIFIER_SCHEMA: String = "DNET Identifier"
 
   val DATE_RELATION_KEY: String = "RelationDate"
+
   case class RelationVocabulary(original: String, inverse: String) {}
 
   case class RelatedEntities(id: String, relatedDataset: Long, relatedPublication: Long) {}
@@ -66,7 +64,6 @@ object ScholixUtils {
   }
 
 
-
   val statsAggregator: Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] = new Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] with Serializable {
     override def zero: RelatedEntities = null
 
@@ -85,8 +82,7 @@ object ScholixUtils {
       if (b1 != null && b2 != null)
         RelatedEntities(b1.id, b1.relatedDataset + b2.relatedDataset, b1.relatedPublication + b2.relatedPublication)
 
-      else
+      else if (b1 != null)
-        if (b1!= null)
         b1
       else
         b2
@@ -144,7 +140,6 @@ object ScholixUtils {
     s
 
 
-
   }
 
 
@@ -203,8 +198,7 @@ object ScholixUtils {
 
     if (summaryObject.getDate != null && !summaryObject.getDate.isEmpty)
       r.setPublicationDate(summaryObject.getDate.get(0))
-    if (summaryObject.getPublisher!= null && !summaryObject.getPublisher.isEmpty)
+    if (summaryObject.getPublisher != null && !summaryObject.getPublisher.isEmpty) {
-    {
       val plist: List[ScholixEntityId] = summaryObject.getPublisher.asScala.map(p => new ScholixEntityId(p, null)).toList
 
       if (plist.nonEmpty)
@@ -228,9 +222,6 @@ object ScholixUtils {
   }
 
 
-
-
-
   def scholixFromSource(relation: Relation, source: ScholixSummary): Scholix = {
 
     if (relation == null || source == null)

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java

@@ -171,24 +171,6 @@ public class DumpJobTest {
 
 		GraphResult gr = verificationDataset.first();
 
-		Assertions.assertEquals(2, gr.getMeasures().size());
-		Assertions
-			.assertTrue(
-				gr
-					.getMeasures()
-					.stream()
-					.anyMatch(
-						m -> m.getKey().equals("influence")
-							&& m.getValue().equals("1.62759106106e-08")));
-		Assertions
-			.assertTrue(
-				gr
-					.getMeasures()
-					.stream()
-					.anyMatch(
-						m -> m.getKey().equals("popularity")
-							&& m.getValue().equals("0.22519296")));
-
 		Assertions.assertEquals(6, gr.getAuthor().size());
 		Assertions
 			.assertTrue(

dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/TestPrepare.scala

@@ -3,13 +3,9 @@ package eu.dnetlib.dhp.oa.graph.hostedbymap
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.oa.graph.hostedbymap.SparkPrepareHostedByInfoToApply.{joinResHBM, prepareResultInfo, toEntityInfo}
 import eu.dnetlib.dhp.oa.graph.hostedbymap.model.EntityInfo
-import eu.dnetlib.dhp.schema.oaf.{Datasource, OpenAccessRoute, Publication}
-import javax.management.openmbean.OpenMBeanAttributeInfo
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
-import org.json4s
 import org.json4s.DefaultFormats
-import eu.dnetlib.dhp.schema.common.ModelConstants
 import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue}
 import org.junit.jupiter.api.Test
 

dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/TestPreprocess.scala

@@ -4,10 +4,9 @@ import eu.dnetlib.dhp.schema.oaf.Datasource
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, Encoder, Encoders, SparkSession}
 import org.json4s.DefaultFormats
-import org.junit.jupiter.api.Assertions.{assertNotNull, assertTrue}
-import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.Assertions._
 import org.json4s.jackson.Serialization.write
+import org.junit.jupiter.api.Assertions._
+import org.junit.jupiter.api.Test
 
 class TestPreprocess extends java.io.Serializable{
 

dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGraphTest.scala

@@ -37,7 +37,8 @@ class ScholixGraphTest extends AbstractVocabularyTest{
     val input = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/scholix/result.json")).mkString
     val res =SparkResolveRelation.extractPidsFromRecord(input)
     assertNotNull(res)
-    assertTrue(res._2.size == 2)
+
+    assertEquals(1,res._2.size)
 
   }