code refactor, created and moved scala code on the correct maven folder under src/main/scala and src/test/scala

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala

@@ -4,20 +4,19 @@ import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf._
 import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, OafMapperUtils}
 import eu.dnetlib.dhp.utils.DHPUtils
-import eu.dnetlib.doiboost.DoiBoostMappingUtil.{decideAccessRight, _}
+import eu.dnetlib.doiboost.DoiBoostMappingUtil
+import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 import org.apache.commons.lang.StringUtils
 import org.json4s
 import org.json4s.DefaultFormats
-import org.json4s.JsonAST.{JValue, _}
+import org.json4s.JsonAST._
 import org.json4s.jackson.JsonMethods._
 import org.slf4j.{Logger, LoggerFactory}
 
+import java.util
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.util.matching.Regex
-import java.util
-
-import eu.dnetlib.doiboost.DoiBoostMappingUtil
 
 case class CrossrefDT(doi: String, json:String, timestamp: Long) {}
 

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/CrossrefDataset.scala

@@ -6,7 +6,7 @@ import org.apache.commons.io.IOUtils
 import org.apache.hadoop.io.{IntWritable, Text}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.expressions.Aggregator
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql.{Dataset, Encoder, SaveMode, SparkSession}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
@@ -30,7 +30,6 @@ object CrossrefDataset {
   def main(args: Array[String]): Unit = {
 
 
-
     val conf: SparkConf = new SparkConf()
     val parser = new ArgumentApplicationParser(IOUtils.toString(CrossrefDataset.getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref_to_dataset_params.json")))
     parser.parseArgument(args)

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/GenerateCrossrefDataset.scala

@@ -2,17 +2,12 @@ package eu.dnetlib.doiboost.crossref
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
-import eu.dnetlib.doiboost.crossref.CrossrefDataset.to_item
-import eu.dnetlib.doiboost.crossref.UnpackCrtossrefEntries.getClass
-import org.apache.hadoop.io.{IntWritable, Text}
-import org.apache.hadoop.io.compress.GzipCodec
 import org.apache.spark.rdd.RDD
-import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.{SparkConf, SparkContext}
 import org.json4s
 import org.json4s.DefaultFormats
-import org.json4s.JsonAST.JArray
-import org.json4s.jackson.JsonMethods.{compact, parse, render}
+import org.json4s.jackson.JsonMethods.parse
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.io.Source
@@ -24,7 +19,6 @@ object GenerateCrossrefDataset {
   implicit val mrEncoder: Encoder[CrossrefDT] = Encoders.kryo[CrossrefDT]
 
 
-
   def crossrefElement(meta: String): CrossrefDT = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
     lazy val json: json4s.JValue = parse(meta)

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/SparkMapDumpIntoOAF.scala

@@ -4,10 +4,8 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.oaf
 import eu.dnetlib.dhp.schema.oaf.{Oaf, Publication, Relation, Dataset => OafDataset}
 import org.apache.commons.io.IOUtils
-
 import org.apache.spark.SparkConf
-
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/UnpackCrtossrefEntries.scala

@@ -2,8 +2,8 @@ package eu.dnetlib.doiboost.crossref
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.hadoop.io.compress.GzipCodec
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.JArray
@@ -17,8 +17,6 @@ object UnpackCrtossrefEntries {
   val log: Logger = LoggerFactory.getLogger(UnpackCrtossrefEntries.getClass)
 
 
-
-
   def extractDump(input: String): List[String] = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
     lazy val json: json4s.JValue = parse(input)
@@ -30,7 +28,6 @@ object UnpackCrtossrefEntries {
   }
 
 
-
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf
     val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref_dump_reader/generate_dataset_params.json")).mkString)

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/mag/MagDataModel.scala

@@ -5,10 +5,10 @@ import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
 import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, StructuredProperty}
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
+import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
-import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/mag/SparkImportMagIntoDataset.scala

@@ -3,8 +3,8 @@ package eu.dnetlib.doiboost.mag
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkImportMagIntoDataset {
@@ -75,7 +75,6 @@ object SparkImportMagIntoDataset {
         .master(parser.get("master")).getOrCreate()
 
 
-
     stream.foreach { case (k, v) =>
       val s: StructType = getSchema(k)
       val df = spark.read

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/mag/SparkProcessMAG.scala

@@ -5,13 +5,10 @@ import eu.dnetlib.dhp.schema.oaf.Publication
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.functions._
+import org.apache.spark.sql.functions.{col, collect_list, struct}
 import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
-
 import scala.collection.JavaConverters._
-
 object SparkProcessMAG {
 
   def getDistinctResults(d: Dataset[MagPapers]): Dataset[MagPapers] = {
@@ -153,6 +150,5 @@ object SparkProcessMAG {
       .write.mode(SaveMode.Overwrite).save(s"$targetPath/magPublication")
 
 
-
   }
 }

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/orcid/ORCIDToOAF.scala

@@ -4,17 +4,16 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
 import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Publication}
-import eu.dnetlib.dhp.schema.orcid.{AuthorData, OrcidDOI}
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
 import eu.dnetlib.doiboost.DoiBoostMappingUtil.{createSP, generateDataInfo}
 import org.apache.commons.lang.StringUtils
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.collection.JavaConverters._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST._
 import org.json4s.jackson.JsonMethods._
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.collection.JavaConverters._
 
 
 case class ORCIDItem(doi:String, authors:List[OrcidAuthor]){}

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/orcid/SparkPreprocessORCID.scala

@@ -1,15 +1,12 @@
 package eu.dnetlib.doiboost.orcid
 
-import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.oa.merge.AuthorMerger
 import eu.dnetlib.dhp.schema.oaf.Publication
-import eu.dnetlib.dhp.schema.orcid.OrcidDOI
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.functions._
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql.functions.{col, collect_list}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkPreprocessORCID {

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/uw/SparkMapUnpayWallToOAF.scala

@@ -1,16 +1,14 @@
 package eu.dnetlib.doiboost.uw
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-
 import eu.dnetlib.dhp.schema.oaf.Publication
 import eu.dnetlib.doiboost.crossref.SparkMapDumpIntoOAF
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
-
 object SparkMapUnpayWallToOAF {
 
   def main(args: Array[String]): Unit = {

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala

@@ -4,14 +4,13 @@ import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
 import eu.dnetlib.dhp.schema.oaf.{AccessRight, Instance, OpenAccessRoute, Publication}
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
+import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.JavaConverters._
-import eu.dnetlib.doiboost.DoiBoostMappingUtil._
-import eu.dnetlib.doiboost.uw.UnpayWallToOAF.get_unpaywall_color
 
 
 

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/doiboost/DoiBoostHostedByMapTest.scala

@@ -1,4 +1,4 @@
-package eu.dnetlib.dhp.doiboost
+package eu.dnetlib.doiboost
 
 import eu.dnetlib.dhp.schema.oaf.{Publication, Dataset => OafDataset}
 import eu.dnetlib.doiboost.{DoiBoostMappingUtil, HostedByItemType}

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/doiboost/NormalizeDoiTest.scala

@@ -1,4 +1,4 @@
-package eu.dnetlib.dhp.doiboost
+package eu.dnetlib.doiboost
 
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
 import org.junit.jupiter.api.Test

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/doiboost/mag/MAGMappingTest.scala

@@ -3,9 +3,9 @@ package eu.dnetlib.doiboost.mag
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, SparkSession}
 import org.codehaus.jackson.map.ObjectMapper
+import org.json4s.DefaultFormats
 import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.Test
-import org.json4s.DefaultFormats
 import org.slf4j.{Logger, LoggerFactory}
 
 import java.sql.Timestamp

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/doiboost/orcid/MappingORCIDToOAFTest.scala

@@ -10,9 +10,8 @@ import org.junit.jupiter.api.io.TempDir
 import org.slf4j.{Logger, LoggerFactory}
 
 import java.nio.file.Path
-import scala.io.Source
-
 import scala.collection.JavaConversions._
+import scala.io.Source
 
 class MappingORCIDToOAFTest {
   val logger: Logger = LoggerFactory.getLogger(ORCIDToOAF.getClass)

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/doiboost/uw/UnpayWallMappingTest.scala

@@ -3,11 +3,11 @@ package eu.dnetlib.doiboost.uw
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.schema.oaf.OpenAccessRoute
+import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.Test
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.io.Source
-import org.junit.jupiter.api.Assertions._
-import org.slf4j.{Logger, LoggerFactory}
 
 class UnpayWallMappingTest {