From 2557bb41f54cbe8232372bfa1c554ce5d3177cbd Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Wed, 6 Oct 2021 16:41:08 +0200
Subject: [PATCH 01/12] Implemented new method for update baseline inside scala
 node

---
 .../ebi/SparkCreateBaselineDataFrame.scala    |  123 +-
 .../sx/graph/ebi/SparkDownloadEBILinks.scala  |   45 +-
 .../sx/graph/ebi/baseline_to_oaf_params.json  |    9 +-
 .../dhp/sx/graph/ebi/ebi_download_update.json |    6 +-
 .../dhp/sx/graph/ebi/oozie_app/workflow.xml   |    2 +-
 .../graph/ebi/update/oozie_app/workflow.xml   |  116 +-
 .../sx/graph/bio/pubmed/BioScholixTest.scala  |    7 +-
 .../dnetlib/dhp/sx/graph/bio/pubmed/ls_result | 1433 +++++++++++++++++
 8 files changed, 1625 insertions(+), 116 deletions(-)
 create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/ls_result

diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkCreateBaselineDataFrame.scala b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkCreateBaselineDataFrame.scala
index 26efd723f..ee76cf8ad 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkCreateBaselineDataFrame.scala
+++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkCreateBaselineDataFrame.scala
@@ -6,18 +6,130 @@ import eu.dnetlib.dhp.schema.oaf.Result
 import eu.dnetlib.dhp.sx.graph.bio.pubmed.{PMArticle, PMAuthor, PMJournal, PMParser, PubMedToOaf}
 import eu.dnetlib.dhp.utils.ISLookupClientFactory
 import org.apache.commons.io.IOUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}
+import org.apache.http.client.config.RequestConfig
+import org.apache.http.client.methods.HttpGet
+import org.apache.http.impl.client.HttpClientBuilder
 import org.apache.spark.SparkConf
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.expressions.Aggregator
 import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
+import java.io.InputStream
 import scala.io.Source
 import scala.xml.pull.XMLEventReader
 
 object SparkCreateBaselineDataFrame {
 
 
+  def requestBaseLineUpdatePage(maxFile:String):List[(String,String)] = {
+    val data =requestPage("https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/")
+
+    val result =data.lines.filter(l => l.startsWith("<a href=")).map{l =>
+      val end = l.lastIndexOf("\">")
+      val start = l.indexOf("<a href=\"")
+
+      if (start>= 0 && end >start)
+        l.substring(start+9, (end-start))
+      else
+        ""
+    }.filter(s =>s.endsWith(".gz") ).filter(s => s > maxFile).map(s => (s,s"https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/$s")).toList
+
+    result
+  }
+
+
+  def downloadBaselinePart(url:String):InputStream = {
+    val r = new HttpGet(url)
+    val timeout = 60; // seconds
+    val config = RequestConfig.custom()
+      .setConnectTimeout(timeout * 1000)
+      .setConnectionRequestTimeout(timeout * 1000)
+      .setSocketTimeout(timeout * 1000).build()
+    val client = HttpClientBuilder.create().setDefaultRequestConfig(config).build()
+    val response = client.execute(r)
+    println(s"get response with status${response.getStatusLine.getStatusCode}")
+    response.getEntity.getContent
+
+  }
+
+  def requestPage(url:String):String = {
+    val r = new HttpGet(url)
+    val timeout = 60; // seconds
+    val config = RequestConfig.custom()
+      .setConnectTimeout(timeout * 1000)
+      .setConnectionRequestTimeout(timeout * 1000)
+      .setSocketTimeout(timeout * 1000).build()
+    val client = HttpClientBuilder.create().setDefaultRequestConfig(config).build()
+    try {
+      var tries = 4
+      while (tries > 0) {
+        println(s"requesting ${r.getURI}")
+        try {
+          val response = client.execute(r)
+          println(s"get response with status${response.getStatusLine.getStatusCode}")
+          if (response.getStatusLine.getStatusCode > 400) {
+            tries -= 1
+          }
+          else
+            return IOUtils.toString(response.getEntity.getContent)
+        } catch {
+          case e: Throwable =>
+            println(s"Error on requesting ${r.getURI}")
+            e.printStackTrace()
+            tries -= 1
+        }
+      }
+      ""
+    } finally {
+      if (client != null)
+        client.close()
+    }
+  }
+
+
+
+
+
+
+  def downloadBaseLineUpdate(baselinePath:String, hdfsServerUri:String ):Unit = {
+
+
+    val conf = new Configuration
+    conf.set("fs.defaultFS", hdfsServerUri)
+    val fs =  FileSystem.get(conf)
+    val p = new Path(baselinePath)
+    val files = fs.listFiles(p,false)
+    var max_file = ""
+    while (files.hasNext) {
+      val c = files.next()
+      val data = c.getPath.toString
+      val fileName = data.substring(data.lastIndexOf("/")+1)
+
+      if (fileName> max_file)
+        max_file = fileName
+    }
+
+    val files_to_download = requestBaseLineUpdatePage(max_file)
+
+    files_to_download.foreach { u =>
+      val hdfsWritePath: Path = new Path(s"$baselinePath/${u._1}")
+      val fsDataOutputStream: FSDataOutputStream = fs.create(hdfsWritePath, true)
+      val i = downloadBaselinePart(u._2)
+      val buffer = Array.fill[Byte](1024)(0)
+      while(i.read(buffer)>0) {
+        fsDataOutputStream.write(buffer)
+      }
+      i.close()
+      println(s"Downloaded ${u._2} into $baselinePath/${u._1}")
+      fsDataOutputStream.close()
+    }
+
+  }
+
+
   val pmArticleAggregator: Aggregator[(String, PMArticle), PMArticle, PMArticle] = new Aggregator[(String, PMArticle), PMArticle, PMArticle] with Serializable {
     override def zero: PMArticle = new PMArticle
 
@@ -51,6 +163,10 @@ object SparkCreateBaselineDataFrame {
     val targetPath = parser.get("targetPath")
     log.info("targetPath: {}", targetPath)
 
+    val hdfsServerUri = parser.get("hdfsServerUri")
+    log.info("hdfsServerUri: {}", targetPath)
+
+
     val isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl)
     val vocabularies = VocabularyGroup.loadVocsFromIS(isLookupService)
     val spark: SparkSession =
@@ -61,16 +177,15 @@ object SparkCreateBaselineDataFrame {
         .master(parser.get("master")).getOrCreate()
     import spark.implicits._
 
-
     val sc = spark.sparkContext
 
-
-
     implicit  val PMEncoder: Encoder[PMArticle] = Encoders.kryo(classOf[PMArticle])
     implicit  val PMJEncoder: Encoder[PMJournal] = Encoders.kryo(classOf[PMJournal])
     implicit  val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
     implicit  val resultEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
 
+    downloadBaseLineUpdate(s"$workingPath/baseline", hdfsServerUri)
+
     val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline",2000)
     val ds:Dataset[PMArticle] = spark.createDataset(k.filter(i => i._1.endsWith(".gz")).flatMap(i =>{
       val xml = new XMLEventReader(Source.fromBytes(i._2.getBytes()))
@@ -87,7 +202,5 @@ object SparkCreateBaselineDataFrame {
       .map(a => PubMedToOaf.convert(a, vocabularies)).as[Result]
       .filter(p => p!= null)
       .write.mode(SaveMode.Overwrite).save(targetPath)
-
-    //s"$workingPath/oaf/baseline_oaf"
   }
 }
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkDownloadEBILinks.scala b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkDownloadEBILinks.scala
index eda825bd0..e940fdff0 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkDownloadEBILinks.scala
+++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkDownloadEBILinks.scala
@@ -4,20 +4,16 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF.EBILinkItem
 import eu.dnetlib.dhp.sx.graph.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
 import org.apache.commons.io.IOUtils
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.http.client.config.RequestConfig
-import org.apache.http.client.methods.{HttpGet, HttpUriRequest}
+import org.apache.http.client.methods.HttpGet
 import org.apache.http.impl.client.HttpClientBuilder
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.expressions.Aggregator
 import org.apache.spark.sql.functions.max
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
 
 object SparkDownloadEBILinks {
 
-
   def createEBILinks(pmid:Long):EBILinkItem = {
 
     val res = requestLinks(pmid)
@@ -26,7 +22,6 @@ object SparkDownloadEBILinks {
     null
   }
 
-
   def requestPage(url:String):String = {
     val r = new HttpGet(url)
     val timeout = 60; // seconds
@@ -61,42 +56,6 @@ object SparkDownloadEBILinks {
     }
   }
 
-
-  def requestBaseLineUpdatePage():List[String] = {
-    val data =requestPage("https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/")
-
-    val result =data.lines.filter(l => l.startsWith("<a href=")).map{l =>
-      val end = l.lastIndexOf("\">")
-      val start = l.indexOf("<a href=\"")
-
-      if (start>= 0 && end >start)
-        l.substring(start+9, (end-start))
-      else
-        ""
-    }.filter(s =>s.endsWith(".gz") ).map(s => s"https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/$s").toList
-
-    result
-  }
-
-  def downloadBaseLineUpdate(baselinePath:String, hdfsServerUri:String ):Unit = {
-
-
-    val conf = new Configuration
-    conf.set("fs.defaultFS", hdfsServerUri)
-    val fs =  FileSystem.get(conf)
-    val p = new Path((baselinePath))
-    val files = fs.listFiles(p,false)
-
-    while (files.hasNext) {
-      val c = files.next()
-      c.getPath
-
-    }
-
-
-  }
-
-
   def requestLinks(PMID:Long):String = {
     requestPage(s"https://www.ebi.ac.uk/europepmc/webservices/rest/MED/$PMID/datalinks?format=json")
 
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/baseline_to_oaf_params.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/baseline_to_oaf_params.json
index 38eb50094..4bee770bd 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/baseline_to_oaf_params.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/baseline_to_oaf_params.json
@@ -1,6 +1,7 @@
 [
-  {"paramName":"mt",  "paramLongName":"master",     "paramDescription": "should be local or yarn",                  "paramRequired": true},
-  {"paramName":"i",   "paramLongName":"isLookupUrl","paramDescription": "isLookupUrl",                              "paramRequired": true},
-  {"paramName":"w",   "paramLongName":"workingPath","paramDescription": "the path of the sequencial file to read",  "paramRequired": true},
-  {"paramName":"t",   "paramLongName":"targetPath","paramDescription": "the oaf path ",                             "paramRequired": true}
+  {"paramName":"mt",  "paramLongName":"master",         "paramDescription": "should be local or yarn",                  "paramRequired": true},
+  {"paramName":"i",   "paramLongName":"isLookupUrl",    "paramDescription": "isLookupUrl",                              "paramRequired": true},
+  {"paramName":"w",   "paramLongName":"workingPath",    "paramDescription": "the path of the sequencial file to read",  "paramRequired": true},
+  {"paramName":"t",   "paramLongName":"targetPath",     "paramDescription": "the oaf path ",                            "paramRequired": true},
+  {"paramName":"h",   "paramLongName":"hdfsServerUri",  "paramDescription": "the  working path ",                       "paramRequired": true}
 ]
\ No newline at end of file
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_download_update.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_download_update.json
index 0ae19234a..0860ed558 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_download_update.json
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_download_update.json
@@ -1,5 +1,5 @@
 [
-  {"paramName":"mt",  "paramLongName":"master",     "paramDescription": "should be local or yarn",    "paramRequired": true},
-  {"paramName":"s",   "paramLongName":"sourcePath","paramDescription": "the source Path",             "paramRequired": true},
-  {"paramName":"w",   "paramLongName":"workingPath","paramDescription": "the  working path ",         "paramRequired": true}
+  {"paramName":"mt",  "paramLongName":"master",         "paramDescription": "should be local or yarn",    "paramRequired": true},
+  {"paramName":"s",   "paramLongName":"sourcePath",     "paramDescription": "the source Path",            "paramRequired": true},
+  {"paramName":"w",   "paramLongName":"workingPath",    "paramDescription": "the  working path ",         "paramRequired": true}
 ]
\ No newline at end of file
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml
index 1fa099ebc..7612321c0 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/oozie_app/workflow.xml
@@ -25,7 +25,6 @@
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
-
     <action name="GenerateBaselineDataset">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn-cluster</master>
@@ -43,6 +42,7 @@
             </spark-opts>
             <arg>--workingPath</arg><arg>${workingPath}</arg>
             <arg>--master</arg><arg>yarn</arg>
+            <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/update/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/update/oozie_app/workflow.xml
index 1b738caed..cd3bb8c71 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/update/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/update/oozie_app/workflow.xml
@@ -1,59 +1,67 @@
-<workflow-app name="Create EBI Dataset" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>sourcePath</name>
-            <description>the Working Path</description>
-        </property>
-        <property>
-            <name>workingPath</name>
-            <description>the Working Path</description>
-        </property>
-        <property>
-            <name>sparkDriverMemory</name>
-            <description>memory for driver process</description>
-        </property>
-        <property>
-            <name>sparkExecutorMemory</name>
-            <description>memory for individual executor</description>
-        </property>
-        <property>
-            <name>sparkExecutorCores</name>
-            <description>number of cores used by single executor</description>
-        </property>
-    </parameters>
+    <workflow-app name="Create EBI Dataset" xmlns="uri:oozie:workflow:0.5">
+        <parameters>
+            <property>
+                <name>sourcePath</name>
+                <description>the Working Path</description>
+            </property>
+            <property>
+                <name>workingPath</name>
+                <description>the Working Path</description>
+            </property>
+            <property>
+                <name>sparkDriverMemory</name>
+                <description>memory for driver process</description>
+            </property>
+            <property>
+                <name>sparkExecutorMemory</name>
+                <description>memory for individual executor</description>
+            </property>
+            <property>
+                <name>sparkExecutorCores</name>
+                <description>number of cores used by single executor</description>
+            </property>
+        </parameters>
 
-    <start to="DownloadEBILinks"/>
+        <start to="DownloadEBILinks"/>
 
 
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
+        <kill name="Kill">
+            <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+        </kill>
 
 
-    <action name="DownloadEBILinks">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Incremental Download EBI Links</name>
-            <class>eu.dnetlib.dhp.sx.graph.ebi.SparkDownloadEBILinks</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.shuffle.partitions=2000
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
-            <arg>--workingPath</arg><arg>${workingPath}</arg>
-            <arg>--master</arg><arg>yarn</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-    <end name="End"/>
-</workflow-app>
\ No newline at end of file
+        <action name="DownloadEBILinks">
+            <spark xmlns="uri:oozie:spark-action:0.2">
+                <master>yarn-cluster</master>
+                <mode>cluster</mode>
+                <name>Incremental Download EBI Links</name>
+                <class>eu.dnetlib.dhp.sx.graph.ebi.SparkDownloadEBILinks</class>
+                <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+                <spark-opts>
+                    --executor-memory=${sparkExecutorMemory}
+                    --executor-cores=${sparkExecutorCores}
+                    --driver-memory=${sparkDriverMemory}
+                    --conf spark.extraListeners=${spark2ExtraListeners}
+                    --conf spark.sql.shuffle.partitions=2000
+                    --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                    --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                    --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                </spark-opts>
+                <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+                <arg>--workingPath</arg><arg>${workingPath}</arg>
+                <arg>--master</arg><arg>yarn</arg>
+            </spark>
+            <ok to="OverrideFolders"/>
+            <error to="Kill"/>
+        </action>
+        <action name="OverrideFolders">
+            <fs>
+                <delete path="${sourcePath}/ebi_links_dataset_old"/>
+                <move source="${sourcePath}/ebi_links_dataset" target="${sourcePath}/ebi_links_dataset_old"/>
+                <move source="${workingPath}/links_final" target="${sourcePath}/ebi_links_dataset"/>
+            </fs>
+            <ok to="End"/>
+            <error to="Kill"/>
+        </action>
+        <end name="End"/>
+    </workflow-app>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/BioScholixTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/BioScholixTest.scala
index b6058f71b..87279eb21 100644
--- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/BioScholixTest.scala
+++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/BioScholixTest.scala
@@ -51,12 +51,7 @@ class BioScholixTest extends AbstractVocabularyTest{
   }
 
 
-  @Test
-  def testDownloadEBIUpdate() = {
-    val data = SparkDownloadEBILinks.requestBaseLineUpdatePage()
-    println(data)
-  }
-
+ 
 
   @Test
   def testEBIData() = {
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/ls_result b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/ls_result
new file mode 100644
index 000000000..98a0841c4
--- /dev/null
+++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/ls_result
@@ -0,0 +1,1433 @@
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0001.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0002.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0003.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0004.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0005.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0006.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0007.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0008.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0009.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0010.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0011.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0012.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0013.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0014.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0015.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0016.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0017.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0018.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0019.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0020.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0021.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0022.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0023.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0024.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0025.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0026.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0027.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0028.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0029.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0030.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0031.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0032.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0033.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0034.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0035.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0036.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0037.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0038.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0039.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0040.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0041.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0042.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0043.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0044.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0045.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0046.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0047.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0048.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0049.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0050.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0051.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0052.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0053.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0054.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0055.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0056.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0057.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0058.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0059.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0060.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0061.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0062.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0063.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0064.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0065.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0066.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0067.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0068.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0069.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0070.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0071.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0072.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0073.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0074.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0075.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0076.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0077.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0078.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0079.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0080.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0081.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0082.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0083.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0084.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0085.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0086.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0087.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0088.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0089.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0090.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0091.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0092.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0093.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0094.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0095.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0096.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0097.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0098.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0099.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0100.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0101.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0102.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0103.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0104.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0105.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0106.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0107.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0108.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0109.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0110.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0111.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0112.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0113.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0114.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0115.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0116.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0117.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0118.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0119.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0120.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0121.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0122.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0123.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0124.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0125.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0126.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0127.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0128.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0129.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0130.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0131.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0132.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0133.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0134.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0135.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0136.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0137.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0138.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0139.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0140.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0141.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0142.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0143.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0144.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0145.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0146.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0147.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0148.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0149.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0150.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0151.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0152.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0153.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0154.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0155.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0156.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0157.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0158.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0159.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0160.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0161.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0162.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0163.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0164.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0165.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0166.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0167.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0168.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0169.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0170.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0171.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0172.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0173.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0174.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0175.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0176.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0177.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0178.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0179.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0180.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0181.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0182.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0183.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0184.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0185.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0186.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0187.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0188.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0189.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0190.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0191.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0192.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0193.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0194.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0195.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0196.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0197.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0198.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0199.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0200.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0201.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0202.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0203.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0204.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0205.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0206.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0207.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0208.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0209.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0210.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0211.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0212.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0213.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0214.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0215.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0216.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0217.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0218.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0219.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0220.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0221.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0222.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0223.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0224.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0225.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0226.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0227.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0228.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0229.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0230.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0231.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0232.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0233.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0234.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0235.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0236.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0237.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0238.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0239.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0240.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0241.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0242.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0243.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0244.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0245.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0246.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0247.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0248.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0249.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0250.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0251.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0252.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0253.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0254.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0255.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0256.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0257.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0258.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0259.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0260.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0261.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0262.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0263.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0264.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0265.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0266.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0267.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0268.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0269.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0270.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0271.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0272.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0273.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0274.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0275.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0276.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0277.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0278.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0279.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0280.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0281.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0282.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0283.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0284.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0285.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0286.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0287.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0288.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0289.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0290.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0291.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0292.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0293.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0294.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0295.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0296.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0297.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0298.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0299.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0300.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0301.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0302.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0303.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0304.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0305.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0306.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0307.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0308.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0309.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0310.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0311.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0312.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0313.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0314.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0315.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0316.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0317.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0318.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0319.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0320.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0321.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0322.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0323.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0324.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0325.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0326.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0327.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0328.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0329.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0330.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0331.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0332.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0333.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0334.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0335.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0336.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0337.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0338.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0339.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0340.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0341.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0342.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0343.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0344.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0345.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0346.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0347.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0348.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0349.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0350.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0351.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0352.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0353.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0354.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0355.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0356.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0357.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0358.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0359.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0360.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0361.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0362.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0363.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0364.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0365.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0366.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0367.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0368.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0369.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0370.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0371.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0372.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0373.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0374.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0375.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0376.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0377.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0378.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0379.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0380.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0381.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0382.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0383.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0384.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0385.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0386.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0387.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0388.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0389.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0390.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0391.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0392.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0393.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0394.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0395.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0396.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0397.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0398.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0399.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0400.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0401.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0402.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0403.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0404.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0405.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0406.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0407.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0408.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0409.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0410.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0411.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0412.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0413.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0414.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0415.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0416.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0417.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0418.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0419.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0420.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0421.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0422.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0423.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0424.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0425.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0426.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0427.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0428.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0429.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0430.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0431.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0432.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0433.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0434.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0435.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0436.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0437.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0438.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0439.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0440.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0441.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0442.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0443.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0444.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0445.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0446.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0447.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0448.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0449.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0450.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0451.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0452.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0453.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0454.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0455.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0456.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0457.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0458.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0459.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0460.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0461.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0462.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0463.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0464.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0465.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0466.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0467.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0468.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0469.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0470.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0471.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0472.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0473.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0474.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0475.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0476.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0477.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0478.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0479.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0480.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0481.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0482.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0483.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0484.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0485.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0486.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0487.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0488.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0489.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0490.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0491.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0492.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0493.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0494.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0495.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0496.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0497.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0498.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0499.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0500.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0501.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0502.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0503.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0504.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0505.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0506.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0507.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0508.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0509.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0510.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0511.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0512.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0513.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0514.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0515.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0516.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0517.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0518.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0519.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0520.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0521.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0522.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0523.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0524.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0525.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0526.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0527.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0528.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0529.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0530.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0531.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0532.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0533.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0534.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0535.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0536.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0537.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0538.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0539.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0540.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0541.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0542.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0543.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0544.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0545.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0546.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0547.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0548.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0549.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0550.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0551.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0552.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0553.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0554.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0555.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0556.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0557.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0558.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0559.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0560.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0561.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0562.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0563.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0564.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0565.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0566.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0567.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0568.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0569.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0570.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0571.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0572.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0573.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0574.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0575.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0576.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0577.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0578.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0579.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0580.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0581.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0582.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0583.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0584.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0585.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0586.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0587.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0588.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0589.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0590.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0591.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0592.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0593.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0594.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0595.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0596.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0597.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0598.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0599.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0600.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0601.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0602.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0603.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0604.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0605.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0606.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0607.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0608.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0609.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0610.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0611.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0612.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0613.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0614.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0615.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0616.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0617.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0618.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0619.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0620.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0621.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0622.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0623.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0624.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0625.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0626.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0627.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0628.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0629.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0630.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0631.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0632.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0633.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0634.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0635.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0636.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0637.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0638.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0639.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0640.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0641.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0642.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0643.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0644.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0645.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0646.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0647.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0648.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0649.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0650.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0651.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0652.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0653.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0654.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0655.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0656.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0657.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0658.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0659.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0660.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0661.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0662.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0663.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0664.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0665.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0666.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0667.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0668.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0669.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0670.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0671.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0672.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0673.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0674.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0675.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0676.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0677.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0678.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0679.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0680.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0681.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0682.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0683.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0684.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0685.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0686.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0687.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0688.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0689.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0690.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0691.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0692.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0693.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0694.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0695.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0696.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0697.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0698.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0699.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0700.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0701.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0702.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0703.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0704.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0705.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0706.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0707.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0708.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0709.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0710.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0711.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0712.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0713.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0714.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0715.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0716.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0717.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0718.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0719.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0720.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0721.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0722.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0723.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0724.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0725.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0726.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0727.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0728.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0729.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0730.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0731.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0732.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0733.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0734.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0735.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0736.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0737.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0738.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0739.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0740.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0741.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0742.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0743.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0744.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0745.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0746.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0747.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0748.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0749.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0750.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0751.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0752.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0753.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0754.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0755.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0756.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0757.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0758.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0759.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0760.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0761.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0762.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0763.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0764.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0765.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0766.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0767.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0768.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0769.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0770.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0771.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0772.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0773.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0774.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0775.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0776.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0777.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0778.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0779.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0780.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0781.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0782.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0783.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0784.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0785.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0786.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0787.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0788.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0789.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0790.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0791.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0792.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0793.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0794.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0795.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0796.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0797.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0798.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0799.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0800.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0801.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0802.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0803.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0804.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0805.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0806.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0807.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0808.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0809.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0810.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0811.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0812.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0813.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0814.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0815.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0816.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0817.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0818.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0819.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0820.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0821.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0822.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0823.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0824.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0825.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0826.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0827.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0828.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0829.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0830.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0831.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0832.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0833.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0834.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0835.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0836.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0837.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0838.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0839.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0840.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0841.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0842.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0843.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0844.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0845.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0846.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0847.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0848.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0849.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0850.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0851.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0852.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0853.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0854.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0855.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0856.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0857.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0858.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0859.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0860.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0861.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0862.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0863.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0864.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0865.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0866.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0867.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0868.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0869.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0870.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0871.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0872.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0873.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0874.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0875.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0876.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0877.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0878.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0879.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0880.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0881.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0882.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0883.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0884.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0885.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0886.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0887.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0888.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0889.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0890.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0891.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0892.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0893.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0894.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0895.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0896.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0897.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0898.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0899.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0900.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0901.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0902.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0903.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0904.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0905.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0906.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0907.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0908.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0909.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0910.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0911.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0912.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0913.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0914.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0915.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0916.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0917.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0918.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0919.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0920.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0921.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0922.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0923.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0924.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0925.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0926.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0927.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0928.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0929.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0930.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0931.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0932.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0933.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0934.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0935.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0936.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0937.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0938.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0939.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0940.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0941.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0942.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0943.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0944.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0945.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0946.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0947.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0948.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0949.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0950.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0951.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0952.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0953.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0954.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0955.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0956.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0957.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0958.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0959.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0960.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0961.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0962.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0963.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0964.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0965.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0966.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0967.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0968.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0969.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0970.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0971.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0972.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0973.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0974.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0975.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0976.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0977.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0978.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0979.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0980.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0981.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0982.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0983.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0984.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0985.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0986.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0987.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0988.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0989.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0990.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0991.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0992.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0993.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0994.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0995.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0996.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0997.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0998.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n0999.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1000.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1001.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1002.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1003.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1004.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1005.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1006.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1007.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1008.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1009.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1010.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1011.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1012.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1013.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1014.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1015.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1016.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1017.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1018.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1019.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1020.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1021.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1022.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1023.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1024.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1025.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1026.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1027.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1028.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1029.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1030.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1031.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1032.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1033.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1034.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1035.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1036.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1037.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1038.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1039.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1040.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1041.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1042.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1043.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1044.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1045.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1046.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1047.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1048.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1049.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1050.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1051.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1052.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1053.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1054.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1055.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1056.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1057.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1058.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1059.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1060.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1061.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1062.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1063.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1064.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1065.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1066.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1067.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1068.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1069.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1070.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1071.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1072.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1073.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1074.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1075.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1076.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1077.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1078.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1079.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1080.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1081.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1082.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1083.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1084.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1085.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1086.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1087.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1088.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1089.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1090.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1091.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1092.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1093.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1094.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1095.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1096.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1097.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1098.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1099.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1100.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1101.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1102.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1103.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1104.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1105.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1106.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1107.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1108.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1109.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1110.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1111.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1112.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1113.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1114.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1115.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1116.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1117.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1118.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1119.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1120.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1121.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1122.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1123.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1124.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1125.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1126.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1127.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1128.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1129.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1130.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1131.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1132.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1133.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1134.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1135.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1136.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1137.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1138.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1139.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1140.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1141.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1142.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1143.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1144.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1145.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1146.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1147.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1148.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1149.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1150.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1151.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1152.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1153.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1154.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1155.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1156.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1157.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1158.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1159.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1160.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1161.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1162.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1163.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1164.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1165.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1166.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1167.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1168.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1169.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1170.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1171.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1172.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1173.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1174.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1175.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1176.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1177.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1178.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1179.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1180.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1181.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1182.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1183.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1184.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1185.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1186.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1187.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1188.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1189.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1190.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1191.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1192.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1193.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1194.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1195.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1196.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1197.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1198.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1199.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1200.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1201.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1202.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1203.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1204.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1205.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1206.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1207.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1208.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1209.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1210.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1211.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1212.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1213.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1214.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1215.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1216.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1217.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1218.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1219.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1220.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1221.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1222.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1223.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1224.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1225.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1226.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1227.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1228.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1229.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1230.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1231.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1232.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1233.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1234.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1235.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1236.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1237.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1238.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1239.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1240.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1241.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1242.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1243.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1244.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1245.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1246.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1247.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1248.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1249.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1250.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1251.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1252.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1253.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1254.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1255.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1256.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1257.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1258.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1259.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1260.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1261.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1262.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1263.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1264.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1265.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1266.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1267.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1268.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1269.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1270.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1271.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1272.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1273.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1274.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1275.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1276.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1277.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1278.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1279.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1280.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1281.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1282.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1283.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1284.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1285.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1286.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1287.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1288.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1289.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1290.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1291.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1292.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1293.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1294.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1295.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1296.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1297.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1298.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1299.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1300.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1301.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1302.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1303.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1304.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1305.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1306.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1307.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1308.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1309.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1310.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1311.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1312.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1313.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1314.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1315.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1316.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1317.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1318.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1319.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1320.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1321.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1322.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1323.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1324.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1325.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1326.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1327.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1328.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1329.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1330.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1331.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1332.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1333.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1334.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1335.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1336.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1337.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1338.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1339.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1340.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1341.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1342.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1343.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1344.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1345.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1346.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1347.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1348.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1349.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1350.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1351.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1352.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1353.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1354.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1355.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1356.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1357.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1358.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1359.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1360.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1361.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1362.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1363.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1364.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1365.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1366.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1367.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1368.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1369.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1370.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1371.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1372.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1373.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1374.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1375.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1376.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1377.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1378.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1379.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1380.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1381.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1382.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1383.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1384.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1385.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1386.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1387.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1388.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1389.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1390.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1391.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1392.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1393.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1394.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1395.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1396.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1397.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1398.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1399.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1400.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1401.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1402.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1403.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1404.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1405.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1406.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1407.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1408.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1409.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1410.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1411.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1412.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1413.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1414.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1415.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1416.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1417.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1418.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1419.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1420.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1421.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1422.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1423.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1424.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1425.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1426.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1427.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1428.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1429.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1430.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1431.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1432.xml.gz
+hdfs://nameservice1/data/scholix/input/baseline/pubmed21n1433.xml.gz
\ No newline at end of file

From 9646b9fd98b2e575dbe2bf258ccf708ad4c4be45 Mon Sep 17 00:00:00 2001
From: miconis <m.debonis991@gmail.com>
Date: Thu, 7 Oct 2021 11:29:11 +0200
Subject: [PATCH 02/12] implementation of the http call for the update of
 openorgs suggestions

---
 .../dhp/oa/dedup/UpdateOpenorgsJob.java       | 115 ++++++++++++++++++
 .../oa/dedup/openorgs/oozie_app/workflow.xml  |  20 +--
 .../dedup/updateOpenorgsJob_parameters.json   |  14 +++
 3 files changed, 141 insertions(+), 8 deletions(-)
 create mode 100644 dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java
 create mode 100644 dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json

diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java
new file mode 100644
index 000000000..f23c54e2e
--- /dev/null
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java
@@ -0,0 +1,115 @@
+package eu.dnetlib.dhp.oa.dedup;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+public class UpdateOpenorgsJob {
+
+    private static final Logger log = LoggerFactory.getLogger(UpdateOpenorgsJob.class);
+
+	public static void main(String[] args) throws Exception {
+	    ArgumentApplicationParser parser = new ArgumentApplicationParser(
+                    IOUtils.toString(SparkCreateSimRels.class.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
+            parser.parseArgument(args);
+
+		final String apiUrl = parser.get("apiUrl");
+		final int delay = Integer.parseInt(parser.get("delay"));
+
+		log.info("apiUrl: '{}'", apiUrl);
+		log.info("delay: '{}'", delay);
+
+		APIResponse res = httpCall(apiUrl);
+		while(res!=null && res.getStatus().equals(ImportStatus.RUNNING)){
+			TimeUnit.MINUTES.sleep(delay);
+			res = httpCall(apiUrl + "/status");
+		}
+
+		if (res==null) {
+			log.error("Openorgs Update FAILED: No response");
+			throw new RuntimeException("Openorgs Update FAILED: No response");
+		}
+
+		if (res.getStatus()==null || !res.getStatus().equals(ImportStatus.SUCCESS)) {
+			log.error("Openorgs Update FAILED: '{}' - '{}'", res.getStatus(), res.getMessage());
+			throw new RuntimeException(res.getMessage());
+		}
+
+	}
+
+	private static APIResponse httpCall(final String url) throws Exception {
+		final HttpGet req = new HttpGet(url);
+
+		try (final CloseableHttpClient client = HttpClients.createDefault()) {
+			try (final CloseableHttpResponse response = client.execute(req)) {
+				final String s = IOUtils.toString(response.getEntity().getContent());
+				return (new ObjectMapper()).readValue(s, APIResponse.class);
+			}
+		}
+	}
+
+}
+
+class APIResponse {
+	private String id;
+	private Long dateStart;
+	private Long dateEnd;
+	private ImportStatus status;
+	private String message;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public Long getDateStart() {
+		return dateStart;
+	}
+
+	public void setDateStart(Long dateStart) {
+		this.dateStart = dateStart;
+	}
+
+	public Long getDateEnd() {
+		return dateEnd;
+	}
+
+	public void setDateEnd(Long dateEnd) {
+		this.dateEnd = dateEnd;
+	}
+
+	public ImportStatus getStatus() {
+		return status;
+	}
+
+	public void setStatus(ImportStatus status) {
+		this.status = status;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(String message) {
+		this.message = message;
+	}
+}
+
+enum ImportStatus {
+	SUCCESS,
+	FAILED,
+	RUNNING,
+	NOT_LAUNCHED,
+	NOT_YET_STARTED
+}
diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml
index 30442406c..c7c6c9257 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml
@@ -28,6 +28,11 @@
             <name>dbPwd</name>
             <description>password to access the OpenOrgs database</description>
         </property>
+        <property>
+            <name>dbConnections</name>
+            <value>10</value>
+            <description>number of connections to the postgres db</description>
+        </property>
         <property>
             <name>workingPath</name>
             <description>path for the working directory</description>
@@ -223,7 +228,7 @@
             <arg>--dbTable</arg><arg>${dbTable}</arg>
             <arg>--dbUser</arg><arg>${dbUser}</arg>
             <arg>--dbPwd</arg><arg>${dbPwd}</arg>
-            <arg>--numConnections</arg><arg>20</arg>
+            <arg>--numConnections</arg><arg>${dbConnections}</arg>
         </spark>
         <ok to="PrepareNewOrgs"/>
         <error to="Kill"/>
@@ -254,19 +259,18 @@
             <arg>--dbTable</arg><arg>${dbTable}</arg>
             <arg>--dbUser</arg><arg>${dbUser}</arg>
             <arg>--dbPwd</arg><arg>${dbPwd}</arg>
-            <arg>--numConnections</arg><arg>20</arg>
+            <arg>--numConnections</arg><arg>${dbConnections}</arg>
         </spark>
         <ok to="update_openorgs"/>
         <error to="Kill"/>
     </action>
 
     <action name="update_openorgs">
-        <shell xmlns="uri:oozie:shell-action:0.1">
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <exec>/usr/bin/curl</exec>
-            <argument>${apiUrl}</argument>
-        </shell>
+        <java>
+            <main-class>eu.dnetlib.dhp.oa.dedup.UpdateOpenorgsJob</main-class>
+            <arg>--apiUrl</arg><arg>${apiUrl}</arg>
+            <arg>--delay</arg><arg>5</arg>
+        </java>
         <ok to="End"/>
         <error to="Kill"/>
     </action>
diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json
new file mode 100644
index 000000000..5ca4a3dba
--- /dev/null
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json
@@ -0,0 +1,14 @@
+[
+  {
+    "paramName": "api",
+    "paramLongName": "apiUrl",
+    "paramDescription": "the url of the API",
+    "paramRequired": true
+  },
+  {
+    "paramName": "d",
+    "paramLongName": "delay",
+    "paramDescription": "delay for the HTTP call in minutes",
+    "paramRequired": true
+  }
+]
\ No newline at end of file

From 611ca511db42d8f07e3c21e7526a19d0755a57b3 Mon Sep 17 00:00:00 2001
From: miconis <m.debonis991@gmail.com>
Date: Thu, 7 Oct 2021 15:39:55 +0200
Subject: [PATCH 03/12] set configuration property in openorgs duplicates wf

---
 .../dhp/oa/dedup/SparkWhitelistSimRels.java   |  201 +--
 .../dhp/oa/dedup/UpdateOpenorgsJob.java       |   32 +-
 .../oa/dedup/openorgs/oozie_app/workflow.xml  |    6 +
 .../dnetlib/dhp/oa/dedup/SparkDedupTest.java  | 1266 +++++++++--------
 4 files changed, 761 insertions(+), 744 deletions(-)

diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java
index fa7d33570..7d91e47cc 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java
@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.oa.dedup;
 
 import java.io.IOException;
@@ -5,10 +6,7 @@ import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.FilterFunction;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.api.java.function.PairFunction;
 import org.apache.spark.sql.Dataset;
@@ -21,7 +19,6 @@ import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
-import eu.dnetlib.dhp.oa.dedup.model.Block;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.utils.ISLookupClientFactory;
@@ -31,121 +28,127 @@ import eu.dnetlib.pace.config.DedupConfig;
 import eu.dnetlib.pace.model.MapDocument;
 import eu.dnetlib.pace.util.MapDocumentUtil;
 import scala.Tuple2;
-import scala.Tuple3;
 
 public class SparkWhitelistSimRels extends AbstractSparkAction {
 
-    private static final Logger log = LoggerFactory.getLogger(SparkCreateSimRels.class);
+	private static final Logger log = LoggerFactory.getLogger(SparkCreateSimRels.class);
 
-    private static final String WHITELIST_SEPARATOR = "####";
+	private static final String WHITELIST_SEPARATOR = "####";
 
-    public SparkWhitelistSimRels(ArgumentApplicationParser parser, SparkSession spark) {
-        super(parser, spark);
-    }
+	public SparkWhitelistSimRels(ArgumentApplicationParser parser, SparkSession spark) {
+		super(parser, spark);
+	}
 
-    public static void main(String[] args) throws Exception {
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkCreateSimRels.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/whitelistSimRels_parameters.json")));
-        parser.parseArgument(args);
+	public static void main(String[] args) throws Exception {
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateSimRels.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/whitelistSimRels_parameters.json")));
+		parser.parseArgument(args);
 
-        SparkConf conf = new SparkConf();
-        new SparkWhitelistSimRels(parser, getSparkSession(conf))
-                .run(ISLookupClientFactory.getLookUpService(parser.get("isLookUpUrl")));
-    }
+		SparkConf conf = new SparkConf();
+		new SparkWhitelistSimRels(parser, getSparkSession(conf))
+			.run(ISLookupClientFactory.getLookUpService(parser.get("isLookUpUrl")));
+	}
 
-    @Override
-    public void run(ISLookUpService isLookUpService)
-            throws DocumentException, IOException, ISLookUpException, SAXException {
+	@Override
+	public void run(ISLookUpService isLookUpService)
+		throws DocumentException, IOException, ISLookUpException, SAXException {
 
-        // read oozie parameters
-        final String graphBasePath = parser.get("graphBasePath");
-        final String isLookUpUrl = parser.get("isLookUpUrl");
-        final String actionSetId = parser.get("actionSetId");
-        final String workingPath = parser.get("workingPath");
-        final int numPartitions = Optional
-                .ofNullable(parser.get("numPartitions"))
-                .map(Integer::valueOf)
-                .orElse(NUM_PARTITIONS);
-        final String whiteListPath = parser.get("whiteListPath");
+		// read oozie parameters
+		final String graphBasePath = parser.get("graphBasePath");
+		final String isLookUpUrl = parser.get("isLookUpUrl");
+		final String actionSetId = parser.get("actionSetId");
+		final String workingPath = parser.get("workingPath");
+		final int numPartitions = Optional
+			.ofNullable(parser.get("numPartitions"))
+			.map(Integer::valueOf)
+			.orElse(NUM_PARTITIONS);
+		final String whiteListPath = parser.get("whiteListPath");
 
-        log.info("numPartitions: '{}'", numPartitions);
-        log.info("graphBasePath: '{}'", graphBasePath);
-        log.info("isLookUpUrl:   '{}'", isLookUpUrl);
-        log.info("actionSetId:   '{}'", actionSetId);
-        log.info("workingPath:   '{}'", workingPath);
-        log.info("whiteListPath: '{}'", whiteListPath);
+		log.info("numPartitions: '{}'", numPartitions);
+		log.info("graphBasePath: '{}'", graphBasePath);
+		log.info("isLookUpUrl:   '{}'", isLookUpUrl);
+		log.info("actionSetId:   '{}'", actionSetId);
+		log.info("workingPath:   '{}'", workingPath);
+		log.info("whiteListPath: '{}'", whiteListPath);
 
-        JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+		JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
 
-        //file format: source####target
-        Dataset<Tuple2<String, String>> whiteListRels = spark.createDataset(sc
-                        .textFile(whiteListPath)
-                        //check if the line is in the correct format: id1####id2
-                        .filter(s -> s.contains(WHITELIST_SEPARATOR) && s.split(WHITELIST_SEPARATOR).length == 2)
-                        .map(s -> new Tuple2<>(s.split(WHITELIST_SEPARATOR)[0], s.split(WHITELIST_SEPARATOR)[1]))
-                        .rdd(),
-                Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+		// file format: source####target
+		Dataset<Tuple2<String, String>> whiteListRels = spark
+			.createDataset(
+				sc
+					.textFile(whiteListPath)
+					// check if the line is in the correct format: id1####id2
+					.filter(s -> s.contains(WHITELIST_SEPARATOR) && s.split(WHITELIST_SEPARATOR).length == 2)
+					.map(s -> new Tuple2<>(s.split(WHITELIST_SEPARATOR)[0], s.split(WHITELIST_SEPARATOR)[1]))
+					.rdd(),
+				Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
-        // for each dedup configuration
-        for (DedupConfig dedupConf : getConfigurations(isLookUpService, actionSetId)) {
+		// for each dedup configuration
+		for (DedupConfig dedupConf : getConfigurations(isLookUpService, actionSetId)) {
 
-            final String entity = dedupConf.getWf().getEntityType();
-            final String subEntity = dedupConf.getWf().getSubEntityValue();
-            log.info("Adding whitelist simrels for: '{}'", subEntity);
+			final String entity = dedupConf.getWf().getEntityType();
+			final String subEntity = dedupConf.getWf().getSubEntityValue();
+			log.info("Adding whitelist simrels for: '{}'", subEntity);
 
-            final String outputPath = DedupUtility.createSimRelPath(workingPath, actionSetId, subEntity);
+			final String outputPath = DedupUtility.createSimRelPath(workingPath, actionSetId, subEntity);
 
-            Dataset<Tuple2<String, String>> entities = spark.createDataset(sc
-                    .textFile(DedupUtility.createEntityPath(graphBasePath, subEntity))
-                    .repartition(numPartitions)
-                    .mapToPair(
-                            (PairFunction<String, String, String>) s -> {
-                                MapDocument d = MapDocumentUtil.asMapDocumentWithJPath(dedupConf, s);
-                                return new Tuple2<>(d.getIdentifier(), "present");
-                            })
-                    .rdd(),
-                    Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+			Dataset<Tuple2<String, String>> entities = spark
+				.createDataset(
+					sc
+						.textFile(DedupUtility.createEntityPath(graphBasePath, subEntity))
+						.repartition(numPartitions)
+						.mapToPair(
+							(PairFunction<String, String, String>) s -> {
+								MapDocument d = MapDocumentUtil.asMapDocumentWithJPath(dedupConf, s);
+								return new Tuple2<>(d.getIdentifier(), "present");
+							})
+						.rdd(),
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
-            Dataset<Tuple2<String, String>> whiteListRels1 = whiteListRels
-                    .joinWith(entities, whiteListRels.col("_1").equalTo(entities.col("_1")), "inner")
-                    .map((MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1, Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+			Dataset<Tuple2<String, String>> whiteListRels1 = whiteListRels
+				.joinWith(entities, whiteListRels.col("_1").equalTo(entities.col("_1")), "inner")
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
-            Dataset<Tuple2<String, String>> whiteListRels2 = whiteListRels1
-                    .joinWith(entities, whiteListRels1.col("_2").equalTo(entities.col("_1")), "inner")
-                    .map((MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1, Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+			Dataset<Tuple2<String, String>> whiteListRels2 = whiteListRels1
+				.joinWith(entities, whiteListRels1.col("_2").equalTo(entities.col("_1")), "inner")
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
 
-            Dataset<Relation> whiteListSimRels = whiteListRels2
-                    .map((MapFunction<Tuple2<String, String>, Relation>)
-                                    r -> createSimRel(r._1(), r._2(), entity),
-                                    Encoders.bean(Relation.class)
-                    );
+			Dataset<Relation> whiteListSimRels = whiteListRels2
+				.map(
+					(MapFunction<Tuple2<String, String>, Relation>) r -> createSimRel(r._1(), r._2(), entity),
+					Encoders.bean(Relation.class));
 
-            saveParquet(whiteListSimRels, outputPath, SaveMode.Append);
-        }
-    }
+			saveParquet(whiteListSimRels, outputPath, SaveMode.Append);
+		}
+	}
 
-    private Relation createSimRel(String source, String target, String entity) {
-        final Relation r = new Relation();
-        r.setSource(source);
-        r.setTarget(target);
-        r.setSubRelType("dedupSimilarity");
-        r.setRelClass("isSimilarTo");
-        r.setDataInfo(new DataInfo());
+	private Relation createSimRel(String source, String target, String entity) {
+		final Relation r = new Relation();
+		r.setSource(source);
+		r.setTarget(target);
+		r.setSubRelType("dedupSimilarity");
+		r.setRelClass("isSimilarTo");
+		r.setDataInfo(new DataInfo());
 
-        switch (entity) {
-            case "result":
-                r.setRelType("resultResult");
-                break;
-            case "organization":
-                r.setRelType("organizationOrganization");
-                break;
-            default:
-                throw new IllegalArgumentException("unmanaged entity type: " + entity);
-        }
-        return r;
-    }
+		switch (entity) {
+			case "result":
+				r.setRelType("resultResult");
+				break;
+			case "organization":
+				r.setRelType("organizationOrganization");
+				break;
+			default:
+				throw new IllegalArgumentException("unmanaged entity type: " + entity);
+		}
+		return r;
+	}
 }
diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java
index f23c54e2e..d094fb72b 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java
@@ -1,7 +1,8 @@
+
 package eu.dnetlib.dhp.oa.dedup;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
@@ -10,16 +11,21 @@ import org.apache.http.impl.client.HttpClients;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 
 public class UpdateOpenorgsJob {
 
-    private static final Logger log = LoggerFactory.getLogger(UpdateOpenorgsJob.class);
+	private static final Logger log = LoggerFactory.getLogger(UpdateOpenorgsJob.class);
 
 	public static void main(String[] args) throws Exception {
-	    ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                    IOUtils.toString(SparkCreateSimRels.class.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
-            parser.parseArgument(args);
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateSimRels.class
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
+		parser.parseArgument(args);
 
 		final String apiUrl = parser.get("apiUrl");
 		final int delay = Integer.parseInt(parser.get("delay"));
@@ -28,17 +34,17 @@ public class UpdateOpenorgsJob {
 		log.info("delay: '{}'", delay);
 
 		APIResponse res = httpCall(apiUrl);
-		while(res!=null && res.getStatus().equals(ImportStatus.RUNNING)){
+		while (res != null && res.getStatus().equals(ImportStatus.RUNNING)) {
 			TimeUnit.MINUTES.sleep(delay);
 			res = httpCall(apiUrl + "/status");
 		}
 
-		if (res==null) {
+		if (res == null) {
 			log.error("Openorgs Update FAILED: No response");
 			throw new RuntimeException("Openorgs Update FAILED: No response");
 		}
 
-		if (res.getStatus()==null || !res.getStatus().equals(ImportStatus.SUCCESS)) {
+		if (res.getStatus() == null || !res.getStatus().equals(ImportStatus.SUCCESS)) {
 			log.error("Openorgs Update FAILED: '{}' - '{}'", res.getStatus(), res.getMessage());
 			throw new RuntimeException(res.getMessage());
 		}
@@ -107,9 +113,5 @@ class APIResponse {
 }
 
 enum ImportStatus {
-	SUCCESS,
-	FAILED,
-	RUNNING,
-	NOT_LAUNCHED,
-	NOT_YET_STARTED
+	SUCCESS, FAILED, RUNNING, NOT_LAUNCHED, NOT_YET_STARTED
 }
diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml
index c7c6c9257..6947019e8 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml
@@ -267,6 +267,12 @@
 
     <action name="update_openorgs">
         <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
             <main-class>eu.dnetlib.dhp.oa.dedup.UpdateOpenorgsJob</main-class>
             <arg>--apiUrl</arg><arg>${apiUrl}</arg>
             <arg>--delay</arg><arg>5</arg>
diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
index fa03f93a6..549988767 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
+++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
@@ -48,634 +48,640 @@ import scala.Tuple2;
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 public class SparkDedupTest implements Serializable {
 
-    @Mock(serializable = true)
-    ISLookUpService isLookUpService;
-
-    private static SparkSession spark;
-    private static JavaSparkContext jsc;
-
-    private static String testGraphBasePath;
-    private static String testOutputBasePath;
-    private static String testDedupGraphBasePath;
-    private static final String testActionSetId = "test-orchestrator";
-    private static String whitelistPath;
-    private static List<String> whiteList;
-
-    private static String WHITELIST_SEPARATOR = "####";
-
-    @BeforeAll
-    public static void cleanUp() throws IOException, URISyntaxException {
-
-        testGraphBasePath = Paths
-                .get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/entities").toURI())
-                .toFile()
-                .getAbsolutePath();
-        testOutputBasePath = createTempDirectory(SparkDedupTest.class.getSimpleName() + "-")
-                .toAbsolutePath()
-                .toString();
-
-        testDedupGraphBasePath = createTempDirectory(SparkDedupTest.class.getSimpleName() + "-")
-                .toAbsolutePath()
-                .toString();
-
-        whitelistPath = Paths
-                .get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/whitelist.simrels.txt").toURI())
-                .toFile()
-                .getAbsolutePath();
-        whiteList = IOUtils.readLines(new FileReader(whitelistPath));
-
-        FileUtils.deleteDirectory(new File(testOutputBasePath));
-        FileUtils.deleteDirectory(new File(testDedupGraphBasePath));
-
-        final SparkConf conf = new SparkConf();
-        conf.set("spark.sql.shuffle.partitions", "200");
-        spark = SparkSession
-                .builder()
-                .appName(SparkDedupTest.class.getSimpleName())
-                .master("local[*]")
-                .config(conf)
-                .getOrCreate();
-
-        jsc = JavaSparkContext.fromSparkContext(spark.sparkContext());
-
-    }
-
-    @BeforeEach
-    public void setUp() throws IOException, ISLookUpException {
-
-        lenient()
-                .when(isLookUpService.getResourceProfileByQuery(Mockito.contains(testActionSetId)))
-                .thenReturn(
-                        IOUtils
-                                .toString(
-                                        SparkDedupTest.class
-                                                .getResourceAsStream(
-                                                        "/eu/dnetlib/dhp/dedup/profiles/mock_orchestrator.xml")));
-
-        lenient()
-                .when(isLookUpService.getResourceProfileByQuery(Mockito.contains("organization")))
-                .thenReturn(
-                        IOUtils
-                                .toString(
-                                        SparkDedupTest.class
-                                                .getResourceAsStream(
-                                                        "/eu/dnetlib/dhp/dedup/conf/org.curr.conf.json")));
-
-        lenient()
-                .when(isLookUpService.getResourceProfileByQuery(Mockito.contains("publication")))
-                .thenReturn(
-                        IOUtils
-                                .toString(
-                                        SparkDedupTest.class
-                                                .getResourceAsStream(
-                                                        "/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json")));
-
-        lenient()
-                .when(isLookUpService.getResourceProfileByQuery(Mockito.contains("software")))
-                .thenReturn(
-                        IOUtils
-                                .toString(
-                                        SparkDedupTest.class
-                                                .getResourceAsStream(
-                                                        "/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json")));
-
-        lenient()
-                .when(isLookUpService.getResourceProfileByQuery(Mockito.contains("dataset")))
-                .thenReturn(
-                        IOUtils
-                                .toString(
-                                        SparkDedupTest.class
-                                                .getResourceAsStream(
-                                                        "/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json")));
-
-        lenient()
-                .when(isLookUpService.getResourceProfileByQuery(Mockito.contains("otherresearchproduct")))
-                .thenReturn(
-                        IOUtils
-                                .toString(
-                                        SparkDedupTest.class
-                                                .getResourceAsStream(
-                                                        "/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json")));
-    }
-
-    @Test
-    @Order(1)
-    void createSimRelsTest() throws Exception {
-
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkCreateSimRels.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/createSimRels_parameters.json")));
-
-        parser
-                .parseArgument(
-                        new String[]{
-                                "-i", testGraphBasePath,
-                                "-asi", testActionSetId,
-                                "-la", "lookupurl",
-                                "-w", testOutputBasePath,
-                                "-np", "50"
-                        });
-
-        new SparkCreateSimRels(parser, spark).run(isLookUpService);
-
-        long orgs_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization"))
-                .count();
-
-        long pubs_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "publication"))
-                .count();
-
-        long sw_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"))
-                .count();
-
-        long ds_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "dataset"))
-                .count();
-
-        long orp_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
-                .count();
-
-        assertEquals(3082, orgs_simrel);
-        assertEquals(7036, pubs_simrel);
-        assertEquals(336, sw_simrel);
-        assertEquals(442, ds_simrel);
-        assertEquals(6750, orp_simrel);
-    }
-
-    @Test
-    @Order(2)
-    void whitelistSimRelsTest() throws Exception {
-
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkWhitelistSimRels.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/whitelistSimRels_parameters.json")));
-
-        parser
-                .parseArgument(
-                        new String[]{
-                                "-i", testGraphBasePath,
-                                "-asi", testActionSetId,
-                                "-la", "lookupurl",
-                                "-w", testOutputBasePath,
-                                "-np", "50",
-                                "-wl", whitelistPath
-                        });
-
-        new SparkWhitelistSimRels(parser, spark).run(isLookUpService);
-
-        long orgs_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization"))
-                .count();
-
-        long pubs_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "publication"))
-                .count();
-
-        long ds_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "dataset"))
-                .count();
-
-        long orp_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
-                .count();
-
-        //entities simrels supposed to be equal to the number of previous step (no rels in whitelist)
-        assertEquals(3082, orgs_simrel);
-        assertEquals(7036, pubs_simrel);
-        assertEquals(442, ds_simrel);
-        assertEquals(6750, orp_simrel);
-
-        //entities simrels to be different from the number of previous step (new simrels in the whitelist)
-        Dataset<Row> sw_simrel = spark
-                .read()
-                .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"));
-
-        //check if the first relation in the whitelist exists
-        assertTrue(sw_simrel
-                .as(Encoders.bean(Relation.class))
-                .toJavaRDD()
-                .filter(rel ->
-                        rel.getSource().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[0]) && rel.getTarget().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[1])).count() > 0);
-        //check if the second relation in the whitelist exists
-        assertTrue(sw_simrel
-                .as(Encoders.bean(Relation.class))
-                .toJavaRDD()
-                .filter(rel ->
-                        rel.getSource().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[0]) && rel.getTarget().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[1])).count() > 0);
-
-        assertEquals(338, sw_simrel.count());
-
-    }
-
-    @Test
-    @Order(3)
-    void cutMergeRelsTest() throws Exception {
-
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkCreateMergeRels.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
-
-        parser
-                .parseArgument(
-                        new String[]{
-                                "-i",
-                                testGraphBasePath,
-                                "-asi",
-                                testActionSetId,
-                                "-la",
-                                "lookupurl",
-                                "-w",
-                                testOutputBasePath,
-                                "-cc",
-                                "3"
-                        });
-
-        new SparkCreateMergeRels(parser, spark).run(isLookUpService);
-
-        long orgs_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
-                .groupBy("source")
-                .agg(count("target").alias("cnt"))
-                .select("source", "cnt")
-                .where("cnt > 3")
-                .count();
-
-        long pubs_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
-                .groupBy("source")
-                .agg(count("target").alias("cnt"))
-                .select("source", "cnt")
-                .where("cnt > 3")
-                .count();
-        long sw_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
-                .groupBy("source")
-                .agg(count("target").alias("cnt"))
-                .select("source", "cnt")
-                .where("cnt > 3")
-                .count();
-
-        long ds_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
-                .groupBy("source")
-                .agg(count("target").alias("cnt"))
-                .select("source", "cnt")
-                .where("cnt > 3")
-                .count();
-
-        long orp_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
-                .groupBy("source")
-                .agg(count("target").alias("cnt"))
-                .select("source", "cnt")
-                .where("cnt > 3")
-                .count();
-
-        assertEquals(0, orgs_mergerel);
-        assertEquals(0, pubs_mergerel);
-        assertEquals(0, sw_mergerel);
-        assertEquals(0, ds_mergerel);
-        assertEquals(0, orp_mergerel);
-
-        FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel"));
-        FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel"));
-        FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/software_mergerel"));
-        FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel"));
-        FileUtils
-                .deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel"));
-    }
-
-    @Test
-    @Order(4)
-    void createMergeRelsTest() throws Exception {
-
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkCreateMergeRels.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
-
-        parser
-                .parseArgument(
-                        new String[]{
-                                "-i",
-                                testGraphBasePath,
-                                "-asi",
-                                testActionSetId,
-                                "-la",
-                                "lookupurl",
-                                "-w",
-                                testOutputBasePath
-                        });
-
-        new SparkCreateMergeRels(parser, spark).run(isLookUpService);
-
-        long orgs_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
-                .count();
-        long pubs_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
-                .count();
-        long sw_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
-                .count();
-        long ds_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
-                .count();
-
-        long orp_mergerel = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
-                .count();
-
-        assertEquals(1272, orgs_mergerel);
-        assertEquals(1438, pubs_mergerel);
-        assertEquals(286, sw_mergerel);
-        assertEquals(472, ds_mergerel);
-        assertEquals(718, orp_mergerel);
-
-    }
-
-    @Test
-    @Order(5)
-    void createDedupRecordTest() throws Exception {
-
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkCreateDedupRecord.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/createDedupRecord_parameters.json")));
-        parser
-                .parseArgument(
-                        new String[]{
-                                "-i",
-                                testGraphBasePath,
-                                "-asi",
-                                testActionSetId,
-                                "-la",
-                                "lookupurl",
-                                "-w",
-                                testOutputBasePath
-                        });
-
-        new SparkCreateDedupRecord(parser, spark).run(isLookUpService);
-
-        long orgs_deduprecord = jsc
-                .textFile(testOutputBasePath + "/" + testActionSetId + "/organization_deduprecord")
-                .count();
-        long pubs_deduprecord = jsc
-                .textFile(testOutputBasePath + "/" + testActionSetId + "/publication_deduprecord")
-                .count();
-        long sw_deduprecord = jsc
-                .textFile(testOutputBasePath + "/" + testActionSetId + "/software_deduprecord")
-                .count();
-        long ds_deduprecord = jsc.textFile(testOutputBasePath + "/" + testActionSetId + "/dataset_deduprecord").count();
-        long orp_deduprecord = jsc
-                .textFile(
-                        testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_deduprecord")
-                .count();
-
-        assertEquals(85, orgs_deduprecord);
-        assertEquals(65, pubs_deduprecord);
-        assertEquals(49, sw_deduprecord);
-        assertEquals(97, ds_deduprecord);
-        assertEquals(89, orp_deduprecord);
-    }
-
-    @Test
-    @Order(6)
-    void updateEntityTest() throws Exception {
-
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkUpdateEntity.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json")));
-        parser
-                .parseArgument(
-                        new String[]{
-                                "-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
-                        });
-
-        new SparkUpdateEntity(parser, spark).run(isLookUpService);
-
-        long organizations = jsc.textFile(testDedupGraphBasePath + "/organization").count();
-        long publications = jsc.textFile(testDedupGraphBasePath + "/publication").count();
-        long projects = jsc.textFile(testDedupGraphBasePath + "/project").count();
-        long datasource = jsc.textFile(testDedupGraphBasePath + "/datasource").count();
-        long softwares = jsc.textFile(testDedupGraphBasePath + "/software").count();
-        long dataset = jsc.textFile(testDedupGraphBasePath + "/dataset").count();
-        long otherresearchproduct = jsc.textFile(testDedupGraphBasePath + "/otherresearchproduct").count();
-
-        long mergedOrgs = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .where("relClass=='merges'")
-                .javaRDD()
-                .map(Relation::getTarget)
-                .distinct()
-                .count();
-
-        long mergedPubs = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .where("relClass=='merges'")
-                .javaRDD()
-                .map(Relation::getTarget)
-                .distinct()
-                .count();
-
-        long mergedSw = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .where("relClass=='merges'")
-                .javaRDD()
-                .map(Relation::getTarget)
-                .distinct()
-                .count();
-
-        long mergedDs = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .where("relClass=='merges'")
-                .javaRDD()
-                .map(Relation::getTarget)
-                .distinct()
-                .count();
-
-        long mergedOrp = spark
-                .read()
-                .load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
-                .as(Encoders.bean(Relation.class))
-                .where("relClass=='merges'")
-                .javaRDD()
-                .map(Relation::getTarget)
-                .distinct()
-                .count();
-
-        assertEquals(896, publications);
-        assertEquals(838, organizations);
-        assertEquals(100, projects);
-        assertEquals(100, datasource);
-        assertEquals(198, softwares);
-        assertEquals(389, dataset);
-        assertEquals(517, otherresearchproduct);
-
-        long deletedOrgs = jsc
-                .textFile(testDedupGraphBasePath + "/organization")
-                .filter(this::isDeletedByInference)
-                .count();
-
-        long deletedPubs = jsc
-                .textFile(testDedupGraphBasePath + "/publication")
-                .filter(this::isDeletedByInference)
-                .count();
-
-        long deletedSw = jsc
-                .textFile(testDedupGraphBasePath + "/software")
-                .filter(this::isDeletedByInference)
-                .count();
-
-        long deletedDs = jsc
-                .textFile(testDedupGraphBasePath + "/dataset")
-                .filter(this::isDeletedByInference)
-                .count();
-
-        long deletedOrp = jsc
-                .textFile(testDedupGraphBasePath + "/otherresearchproduct")
-                .filter(this::isDeletedByInference)
-                .count();
-
-        assertEquals(mergedOrgs, deletedOrgs);
-        assertEquals(mergedPubs, deletedPubs);
-        assertEquals(mergedSw, deletedSw);
-        assertEquals(mergedDs, deletedDs);
-        assertEquals(mergedOrp, deletedOrp);
-    }
-
-    @Test
-    @Order(7)
-    void propagateRelationTest() throws Exception {
-
-        ArgumentApplicationParser parser = new ArgumentApplicationParser(
-                IOUtils
-                        .toString(
-                                SparkPropagateRelation.class
-                                        .getResourceAsStream(
-                                                "/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")));
-        parser
-                .parseArgument(
-                        new String[]{
-                                "-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
-                        });
-
-        new SparkPropagateRelation(parser, spark).run(isLookUpService);
-
-        long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
-
-        assertEquals(4860, relations);
-
-        // check deletedbyinference
-        final Dataset<Relation> mergeRels = spark
-                .read()
-                .load(DedupUtility.createMergeRelPath(testOutputBasePath, "*", "*"))
-                .as(Encoders.bean(Relation.class));
-        final JavaPairRDD<String, String> mergedIds = mergeRels
-                .where("relClass == 'merges'")
-                .select(mergeRels.col("target"))
-                .distinct()
-                .toJavaRDD()
-                .mapToPair(
-                        (PairFunction<Row, String, String>) r -> new Tuple2<String, String>(r.getString(0), "d"));
-
-        JavaRDD<String> toCheck = jsc
-                .textFile(testDedupGraphBasePath + "/relation")
-                .mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.source", json), json))
-                .join(mergedIds)
-                .map(t -> t._2()._1())
-                .mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.target", json), json))
-                .join(mergedIds)
-                .map(t -> t._2()._1());
-
-        long deletedbyinference = toCheck.filter(this::isDeletedByInference).count();
-        long updated = toCheck.count();
-
-        assertEquals(updated, deletedbyinference);
-    }
-
-    @Test
-    @Order(8)
-    void testRelations() throws Exception {
-        testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_1.json", 12, 10);
-        testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_2.json", 10, 2);
-    }
-
-    private void testUniqueness(String path, int expected_total, int expected_unique) {
-        Dataset<Relation> rel = spark
-                .read()
-                .textFile(getClass().getResource(path).getPath())
-                .map(
-                        (MapFunction<String, Relation>) s -> new ObjectMapper().readValue(s, Relation.class),
-                        Encoders.bean(Relation.class));
-
-        assertEquals(expected_total, rel.count());
-        assertEquals(expected_unique, rel.distinct().count());
-    }
-
-    @AfterAll
-    public static void finalCleanUp() throws IOException {
-        FileUtils.deleteDirectory(new File(testOutputBasePath));
-        FileUtils.deleteDirectory(new File(testDedupGraphBasePath));
-    }
-
-    public boolean isDeletedByInference(String s) {
-        return s.contains("\"deletedbyinference\":true");
-    }
+	@Mock(serializable = true)
+	ISLookUpService isLookUpService;
+
+	private static SparkSession spark;
+	private static JavaSparkContext jsc;
+
+	private static String testGraphBasePath;
+	private static String testOutputBasePath;
+	private static String testDedupGraphBasePath;
+	private static final String testActionSetId = "test-orchestrator";
+	private static String whitelistPath;
+	private static List<String> whiteList;
+
+	private static String WHITELIST_SEPARATOR = "####";
+
+	@BeforeAll
+	public static void cleanUp() throws IOException, URISyntaxException {
+
+		testGraphBasePath = Paths
+			.get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/entities").toURI())
+			.toFile()
+			.getAbsolutePath();
+		testOutputBasePath = createTempDirectory(SparkDedupTest.class.getSimpleName() + "-")
+			.toAbsolutePath()
+			.toString();
+
+		testDedupGraphBasePath = createTempDirectory(SparkDedupTest.class.getSimpleName() + "-")
+			.toAbsolutePath()
+			.toString();
+
+		whitelistPath = Paths
+			.get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/whitelist.simrels.txt").toURI())
+			.toFile()
+			.getAbsolutePath();
+		whiteList = IOUtils.readLines(new FileReader(whitelistPath));
+
+		FileUtils.deleteDirectory(new File(testOutputBasePath));
+		FileUtils.deleteDirectory(new File(testDedupGraphBasePath));
+
+		final SparkConf conf = new SparkConf();
+		conf.set("spark.sql.shuffle.partitions", "200");
+		spark = SparkSession
+			.builder()
+			.appName(SparkDedupTest.class.getSimpleName())
+			.master("local[*]")
+			.config(conf)
+			.getOrCreate();
+
+		jsc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+	}
+
+	@BeforeEach
+	public void setUp() throws IOException, ISLookUpException {
+
+		lenient()
+			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains(testActionSetId)))
+			.thenReturn(
+				IOUtils
+					.toString(
+						SparkDedupTest.class
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/profiles/mock_orchestrator.xml")));
+
+		lenient()
+			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("organization")))
+			.thenReturn(
+				IOUtils
+					.toString(
+						SparkDedupTest.class
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/org.curr.conf.json")));
+
+		lenient()
+			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("publication")))
+			.thenReturn(
+				IOUtils
+					.toString(
+						SparkDedupTest.class
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json")));
+
+		lenient()
+			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("software")))
+			.thenReturn(
+				IOUtils
+					.toString(
+						SparkDedupTest.class
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json")));
+
+		lenient()
+			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("dataset")))
+			.thenReturn(
+				IOUtils
+					.toString(
+						SparkDedupTest.class
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json")));
+
+		lenient()
+			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("otherresearchproduct")))
+			.thenReturn(
+				IOUtils
+					.toString(
+						SparkDedupTest.class
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json")));
+	}
+
+	@Test
+	@Order(1)
+	void createSimRelsTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateSimRels.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createSimRels_parameters.json")));
+
+		parser
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath,
+					"-asi", testActionSetId,
+					"-la", "lookupurl",
+					"-w", testOutputBasePath,
+					"-np", "50"
+				});
+
+		new SparkCreateSimRels(parser, spark).run(isLookUpService);
+
+		long orgs_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization"))
+			.count();
+
+		long pubs_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "publication"))
+			.count();
+
+		long sw_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"))
+			.count();
+
+		long ds_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "dataset"))
+			.count();
+
+		long orp_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
+			.count();
+
+		assertEquals(3082, orgs_simrel);
+		assertEquals(7036, pubs_simrel);
+		assertEquals(336, sw_simrel);
+		assertEquals(442, ds_simrel);
+		assertEquals(6750, orp_simrel);
+	}
+
+	@Test
+	@Order(2)
+	void whitelistSimRelsTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkWhitelistSimRels.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/whitelistSimRels_parameters.json")));
+
+		parser
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath,
+					"-asi", testActionSetId,
+					"-la", "lookupurl",
+					"-w", testOutputBasePath,
+					"-np", "50",
+					"-wl", whitelistPath
+				});
+
+		new SparkWhitelistSimRels(parser, spark).run(isLookUpService);
+
+		long orgs_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization"))
+			.count();
+
+		long pubs_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "publication"))
+			.count();
+
+		long ds_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "dataset"))
+			.count();
+
+		long orp_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
+			.count();
+
+		// entities simrels supposed to be equal to the number of previous step (no rels in whitelist)
+		assertEquals(3082, orgs_simrel);
+		assertEquals(7036, pubs_simrel);
+		assertEquals(442, ds_simrel);
+		assertEquals(6750, orp_simrel);
+
+		// entities simrels to be different from the number of previous step (new simrels in the whitelist)
+		Dataset<Row> sw_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"));
+
+		// check if the first relation in the whitelist exists
+		assertTrue(
+			sw_simrel
+				.as(Encoders.bean(Relation.class))
+				.toJavaRDD()
+				.filter(
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[0])
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[1]))
+				.count() > 0);
+		// check if the second relation in the whitelist exists
+		assertTrue(
+			sw_simrel
+				.as(Encoders.bean(Relation.class))
+				.toJavaRDD()
+				.filter(
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[0])
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[1]))
+				.count() > 0);
+
+		assertEquals(338, sw_simrel.count());
+
+	}
+
+	@Test
+	@Order(3)
+	void cutMergeRelsTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateMergeRels.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
+
+		parser
+			.parseArgument(
+				new String[] {
+					"-i",
+					testGraphBasePath,
+					"-asi",
+					testActionSetId,
+					"-la",
+					"lookupurl",
+					"-w",
+					testOutputBasePath,
+					"-cc",
+					"3"
+				});
+
+		new SparkCreateMergeRels(parser, spark).run(isLookUpService);
+
+		long orgs_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
+			.groupBy("source")
+			.agg(count("target").alias("cnt"))
+			.select("source", "cnt")
+			.where("cnt > 3")
+			.count();
+
+		long pubs_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
+			.groupBy("source")
+			.agg(count("target").alias("cnt"))
+			.select("source", "cnt")
+			.where("cnt > 3")
+			.count();
+		long sw_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
+			.groupBy("source")
+			.agg(count("target").alias("cnt"))
+			.select("source", "cnt")
+			.where("cnt > 3")
+			.count();
+
+		long ds_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
+			.groupBy("source")
+			.agg(count("target").alias("cnt"))
+			.select("source", "cnt")
+			.where("cnt > 3")
+			.count();
+
+		long orp_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.filter((FilterFunction<Relation>) r -> r.getRelClass().equalsIgnoreCase("merges"))
+			.groupBy("source")
+			.agg(count("target").alias("cnt"))
+			.select("source", "cnt")
+			.where("cnt > 3")
+			.count();
+
+		assertEquals(0, orgs_mergerel);
+		assertEquals(0, pubs_mergerel);
+		assertEquals(0, sw_mergerel);
+		assertEquals(0, ds_mergerel);
+		assertEquals(0, orp_mergerel);
+
+		FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel"));
+		FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel"));
+		FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/software_mergerel"));
+		FileUtils.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel"));
+		FileUtils
+			.deleteDirectory(new File(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel"));
+	}
+
+	@Test
+	@Order(4)
+	void createMergeRelsTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateMergeRels.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
+
+		parser
+			.parseArgument(
+				new String[] {
+					"-i",
+					testGraphBasePath,
+					"-asi",
+					testActionSetId,
+					"-la",
+					"lookupurl",
+					"-w",
+					testOutputBasePath
+				});
+
+		new SparkCreateMergeRels(parser, spark).run(isLookUpService);
+
+		long orgs_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
+			.count();
+		long pubs_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
+			.count();
+		long sw_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
+			.count();
+		long ds_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
+			.count();
+
+		long orp_mergerel = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
+			.count();
+
+		assertEquals(1272, orgs_mergerel);
+		assertEquals(1438, pubs_mergerel);
+		assertEquals(286, sw_mergerel);
+		assertEquals(472, ds_mergerel);
+		assertEquals(718, orp_mergerel);
+
+	}
+
+	@Test
+	@Order(5)
+	void createDedupRecordTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateDedupRecord.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createDedupRecord_parameters.json")));
+		parser
+			.parseArgument(
+				new String[] {
+					"-i",
+					testGraphBasePath,
+					"-asi",
+					testActionSetId,
+					"-la",
+					"lookupurl",
+					"-w",
+					testOutputBasePath
+				});
+
+		new SparkCreateDedupRecord(parser, spark).run(isLookUpService);
+
+		long orgs_deduprecord = jsc
+			.textFile(testOutputBasePath + "/" + testActionSetId + "/organization_deduprecord")
+			.count();
+		long pubs_deduprecord = jsc
+			.textFile(testOutputBasePath + "/" + testActionSetId + "/publication_deduprecord")
+			.count();
+		long sw_deduprecord = jsc
+			.textFile(testOutputBasePath + "/" + testActionSetId + "/software_deduprecord")
+			.count();
+		long ds_deduprecord = jsc.textFile(testOutputBasePath + "/" + testActionSetId + "/dataset_deduprecord").count();
+		long orp_deduprecord = jsc
+			.textFile(
+				testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_deduprecord")
+			.count();
+
+		assertEquals(85, orgs_deduprecord);
+		assertEquals(65, pubs_deduprecord);
+		assertEquals(49, sw_deduprecord);
+		assertEquals(97, ds_deduprecord);
+		assertEquals(89, orp_deduprecord);
+	}
+
+	@Test
+	@Order(6)
+	void updateEntityTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkUpdateEntity.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json")));
+		parser
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
+				});
+
+		new SparkUpdateEntity(parser, spark).run(isLookUpService);
+
+		long organizations = jsc.textFile(testDedupGraphBasePath + "/organization").count();
+		long publications = jsc.textFile(testDedupGraphBasePath + "/publication").count();
+		long projects = jsc.textFile(testDedupGraphBasePath + "/project").count();
+		long datasource = jsc.textFile(testDedupGraphBasePath + "/datasource").count();
+		long softwares = jsc.textFile(testDedupGraphBasePath + "/software").count();
+		long dataset = jsc.textFile(testDedupGraphBasePath + "/dataset").count();
+		long otherresearchproduct = jsc.textFile(testDedupGraphBasePath + "/otherresearchproduct").count();
+
+		long mergedOrgs = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.where("relClass=='merges'")
+			.javaRDD()
+			.map(Relation::getTarget)
+			.distinct()
+			.count();
+
+		long mergedPubs = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.where("relClass=='merges'")
+			.javaRDD()
+			.map(Relation::getTarget)
+			.distinct()
+			.count();
+
+		long mergedSw = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.where("relClass=='merges'")
+			.javaRDD()
+			.map(Relation::getTarget)
+			.distinct()
+			.count();
+
+		long mergedDs = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.where("relClass=='merges'")
+			.javaRDD()
+			.map(Relation::getTarget)
+			.distinct()
+			.count();
+
+		long mergedOrp = spark
+			.read()
+			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
+			.as(Encoders.bean(Relation.class))
+			.where("relClass=='merges'")
+			.javaRDD()
+			.map(Relation::getTarget)
+			.distinct()
+			.count();
+
+		assertEquals(896, publications);
+		assertEquals(838, organizations);
+		assertEquals(100, projects);
+		assertEquals(100, datasource);
+		assertEquals(198, softwares);
+		assertEquals(389, dataset);
+		assertEquals(517, otherresearchproduct);
+
+		long deletedOrgs = jsc
+			.textFile(testDedupGraphBasePath + "/organization")
+			.filter(this::isDeletedByInference)
+			.count();
+
+		long deletedPubs = jsc
+			.textFile(testDedupGraphBasePath + "/publication")
+			.filter(this::isDeletedByInference)
+			.count();
+
+		long deletedSw = jsc
+			.textFile(testDedupGraphBasePath + "/software")
+			.filter(this::isDeletedByInference)
+			.count();
+
+		long deletedDs = jsc
+			.textFile(testDedupGraphBasePath + "/dataset")
+			.filter(this::isDeletedByInference)
+			.count();
+
+		long deletedOrp = jsc
+			.textFile(testDedupGraphBasePath + "/otherresearchproduct")
+			.filter(this::isDeletedByInference)
+			.count();
+
+		assertEquals(mergedOrgs, deletedOrgs);
+		assertEquals(mergedPubs, deletedPubs);
+		assertEquals(mergedSw, deletedSw);
+		assertEquals(mergedDs, deletedDs);
+		assertEquals(mergedOrp, deletedOrp);
+	}
+
+	@Test
+	@Order(7)
+	void propagateRelationTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkPropagateRelation.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")));
+		parser
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
+				});
+
+		new SparkPropagateRelation(parser, spark).run(isLookUpService);
+
+		long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
+
+		assertEquals(4860, relations);
+
+		// check deletedbyinference
+		final Dataset<Relation> mergeRels = spark
+			.read()
+			.load(DedupUtility.createMergeRelPath(testOutputBasePath, "*", "*"))
+			.as(Encoders.bean(Relation.class));
+		final JavaPairRDD<String, String> mergedIds = mergeRels
+			.where("relClass == 'merges'")
+			.select(mergeRels.col("target"))
+			.distinct()
+			.toJavaRDD()
+			.mapToPair(
+				(PairFunction<Row, String, String>) r -> new Tuple2<String, String>(r.getString(0), "d"));
+
+		JavaRDD<String> toCheck = jsc
+			.textFile(testDedupGraphBasePath + "/relation")
+			.mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.source", json), json))
+			.join(mergedIds)
+			.map(t -> t._2()._1())
+			.mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.target", json), json))
+			.join(mergedIds)
+			.map(t -> t._2()._1());
+
+		long deletedbyinference = toCheck.filter(this::isDeletedByInference).count();
+		long updated = toCheck.count();
+
+		assertEquals(updated, deletedbyinference);
+	}
+
+	@Test
+	@Order(8)
+	void testRelations() throws Exception {
+		testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_1.json", 12, 10);
+		testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_2.json", 10, 2);
+	}
+
+	private void testUniqueness(String path, int expected_total, int expected_unique) {
+		Dataset<Relation> rel = spark
+			.read()
+			.textFile(getClass().getResource(path).getPath())
+			.map(
+				(MapFunction<String, Relation>) s -> new ObjectMapper().readValue(s, Relation.class),
+				Encoders.bean(Relation.class));
+
+		assertEquals(expected_total, rel.count());
+		assertEquals(expected_unique, rel.distinct().count());
+	}
+
+	@AfterAll
+	public static void finalCleanUp() throws IOException {
+		FileUtils.deleteDirectory(new File(testOutputBasePath));
+		FileUtils.deleteDirectory(new File(testDedupGraphBasePath));
+	}
+
+	public boolean isDeletedByInference(String s) {
+		return s.contains("\"deletedbyinference\":true");
+	}
 }

From 8d3b60f4469f553acfad14e618414c930daebe05 Mon Sep 17 00:00:00 2001
From: Alessia Bardi <alessia.bardi@isti.cnr.it>
Date: Thu, 7 Oct 2021 17:30:45 +0200
Subject: [PATCH 04/12] test for patching records for EOSC Future

---
 .../provision/IndexRecordTransformerTest.java |  20 ++
 .../eosc-future/air-quality-copernicus.xml    | 114 +++++++
 .../eosc-future/b2share-plot-related-orp.xml  | 288 ++++++++++++++++++
 .../provision/eosc-future/b2share-plot-sw.xml | 112 +++++++
 .../eosc-future/data-transfer-pilot.xml       |  23 +-
 .../training-notebooks-seadatanet.xml         |  11 +-
 6 files changed, 551 insertions(+), 17 deletions(-)
 create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/air-quality-copernicus.xml
 create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-related-orp.xml
 create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-sw.xml

diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java
index 8daf318be..1c7dce3f2 100644
--- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java
+++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java
@@ -88,6 +88,26 @@ public class IndexRecordTransformerTest {
 		testRecordTransformation(record);
 	}
 
+	@Test
+	public void testForEOSCFutureAirQualityCopernicus() throws IOException, TransformerException {
+		final String record = IOUtils.toString(getClass().getResourceAsStream("eosc-future/air-quality-copernicus.xml"));
+		testRecordTransformation(record);
+	}
+
+	@Test
+	public void testForEOSCFutureB2SharePlotSw() throws IOException, TransformerException {
+		final String record = IOUtils.toString(getClass().getResourceAsStream("eosc-future/b2share-plot-sw.xml"));
+		testRecordTransformation(record);
+	}
+
+	@Test
+	public void testForEOSCFutureB2SharePlotRelatedORP() throws IOException, TransformerException {
+		final String record = IOUtils.toString(getClass().getResourceAsStream("eosc-future/b2share-plot-related-orp.xml"));
+		testRecordTransformation(record);
+	}
+
+
+
 	private void testRecordTransformation(final String record) throws IOException, TransformerException {
 		final String fields = IOUtils.toString(getClass().getResourceAsStream("fields.xml"));
 		final String xslt = IOUtils.toString(getClass().getResourceAsStream("layoutToRecordTransformer.xsl"));
diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/air-quality-copernicus.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/air-quality-copernicus.xml
new file mode 100644
index 000000000..43b256bbb
--- /dev/null
+++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/air-quality-copernicus.xml
@@ -0,0 +1,114 @@
+<record>
+    <result xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+        <header>
+            <dri:objIdentifier>r37b0ad08687::a8df7db30ae0e4e0b875a098df7b652f</dri:objIdentifier>
+            <dri:dateOfCollection>2021-10-07T01:56:56Z</dri:dateOfCollection>
+            <dri:status>under curation</dri:status>
+            <counters/>
+        </header>
+        <metadata>
+            <oaf:entity xmlns:oaf="http://namespace.openaire.eu/oaf"
+                        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                        xsi:schemaLocation="http://namespace.openaire.eu/oaf http://namespace.openaire.eu/oaf http://www.openaire.eu/schema/0.2/oaf-0.2.xsd">
+
+                <oaf:result>
+                    <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                           schemename="dnet:dataCite_title">Using CAMS European air quality analysis from Copernicus
+                        Atmosphere Monitoring with RELIANCE services
+                    </title>
+                    <creator rank="1" name="Simone" surname="Mantovani" orcid_pending="0000-0003-3979-3645">Simone Mantovani</creator>
+                    <dateofacceptance>2021-10-07</dateofacceptance>
+                    <resulttype classid="software" classname="software" schemeid="dnet:result_typologies"
+                                schemename="dnet:result_typologies"/>
+                    <language classid="" classname="" schemeid="dnet:languages" schemename="dnet:languages"/>
+                    <description>
+                        This notebook shows how to discover and access the Copernicus Atmosphere Monitoring products available in the RELIANCE datacube resources.
+                        The process is structured in 6 steps, including example of data analysis and visualization with the Python libraries installed in the Jupyter environment
+                    </description>
+                    <country classid="" classname="" schemeid="" schemename=""/>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">EOSC Jupyter Notebook
+                    </subject>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">RELIANCE
+                    </subject>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">Copernicus
+                    </subject>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">Air quality
+                    </subject>
+
+                    <relevantdate classid="" classname="" schemeid="" schemename=""/>
+                    <publisher>Zenodo</publisher>
+                    <embargoenddate/>
+                    <journal issn="" eissn="" lissn="" ep="" iss="" sp="" vol=""/>
+                    <source/>
+                    <fulltext/>
+                    <format/>
+                    <storagedate/>
+                    <resourcetype classid="" classname="" schemeid="" schemename=""/>
+                    <device/>
+                    <size/>
+                    <version/>
+                    <lastmetadataupdate/>
+                    <metadataversionnumber/>
+                    <documentationUrl/>
+                    <codeRepositoryUrl/>
+                    <programmingLanguage classid="" classname="" schemeid="" schemename=""/>
+                    <contactperson/>
+                    <contactgroup/>
+                    <tool/>
+                    <originalId>oai:zenodo.org:5554786</originalId>
+                    <collectedfrom name="Zenodo" id="re3data_____::7b0ad08687b2c960d5aeef06f811d5e6"/>
+                    <pid classid="oai" classname="Open Archives Initiative" schemeid="dnet:pid_types"
+                         schemename="dnet:pid_types">oai:zenodo.org:5554786
+                    </pid>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                         schemename="dnet:pid_types">10.5281/zenodo.5554786
+                    </pid>
+                    <bestaccessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                     schemename="dnet:access_modes"/>
+                    <datainfo>
+                        <inferred>false</inferred>
+                        <deletedbyinference>false</deletedbyinference>
+                        <trust>0.9</trust>
+                        <inferenceprovenance/>
+                        <provenanceaction classid="user:insert" classname="user:insert"
+                                          schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
+                    </datainfo>
+                    <rels>
+                        <rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="user:claim">
+                            <to class="isProducedBy" scheme="dnet:result_project_relations" type="project">
+                                corda__h2020::8771f523c34e38902d4921037d545ef8
+                            </to>
+                            <title>REsearch LIfecycle mAnagemeNt for Earth Science Communities and CopErnicus users in EOSC</title>
+                            <code>101017501</code>
+                            <acronym>RELIANCE</acronym>
+                            <funding>
+                                <funder id="ec__________::EC" shortname="EC" name="European Commission" jurisdiction="EU" />
+                                <funding_level_0 name="H2020">ec__________::EC::H2020</funding_level_0>
+                                <funding_level_1 name="RIA">ec__________::EC::H2020::RIA</funding_level_1>
+                            </funding>
+                        </rel>
+                    </rels>
+                    <children>
+                        <instance id="r37b0ad08687::a8df7db30ae0e4e0b875a098df7b652f">
+                            <instancetype classid="0029" classname="Software" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <collectedfrom name="Zenodo" id="re3data_____::7b0ad08687b2c960d5aeef06f811d5e6"/>
+                            <hostedby name="Zenodo" id="re3data_____::7b0ad08687b2c960d5aeef06f811d5e6"/>
+                            <accessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <dateofacceptance/>
+                            <webresource>
+                                <url>https://zenodo.org/record/5554786</url>
+                            </webresource>
+                        </instance>
+                    </children>
+                </oaf:result>
+            </oaf:entity>
+        </metadata>
+    </result>
+</record>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-related-orp.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-related-orp.xml
new file mode 100644
index 000000000..3c2c6440f
--- /dev/null
+++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-related-orp.xml
@@ -0,0 +1,288 @@
+<record>
+    <result xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+        <header>
+            <dri:objIdentifier>doi_dedup___::44fd8a9b5b79adb0783ac245b21e3127</dri:objIdentifier>
+            <dri:dateOfCollection>2019-09-19T07:43:31+0000</dri:dateOfCollection>
+            <dri:dateOfTransformation>2019-09-19T07:43:31+0000</dri:dateOfTransformation>
+        </header>
+        <metadata>
+            <oaf:entity xmlns:oaf="http://namespace.openaire.eu/oaf"
+                        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                        xsi:schemaLocation="http://namespace.openaire.eu/oaf http://namespace.openaire.eu/oaf http://www.openaire.eu/schema/0.2/oaf-0.2.xsd">
+
+                <oaf:result>
+                    <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                    <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                    <originalId>10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6</originalId>
+                    <originalId>10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906</originalId>
+                    <originalId>6a93c069-a167-44cb-bfe8-74c275637347</originalId>
+                    <originalId>50|r3730f562f9e::9b434fedc00d568b8e00611a7fa19f41</originalId>
+                    <originalId>10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016</originalId>
+                    <originalId>ada23067-496a-494f-bd82-6ffe3cf4f0fb</originalId>
+                    <originalId>50|r3730f562f9e::b9cd774e8126b6902d56f9a4aa03e1dc</originalId>
+                    <originalId>f3bd1041-422c-439d-8e68-c1d0711d130d</originalId>
+                    <originalId>50|r3730f562f9e::b847821a0ca5365b0d971dd89dea6bf1</originalId>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                         schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                         trust="0.9">10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6
+                    </pid>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                         schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                         trust="0.9">10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906
+                    </pid>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                         schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                         trust="0.9">10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016
+                    </pid>
+                    <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                           schemename="dnet:dataCite_title">HCG16 L-band VLA C+D array final data
+                    </title>
+                    <bestaccessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                     schemename="dnet:access_modes"/>
+                    <creator rank="1" name="Michael G." surname="Jones">Jones, Michael G.</creator>
+                    <dateofacceptance>2019-01-01</dateofacceptance>
+                    <description>These are the reduced final data associated with the paper Jones et al. 2019 submitted
+                        to Astronomy &amp; Astrophysics. They are used by a mybinder (https://gke.mybinder.org/)
+                        executable environment to generate the final plots of that paper. The link for this environment
+                        is https://mybinder.org/v2/gh/AMIGA-IAA/hcg-16/master. The raw VLA D and C array data of HCG 16
+                        were collected by the Very Large Array (http://www.vla.nrao.edu/) in 1989 and 1999, under PI
+                        projects of Barbara Williams. The project numbers are AW234 and AW500 respectively. The file
+                        also includes a grz colour image and r-band image from DECaLS DR8
+                        (http://legacysurvey.org/decamls/), a GBT HI spectrum published in Borthakur et al. 2010 (ApJ
+                        710, 385), an HI data cube from HIPASS (https://www.atnf.csiro.au/research/multibeam/release/),
+                        and a source mask (and associated parameters file) for the HIPASS cube generated using SoFiA
+                        (https://github.com/SoFiA-Admin/SoFiA-2).
+                    </description>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">3.5.2.1.1 → Observational astronomy →
+                        Radio astronomy
+                    </subject>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">HI
+                    </subject>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">VLA
+                    </subject>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">HCG16
+                    </subject>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies" inferred="false"
+                             provenanceaction="sysimport:crosswalk:repository" trust="0.9">Various
+                    </subject>
+                    <language classid="und" classname="Undetermined" schemeid="dnet:languages"
+                              schemename="dnet:languages"/>
+                    <relevantdate classid="issued" classname="issued" schemeid="dnet:dataCite_date"
+                                  schemename="dnet:dataCite_date">2019-01-01
+                    </relevantdate>
+                    <publisher>https://b2share.eudat.eu</publisher>
+                    <resulttype classid="other" classname="other" schemeid="dnet:result_typologies"
+                                schemename="dnet:result_typologies"/>
+                    <resourcetype classid="UNKNOWN" classname="Unknown" schemeid="dnet:dataCite_resource"
+                                  schemename="dnet:dataCite_resource"/>
+                    <datainfo>
+                        <inferred>true</inferred>
+                        <deletedbyinference>false</deletedbyinference>
+                        <trust>0.8</trust>
+                        <inferenceprovenance>dedup-similarity-result-decisiontree-v2</inferenceprovenance>
+                        <provenanceaction classid="sysimport:dedup" classname="Inferred by OpenAIRE"
+                                          schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
+                    </datainfo>
+                    <rels>
+                        <rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="user:claim">
+                            <to class="IsRelatedTo" scheme="dnet:result_result_relations" type="software">userclaim___::ee29372a239b79db3ac4c5debe44d6e6</to>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">Plot scripts for HCG-16 Project</title>
+                        </rel>
+                    </rels>
+                    <children>
+                        <result objidentifier="r3730f562f9e::9b434fedc00d568b8e00611a7fa19f41">
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                                   schemename="dnet:dataCite_title" inferred="false"
+                                   provenanceaction="sysimport:crosswalk:repository" trust="0.9">HCG16 L-band VLA C+D
+                                array final data
+                            </title>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <publisher>B2SHARE</publisher>
+                        </result>
+                        <result objidentifier="doi_________::929c1e415f4bb04797679d1af7cb706f">
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906
+                            </pid>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                                   schemename="dnet:dataCite_title">HCG16 L-band VLA C+D array final data
+                            </title>
+                            <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                            <publisher>https://b2share.eudat.eu</publisher>
+                        </result>
+                        <result objidentifier="doi_________::e24d8a6399c5d8df9a78aed032573b81">
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                                   schemename="dnet:dataCite_title">HCG16 L-band VLA C+D array final data
+                            </title>
+                            <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016
+                            </pid>
+                            <publisher>https://b2share.eudat.eu</publisher>
+                        </result>
+                        <result objidentifier="r3730f562f9e::b9cd774e8126b6902d56f9a4aa03e1dc">
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                                   schemename="dnet:dataCite_title" inferred="false"
+                                   provenanceaction="sysimport:crosswalk:repository" trust="0.9">HCG16 L-band VLA C+D
+                                array final data
+                            </title>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <publisher>B2SHARE</publisher>
+                        </result>
+                        <result objidentifier="doi_________::44fd8a9b5b79adb0783ac245b21e3127">
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                                   schemename="dnet:dataCite_title">HCG16 L-band VLA C+D array final data
+                            </title>
+                            <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                            <publisher>https://b2share.eudat.eu</publisher>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6
+                            </pid>
+                        </result>
+                        <result objidentifier="r3730f562f9e::b847821a0ca5365b0d971dd89dea6bf1">
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                                   schemename="dnet:dataCite_title" inferred="false"
+                                   provenanceaction="sysimport:crosswalk:repository" trust="0.9">HCG16 L-band VLA C+D
+                                array final data
+                            </title>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <publisher>B2SHARE</publisher>
+                        </result>
+                        <instance id="openaire____::55045bd2a65019fd8e6741a755395c8c">
+                            <accessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                            <hostedby name="Unknown Repository" id="openaire____::55045bd2a65019fd8e6741a755395c8c"/>
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <instancetype classid="0020" classname="Other ORP type" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6
+                            </pid>
+                            <refereed classid="0000" classname="Unknown" schemeid="dnet:review_levels"
+                                      schemename="dnet:review_levels"/>
+                            <webresource>
+                                <url>https://dx.doi.org/10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6</url>
+                            </webresource>
+                        </instance>
+                        <instance id="openaire____::55045bd2a65019fd8e6741a755395c8c">
+                            <accessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                            <hostedby name="Unknown Repository" id="openaire____::55045bd2a65019fd8e6741a755395c8c"/>
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <instancetype classid="0020" classname="Other ORP type" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906
+                            </pid>
+                            <refereed classid="0000" classname="Unknown" schemeid="dnet:review_levels"
+                                      schemename="dnet:review_levels"/>
+                            <webresource>
+                                <url>https://dx.doi.org/10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906</url>
+                            </webresource>
+                        </instance>
+                        <instance id="re3data_____::730f562f9efe8a3b3742d2da510d4335">
+                            <accessright classid="UNKNOWN" classname="not available" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <hostedby name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <instancetype classid="0000" classname="Unknown" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <alternateidentifier classid="doi" classname="Digital Object Identifier"
+                                                 schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false"
+                                                 provenanceaction="sysimport:crosswalk:repository" trust="0.9">
+                                https://doi.org10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6
+                            </alternateidentifier>
+                            <refereed classid="0000" classname="Unknown" schemeid="dnet:review_levels"
+                                      schemename="dnet:review_levels"/>
+                            <webresource>
+                                <url>
+                                    http://dx.doi.org/https://doi.org/10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6
+                                </url>
+                            </webresource>
+                        </instance>
+                        <instance id="openaire____::55045bd2a65019fd8e6741a755395c8c">
+                            <accessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                            <hostedby name="Unknown Repository" id="openaire____::55045bd2a65019fd8e6741a755395c8c"/>
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <instancetype classid="0020" classname="Other ORP type" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016
+                            </pid>
+                            <refereed classid="0000" classname="Unknown" schemeid="dnet:review_levels"
+                                      schemename="dnet:review_levels"/>
+                            <webresource>
+                                <url>https://dx.doi.org/10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016</url>
+                            </webresource>
+                        </instance>
+                        <instance id="re3data_____::730f562f9efe8a3b3742d2da510d4335">
+                            <accessright classid="UNKNOWN" classname="not available" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <hostedby name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <instancetype classid="0000" classname="Unknown" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <alternateidentifier classid="doi" classname="Digital Object Identifier"
+                                                 schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false"
+                                                 provenanceaction="sysimport:crosswalk:repository" trust="0.9">
+                                https://doi.org10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016
+                            </alternateidentifier>
+                            <refereed classid="0000" classname="Unknown" schemeid="dnet:review_levels"
+                                      schemename="dnet:review_levels"/>
+                            <webresource>
+                                <url>
+                                    http://dx.doi.org/https://doi.org/10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016
+                                </url>
+                            </webresource>
+                        </instance>
+                        <instance id="re3data_____::730f562f9efe8a3b3742d2da510d4335">
+                            <accessright classid="UNKNOWN" classname="not available" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <hostedby name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <instancetype classid="0000" classname="Unknown" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <alternateidentifier classid="doi" classname="Digital Object Identifier"
+                                                 schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false"
+                                                 provenanceaction="sysimport:crosswalk:repository" trust="0.9">
+                                https://doi.org10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906
+                            </alternateidentifier>
+                            <refereed classid="0000" classname="Unknown" schemeid="dnet:review_levels"
+                                      schemename="dnet:review_levels"/>
+                            <webresource>
+                                <url>
+                                    http://dx.doi.org/https://doi.org/10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906
+                                </url>
+                            </webresource>
+                        </instance>
+                    </children>
+                </oaf:result>
+            </oaf:entity>
+        </metadata>
+    </result>
+</record>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-sw.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-sw.xml
new file mode 100644
index 000000000..5f44f6b1f
--- /dev/null
+++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/b2share-plot-sw.xml
@@ -0,0 +1,112 @@
+<record>
+    <result xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+        <header>
+            <dri:objIdentifier>userclaim___::ee29372a239b79db3ac4c5debe44d6e6</dri:objIdentifier>
+            <dri:dateOfCollection>2021-10-07T12:42:54Z</dri:dateOfCollection>
+            <counters/>
+        </header>
+        <metadata>
+            <oaf:entity xmlns:oaf="http://namespace.openaire.eu/oaf"
+                        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                        xsi:schemaLocation="http://namespace.openaire.eu/oaf http://namespace.openaire.eu/oaf http://www.openaire.eu/schema/0.2/oaf-0.2.xsd">
+
+                <oaf:result>
+                    <title classid="main title" classname="main title" schemeid="dnet:dataCite_title"
+                           schemename="dnet:dataCite_title">Plot scripts for HCG-16 Project
+                    </title>
+                    <creator rank="1" name="Michael G." surname="Jones">Jones, Michael G.</creator>
+                    <creator rank="2" name="Sebastián" surname="Luna-Valero">Jones, Michael G.</creator>
+                    <dateofacceptance>2021-09-30</dateofacceptance>
+                    <resulttype classid="software" classname="software" schemeid="dnet:result_typologies"
+                                schemename="dnet:result_typologies"/>
+                    <language classid="" classname="" schemeid="dnet:languages" schemename="dnet:languages"/>
+                    <description>These are the notebooks to general the final data plots of the paper Jones et al. 2019
+                        submitted to Astronomy &amp; Astrophysics. They can be used in a notebooks environment (like
+                        https://notebooks.egi.eu/) with the proper libraries installed. A mybinder
+                        (https://mybinder.org/)
+                        ready version can be started from https://mybinder.org/v2/gh/AMIGA-IAA/hcg-16/master. Data to
+                        generate plots is also available from B2SHARE:
+                        https://b2share.eudat.eu/records/a69a7b2dcc22449e8734552dde4d3906
+                    </description>
+                    <country classid="" classname="" schemeid="" schemename=""/>
+                    <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies"
+                             schemename="dnet:subject_classification_typologies">EOSC Jupyter Notebook
+                    </subject>
+                    <relevantdate classid="" classname="" schemeid="" schemename=""/>
+                    <publisher>B2SHARE</publisher>
+                    <embargoenddate/>
+                    <journal issn="" eissn="" lissn="" ep="" iss="" sp="" vol=""/>
+                    <source/>
+                    <fulltext/>
+                    <format/>
+                    <storagedate/>
+                    <resourcetype classid="" classname="" schemeid="" schemename=""/>
+                    <device/>
+                    <size/>
+                    <version/>
+                    <lastmetadataupdate/>
+                    <metadataversionnumber/>
+                    <documentationUrl/>
+                    <codeRepositoryUrl/>
+                    <programmingLanguage classid="" classname="" schemeid="" schemename=""/>
+                    <contactperson/>
+                    <contactgroup/>
+                    <tool/>
+                    <originalId>userclaim___::ee29372a239b79db3ac4c5debe44d6e6</originalId>
+                    <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                         schemename="dnet:pid_types">10.23728/b2share.adf6e2e942b04561a8640c449b48c14a
+                    </pid>
+                    <bestaccessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                     schemename="dnet:access_modes"/>
+                    <datainfo>
+                        <inferred>false</inferred>
+                        <deletedbyinference>false</deletedbyinference>
+                        <trust>0.9</trust>
+                        <inferenceprovenance/>
+                        <provenanceaction classid="user:insert" classname="user:insert"
+                                          schemeid="dnet:provenanceActions"
+                                          schemename="dnet:provenanceActions"/>
+                    </datainfo>
+                    <rels>
+                        <rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="user:claim">
+                            <to class="IsRelatedTo" scheme="dnet:result_result_relations" type="otherresearchproduct">doi_dedup___::44fd8a9b5b79adb0783ac245b21e3127</to>
+                            <title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">HCG16 L-band VLA C+D array final data</title>
+                            <dateofacceptance>2019-01-01</dateofacceptance>
+                            <publisher>https://b2share.eudat.eu</publisher>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.ebcd2972c5fb44199f8b3fdf9f6413c6
+                            </pid>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.a69a7b2dcc22449e8734552dde4d3906
+                            </pid>
+                            <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types"
+                                 schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset"
+                                 trust="0.9">10.23728/b2share.7c8655b6f25348358b4e6fece7ab6016
+                            </pid>
+                            <collectedfrom name="Datacite" id="openaire____::9e3be59865b2c1c335d32dae2fe7b254"/>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                        </rel>
+                    </rels>
+                    <children>
+                        <instance id="userclaim___::ee29372a239b79db3ac4c5debe44d6e6">
+                            <instancetype classid="0029" classname="Software" schemeid="dnet:publication_resource"
+                                          schemename="dnet:publication_resource"/>
+                            <collectedfrom name="B2FIND" id="re3data_____::730f562f9efe8a3b3742d2da510d4335"/>
+                            <hostedby name="B2SHARE" id="re3data_____::ad3609c351bd520edf6f10f5e0d9b877"/>
+                            <accessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes"
+                                         schemename="dnet:access_modes"/>
+                            <dateofacceptance>2021-09-30</dateofacceptance>
+                            <webresource>
+                                <url>http://dx.doi.org/10.23728/b2share.adf6e2e942b04561a8640c449b48c14a</url>
+                            </webresource>
+                        </instance>
+                    </children>
+                </oaf:result>
+            </oaf:entity>
+        </metadata>
+    </result>
+</record>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/data-transfer-pilot.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/data-transfer-pilot.xml
index 23dd6c6ed..6d2ac7630 100644
--- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/data-transfer-pilot.xml
+++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/data-transfer-pilot.xml
@@ -1,26 +1,25 @@
 <record>
     <result xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
         <header>
-            <dri:objIdentifier>r37b0ad08687::dec0d8520e726f2adda9a51280ac7299</dri:objIdentifier>
-            <dri:dateOfCollection>2021-09-22T08:53:16Z</dri:dateOfCollection>
-            <dri:status>under curation</dri:status>
-            <counters />
+            <dri:objIdentifier>doi_dedup___::ab57f086011a9ae23d1165211dc6e04b</dri:objIdentifier>
+            <dri:dateOfCollection>2020-11-03T05:39:50+0000</dri:dateOfCollection>
+            <dri:dateOfTransformation>2020-11-03T05:39:50+0000</dri:dateOfTransformation>
         </header>
         <metadata>
             <oaf:entity xmlns:oaf="http://namespace.openaire.eu/oaf"
                         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://namespace.openaire.eu/oaf http://namespace.openaire.eu/oaf http://www.openaire.eu/schema/0.2/oaf-0.2.xsd">
             <oaf:result>
                     <title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">EGI-Foundation/data-transfer-pilot: Include libraries in environment.yml</title>
-                    <creator rank="1" name="" surname="">Giuseppe La Rocca</creator>
-                    <creator rank="2" name="" surname="">Enol Fernández</creator>
-                    <creator rank="3" name="" surname="">Andrea Manzi</creator>
-                    <dateofacceptance />
+                    <creator rank="1" name="Giuseppe" surname="La Rocca">Giuseppe La Rocca</creator>
+                    <creator rank="2" name="Enol" surname="Fernández">Enol Fernández</creator>
+                    <creator rank="3" name="Andrea" surname="Manzi">Andrea Manzi</creator>
+                    <dateofacceptance>2020-11-03</dateofacceptance>
                     <resulttype classid="software" classname="software" schemeid="dnet:result_typologies" schemename="dnet:result_typologies" />
                     <language classid="" classname="" schemeid="dnet:languages" schemename="dnet:languages" />
                     <description>This notebook is used to demonstrate how a scientist from one of the PaNOSC RIs can use the resources provided by EGI to perform analysis on the data sets obtained during an expirement.</description>
                     <country classid="" classname="" schemeid="" schemename="" />
                     <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies" schemename="dnet:subject_classification_typologies">EOSC Jupyter Notebook</subject>
-                    <relevantdate classid="" classname="" schemeid="" schemename="" />
+                    <relevantdate classid="issued" classname="issued" schemeid="dnet:dataCite_date" schemename="dnet:dataCite_date">2020-11-03</relevantdate>
                     <publisher>Zenodo</publisher>
                     <embargoenddate />
                     <journal issn="" eissn="" lissn="" ep="" iss="" sp="" vol="" />
@@ -43,8 +42,8 @@
                     <originalId>oai:zenodo.org:4218562</originalId>
                     <collectedfrom name="Zenodo" id="re3data_____::7b0ad08687b2c960d5aeef06f811d5e6" />
                     <pid classid="oai" classname="Open Archives Initiative" schemeid="dnet:pid_types" schemename="dnet:pid_types">oai:zenodo.org:4218562</pid>
-                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types" schemename="dnet:pid_types">10.5281/zenodo.4218562</pid>
-                    <bestaccessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes" schemename="dnet:access_modes" />
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset" trust="0.9">10.5281/zenodo.4195418</pid>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset" trust="0.9">10.5281/zenodo.4218562</pid><bestaccessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes" schemename="dnet:access_modes" />
                     <datainfo>
                         <inferred>false</inferred>
                         <deletedbyinference>false</deletedbyinference>
@@ -59,7 +58,7 @@
                             <collectedfrom name="Zenodo" id="re3data_____::7b0ad08687b2c960d5aeef06f811d5e6" />
                             <hostedby name="Zenodo" id="re3data_____::7b0ad08687b2c960d5aeef06f811d5e6" />
                             <accessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes" schemename="dnet:access_modes" />
-                            <dateofacceptance />
+                            <dateofacceptance>2020-11-03</dateofacceptance>
                             <webresource>
                                 <url>https://zenodo.org/record/4218562</url>
                             </webresource>
diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/training-notebooks-seadatanet.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/training-notebooks-seadatanet.xml
index 9995b902f..9ab9b9861 100644
--- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/training-notebooks-seadatanet.xml
+++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/eosc-future/training-notebooks-seadatanet.xml
@@ -1,10 +1,8 @@
 <record>
     <result xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
         <header>
-            <dri:objIdentifier>r37b0ad08687::eb430fb7438e1533ba95d6aa50a477eb</dri:objIdentifier>
+            <dri:objIdentifier>doi_dedup___::8539a8de8996e01350f0de8ca4899b7f</dri:objIdentifier>
             <dri:dateOfCollection>2021-09-22T08:53:13Z</dri:dateOfCollection>
-            <dri:status>under curation</dri:status>
-            <counters />
         </header>
         <metadata>
         <oaf:entity xmlns:oaf="http://namespace.openaire.eu/oaf"
@@ -13,10 +11,10 @@
             <oaf:result>
                     <title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">EGI-Foundation/training-notebooks-seadatanet: Version 0.4</title>
                     <creator rank="1" name="" surname="">Enol Fernández</creator>
-                    <dateofacceptance />
+                    <dateofacceptance>2019-12-04</dateofacceptance>
                     <resulttype classid="software" classname="software" schemeid="dnet:result_typologies" schemename="dnet:result_typologies" />
                     <language classid="" classname="" schemeid="dnet:languages" schemename="dnet:languages" />
-                    <description>A sample notebook using SeaDataNet data to plot a map that shows surface temperature of Black Sea, Arctic Sea and Baltic Sea. The data is available at EGI DataHub with PID http://hdl.handle.net/21.T15999/qVk6JWQ (run at EGI Notebooks service for easy access to data).This release updates the PID for the data.</description>
+                    <description>A sample notebook using SeaDataNet data to plot a map that shows surface temperature of Black Sea, Arctic Sea and Baltic Sea. The data is available at EGI DataHub with PID http://hdl.handle.net/21.T15999/3Byz9Cw (run at EGI Notebooks service for easy access to data). This release uses the correct path of the data share from the EGI DataHub.</description>
                     <country classid="" classname="" schemeid="" schemename="" />
                     <subject classid="keyword" classname="keyword" schemeid="dnet:subject_classification_typologies" schemename="dnet:subject_classification_typologies">EOSC Jupyter Notebook</subject>
                     <relevantdate classid="" classname="" schemeid="" schemename="" />
@@ -43,6 +41,9 @@
                     <collectedfrom name="Zenodo" id="re3data_____::7b0ad08687b2c960d5aeef06f811d5e6" />
                     <pid classid="oai" classname="Open Archives Initiative" schemeid="dnet:pid_types" schemename="dnet:pid_types">oai:zenodo.org:3561323</pid>
                     <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types" schemename="dnet:pid_types">10.5281/zenodo.3561323</pid>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset" trust="0.9">10.5281/zenodo.3443996</pid>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset" trust="0.9">10.5281/zenodo.3475539</pid>
+                    <pid classid="doi" classname="Digital Object Identifier" schemeid="dnet:pid_types" schemename="dnet:pid_types" inferred="false" provenanceaction="sysimport:actionset" trust="0.9">10.5281/zenodo.3475785</pid>
                     <bestaccessright classid="OPEN" classname="Open Access" schemeid="dnet:access_modes" schemename="dnet:access_modes" />
                     <datainfo>
                         <inferred>false</inferred>

From 8f99d2af86d96194b5822b0a71991d568d034369 Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Tue, 28 Sep 2021 11:31:19 +0200
Subject: [PATCH 05/12] Make the node of doiBoost to point to the correct
 OpenAire Organization in relations

---
 .../main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala
index e501b4823..501073e74 100644
--- a/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala
+++ b/dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala
@@ -208,7 +208,7 @@ object SparkGenerateDoiBoost {
         (r.getTarget,r)
         else
         ("resolved", r)
-    })
+    })(Encoders.tuple(Encoders.STRING, mapEncoderRel))
 
     val openaireOrganization:Dataset[(String,String)] = spark.read.text(openaireOrganizationPath).as[String].flatMap(s => extractIdGRID(s)).groupByKey(_._2).reduceGroups((x,y) => if (x != null) x else y ).map(_._2)
 
@@ -222,7 +222,7 @@ object SparkGenerateDoiBoost {
           else
             currentRels.setTarget(currentOrgs._1)
       currentRels
-      }.write.save(s"$workingDirPath/doiBoostPublicationAffiliation")
+      }.filter(r=> !r.getSource.startsWith("unresolved") && !r.getTarget.startsWith("unresolved")).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationAffiliation")
 
     magPubs.joinWith(a,magPubs("_1").equalTo(a("PaperId"))).map( item => {
       val affiliation = item._2

From 5606014b178bc6fd72b1121652de816b5be717ef Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Tue, 12 Oct 2021 08:11:53 +0200
Subject: [PATCH 06/12] code refactor see ticket #7065

---
 .../eu/dnetllib/dhp/sx}/bio/BioDBToOAF.scala  |  77 +++++++++---------
 .../bio/SparkTransformBioDatabaseToOAF.scala  |  15 ++--
 .../ebi/SparkCreateBaselineDataFrame.scala    |  62 +++++++-------
 .../sx/bio}/ebi/SparkDownloadEBILinks.scala   |  39 ++++-----
 .../dhp/sx/bio}/ebi/SparkEBILinksToOaf.scala  |  21 +++--
 .../dhp/sx}/bio/pubmed/PMArticle.java         |   2 +-
 .../dnetllib/dhp/sx}/bio/pubmed/PMAuthor.java |   2 +-
 .../dnetllib/dhp/sx}/bio/pubmed/PMGrant.java  |   2 +-
 .../dhp/sx}/bio/pubmed/PMJournal.java         |   2 +-
 .../dhp/sx}/bio/pubmed/PMParser.scala         |   2 +-
 .../dhp/sx}/bio/pubmed/PMSubject.java         |   2 +-
 .../dhp/sx}/bio/pubmed/PubMedToOaf.scala      |  23 +++---
 .../sx/bio}/ebi/baseline_to_oaf_params.json   |   0
 .../dhp/sx/bio}/ebi/ebi_download_update.json  |   0
 .../dhp/sx/bio}/ebi/ebi_to_df_params.json     |   0
 .../bio}/pubmed/oozie_app/config-default.xml  |   0
 .../dhp/sx/bio}/pubmed/oozie_app/workflow.xml |  13 ++-
 .../dnetllib/dhp/sx/bio}/BioScholixTest.scala |  15 ++--
 .../dnetlib/dhp/sx/graph/bio/crossref_links   |   0
 .../eu/dnetlib/dhp/sx/graph/bio/ebi_links.gz  | Bin
 .../eu/dnetlib/dhp/sx/graph/bio}/ls_result    |   0
 .../eu/dnetlib/dhp/sx/graph/bio/pdb_dump      |   0
 .../eu/dnetlib/dhp/sx/graph/bio}/pubmed.xml   |   0
 .../eu/dnetlib/dhp/sx/graph/bio}/pubmed_dump  |   0
 .../dnetlib/dhp/sx/graph/bio/scholix_resolved |   0
 .../eu/dnetlib/dhp/sx/graph/bio/uniprot_dump  |   0
 .../eu/dnetlib/dhp/PropagationConstant.java   |  11 ++-
 .../SparkOrcidToResultFromSemRelJob.java      |   7 +-
 ...kResultToCommunityFromOrganizationJob.java |   4 +-
 ...parkResultToCommunityThroughSemRelJob.java |   4 +-
 .../SparkGeneratePanagaeaDataset.scala        |   1 -
 .../provision/IndexRecordTransformerTest.java |  11 +--
 32 files changed, 155 insertions(+), 160 deletions(-)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/BioDBToOAF.scala (83%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/SparkTransformBioDatabaseToOAF.scala (86%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio}/ebi/SparkCreateBaselineDataFrame.scala (76%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio}/ebi/SparkDownloadEBILinks.scala (70%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio}/ebi/SparkEBILinksToOaf.scala (67%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/pubmed/PMArticle.java (97%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/pubmed/PMAuthor.java (92%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/pubmed/PMGrant.java (93%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/pubmed/PMJournal.java (94%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/pubmed/PMParser.scala (99%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/pubmed/PMSubject.java (94%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx}/bio/pubmed/PubMedToOaf.scala (93%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio}/ebi/baseline_to_oaf_params.json (100%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio}/ebi/ebi_download_update.json (100%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio}/ebi/ebi_to_df_params.json (100%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio}/pubmed/oozie_app/config-default.xml (100%)
 rename dhp-workflows/{dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph => dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio}/pubmed/oozie_app/workflow.xml (78%)
 rename dhp-workflows/{dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed => dhp-aggregation/src/test/java/eu/dnetllib/dhp/sx/bio}/BioScholixTest.scala (92%)
 rename dhp-workflows/{dhp-graph-mapper => dhp-aggregation}/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/crossref_links (100%)
 rename dhp-workflows/{dhp-graph-mapper => dhp-aggregation}/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/ebi_links.gz (100%)
 rename dhp-workflows/{dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed => dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio}/ls_result (100%)
 rename dhp-workflows/{dhp-graph-mapper => dhp-aggregation}/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pdb_dump (100%)
 rename dhp-workflows/{dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed => dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio}/pubmed.xml (100%)
 rename dhp-workflows/{dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed => dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio}/pubmed_dump (100%)
 rename dhp-workflows/{dhp-graph-mapper => dhp-aggregation}/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/scholix_resolved (100%)
 rename dhp-workflows/{dhp-graph-mapper => dhp-aggregation}/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/uniprot_dump (100%)

diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/BioDBToOAF.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/BioDBToOAF.scala
similarity index 83%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/BioDBToOAF.scala
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/BioDBToOAF.scala
index 90b65c8f7..dffc88c6c 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/BioDBToOAF.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/BioDBToOAF.scala
@@ -1,14 +1,12 @@
-package eu.dnetlib.dhp.sx.graph.bio
+package eu.dnetllib.dhp.sx.bio
 
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.{GraphCleaningFunctions, OafMapperUtils}
-import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Dataset, Instance, KeyValue, Oaf, Relation, StructuredProperty}
+import eu.dnetlib.dhp.schema.oaf._
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.{compact, parse, render}
-
-import scala.collection.JavaConverters._
-
+import collection.JavaConverters._
 object BioDBToOAF {
 
   case class EBILinkItem(id: Long, links: String) {}
@@ -17,23 +15,23 @@ object BioDBToOAF {
 
   case class UniprotDate(date: String, date_info: String) {}
 
-  case class ScholixResolved(pid:String, pidType:String, typology:String, tilte:List[String], datasource:List[String], date:List[String], authors:List[String]){}
+  case class ScholixResolved(pid: String, pidType: String, typology: String, tilte: List[String], datasource: List[String], date: List[String], authors: List[String]) {}
 
   val DATA_INFO: DataInfo = OafMapperUtils.dataInfo(false, null, false, false, ModelConstants.PROVENANCE_ACTION_SET_QUALIFIER, "0.9")
   val SUBJ_CLASS = "Keywords"
 
   val DATE_RELATION_KEY = "RelationDate"
 
-  val resolvedURL:Map[String,String] = Map(
-    "genbank"->             "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "ncbi-n" ->             "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "ncbi-wgs" ->           "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "ncbi-p" ->             "https://www.ncbi.nlm.nih.gov/protein/",
-    "ena" ->                "https://www.ebi.ac.uk/ena/browser/view/",
-    "clinicaltrials.gov"->  "https://clinicaltrials.gov/ct2/show/",
-    "onim"->                "https://omim.org/entry/",
-    "refseq"->              "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "geo"->                 "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc="
+  val resolvedURL: Map[String, String] = Map(
+    "genbank" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "ncbi-n" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "ncbi-wgs" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "ncbi-p" -> "https://www.ncbi.nlm.nih.gov/protein/",
+    "ena" -> "https://www.ebi.ac.uk/ena/browser/view/",
+    "clinicaltrials.gov" -> "https://clinicaltrials.gov/ct2/show/",
+    "onim" -> "https://omim.org/entry/",
+    "refseq" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "geo" -> "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc="
   )
 
 
@@ -45,7 +43,7 @@ object BioDBToOAF {
     val ElsevierCollectedFrom: KeyValue = OafMapperUtils.keyValue("10|openaire____::8f87e10869299a5fe80b315695296b88", "Elsevier")
     val springerNatureCollectedFrom: KeyValue = OafMapperUtils.keyValue("10|openaire____::6e380d9cf51138baec8480f5a0ce3a2e", "Springer Nature")
     val EBICollectedFrom: KeyValue = OafMapperUtils.keyValue("10|opendoar____::83e60e09c222f206c725385f53d7e567c", "EMBL-EBIs Protein Data Bank in Europe (PDBe)")
-    val pubmedCollectedFrom:KeyValue = OafMapperUtils.keyValue(ModelConstants.EUROPE_PUBMED_CENTRAL_ID, "Europe PubMed Central")
+    val pubmedCollectedFrom: KeyValue = OafMapperUtils.keyValue(ModelConstants.EUROPE_PUBMED_CENTRAL_ID, "Europe PubMed Central")
 
     UNIPROTCollectedFrom.setDataInfo(DATA_INFO)
     PDBCollectedFrom.setDataInfo(DATA_INFO)
@@ -58,9 +56,9 @@ object BioDBToOAF {
 
     Map(
       "uniprot" -> UNIPROTCollectedFrom,
-      "pdb"-> PDBCollectedFrom,
-      "elsevier" ->ElsevierCollectedFrom,
-      "ebi" ->EBICollectedFrom,
+      "pdb" -> PDBCollectedFrom,
+      "elsevier" -> ElsevierCollectedFrom,
+      "ebi" -> EBICollectedFrom,
       "Springer Nature" -> springerNatureCollectedFrom,
       "NCBI Nucleotide" -> ncbiCollectedFrom,
       "European Nucleotide Archive" -> enaCollectedFrom,
@@ -68,7 +66,7 @@ object BioDBToOAF {
     )
   }
 
-  def crossrefLinksToOaf(input:String):Oaf = {
+  def crossrefLinksToOaf(input: String): Oaf = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
     lazy val json = parse(input)
     val source_pid = (json \ "Source" \ "Identifier" \ "ID").extract[String].toLowerCase
@@ -77,16 +75,16 @@ object BioDBToOAF {
     val target_pid = (json \ "Target" \ "Identifier" \ "ID").extract[String].toLowerCase
     val target_pid_type = (json \ "Target" \ "Identifier" \ "IDScheme").extract[String].toLowerCase
 
-    val relation_semantic= (json \ "RelationshipType" \ "Name").extract[String]
+    val relation_semantic = (json \ "RelationshipType" \ "Name").extract[String]
 
     val date = GraphCleaningFunctions.cleanDate((json \ "LinkedPublicationDate").extract[String])
 
-    createRelation(target_pid, target_pid_type, generate_unresolved_id(source_pid, source_pid_type),collectedFromMap("elsevier"),"relationship", relation_semantic, date)
+    createRelation(target_pid, target_pid_type, generate_unresolved_id(source_pid, source_pid_type), collectedFromMap("elsevier"), "relationship", relation_semantic, date)
 
   }
 
 
-  def scholixResolvedToOAF(input:ScholixResolved):Oaf = {
+  def scholixResolvedToOAF(input: ScholixResolved): Oaf = {
 
     val d = new Dataset
 
@@ -127,18 +125,18 @@ object BioDBToOAF {
     d.setInstance(List(i).asJava)
 
     if (input.authors != null && input.authors.nonEmpty) {
-      val authors = input.authors.map(a =>{
+      val authors = input.authors.map(a => {
         val authorOAF = new Author
         authorOAF.setFullname(a)
         authorOAF
       })
       d.setAuthor(authors.asJava)
     }
-    if (input.date!= null && input.date.nonEmpty) {
-        val dt = input.date.head
-        i.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
-        d.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
-      }
+    if (input.date != null && input.date.nonEmpty) {
+      val dt = input.date.head
+      i.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
+      d.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
+    }
     d
   }
 
@@ -190,7 +188,7 @@ object BioDBToOAF {
           OafMapperUtils.structuredProperty(s, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, null)
         ).asJava)
     }
-    var i_date:Option[UniprotDate] = None
+    var i_date: Option[UniprotDate] = None
 
     if (dates.nonEmpty) {
       i_date = dates.find(d => d.date_info.contains("entry version"))
@@ -218,12 +216,12 @@ object BioDBToOAF {
 
 
     if (references_pmid != null && references_pmid.nonEmpty) {
-      val rel = createRelation(references_pmid.head, "pmid", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if  (i_date.isDefined) i_date.get.date else null)
+      val rel = createRelation(references_pmid.head, "pmid", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if (i_date.isDefined) i_date.get.date else null)
       rel.getCollectedfrom
       List(d, rel)
     }
     else if (references_doi != null && references_doi.nonEmpty) {
-      val rel = createRelation(references_doi.head, "doi", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if  (i_date.isDefined) i_date.get.date else null)
+      val rel = createRelation(references_doi.head, "doi", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if (i_date.isDefined) i_date.get.date else null)
       List(d, rel)
     }
     else
@@ -231,13 +229,12 @@ object BioDBToOAF {
   }
 
 
-
-  def generate_unresolved_id(pid:String, pidType:String) :String = {
+  def generate_unresolved_id(pid: String, pidType: String): String = {
     s"unresolved::$pid::$pidType"
   }
 
 
-  def createRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, subRelType:String, relClass:String, date:String):Relation = {
+  def createRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, subRelType: String, relClass: String, date: String): Relation = {
 
     val rel = new Relation
     rel.setCollectedfrom(List(collectedFromMap("pdb")).asJava)
@@ -251,7 +248,7 @@ object BioDBToOAF {
     rel.setTarget(s"unresolved::$pid::$pidType")
 
 
-    val dateProps:KeyValue = OafMapperUtils.keyValue(DATE_RELATION_KEY, date)
+    val dateProps: KeyValue = OafMapperUtils.keyValue(DATE_RELATION_KEY, date)
 
     rel.setProperties(List(dateProps).asJava)
 
@@ -262,8 +259,8 @@ object BioDBToOAF {
   }
 
 
-  def createSupplementaryRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, date:String): Relation = {
-    createRelation(pid,pidType,sourceId,collectedFrom, ModelConstants.SUPPLEMENT, ModelConstants.IS_SUPPLEMENT_TO, date)
+  def createSupplementaryRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, date: String): Relation = {
+    createRelation(pid, pidType, sourceId, collectedFrom, ModelConstants.SUPPLEMENT, ModelConstants.IS_SUPPLEMENT_TO, date)
   }
 
 
@@ -338,7 +335,7 @@ object BioDBToOAF {
 
   def EBITargetLinksFilter(input: EBILinks): Boolean = {
 
-    input.targetPidType.equalsIgnoreCase("ena") ||   input.targetPidType.equalsIgnoreCase("pdb") ||  input.targetPidType.equalsIgnoreCase("uniprot")
+    input.targetPidType.equalsIgnoreCase("ena") || input.targetPidType.equalsIgnoreCase("pdb") || input.targetPidType.equalsIgnoreCase("uniprot")
 
   }
 
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/SparkTransformBioDatabaseToOAF.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/SparkTransformBioDatabaseToOAF.scala
similarity index 86%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/SparkTransformBioDatabaseToOAF.scala
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/SparkTransformBioDatabaseToOAF.scala
index d66cc84ec..16d2b25a6 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/SparkTransformBioDatabaseToOAF.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/SparkTransformBioDatabaseToOAF.scala
@@ -1,8 +1,8 @@
-package eu.dnetlib.dhp.sx.graph.bio
+package eu.dnetllib.dhp.sx.bio
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Oaf, Result}
-import BioDBToOAF.ScholixResolved
+import eu.dnetlib.dhp.schema.oaf.Oaf
+import eu.dnetllib.dhp.sx.bio.BioDBToOAF.ScholixResolved
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
@@ -31,17 +31,16 @@ object SparkTransformBioDatabaseToOAF {
         .master(parser.get("master")).getOrCreate()
     val sc = spark.sparkContext
 
-    implicit  val resultEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
-    import  spark.implicits._
-
+    implicit val resultEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
+    import spark.implicits._
     database.toUpperCase() match {
       case "UNIPROT" =>
         spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.uniprotToOAF(i))).write.mode(SaveMode.Overwrite).save(targetPath)
-      case "PDB"=>
+      case "PDB" =>
         spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.pdbTOOaf(i))).write.mode(SaveMode.Overwrite).save(targetPath)
       case "SCHOLIX" =>
         spark.read.load(dbPath).as[ScholixResolved].map(i => BioDBToOAF.scholixResolvedToOAF(i)).write.mode(SaveMode.Overwrite).save(targetPath)
-      case "CROSSREF_LINKS"=>
+      case "CROSSREF_LINKS" =>
         spark.createDataset(sc.textFile(dbPath).map(i => BioDBToOAF.crossrefLinksToOaf(i))).write.mode(SaveMode.Overwrite).save(targetPath)
     }
   }
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkCreateBaselineDataFrame.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
similarity index 76%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkCreateBaselineDataFrame.scala
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
index ee76cf8ad..97b3cdc99 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkCreateBaselineDataFrame.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
@@ -1,10 +1,10 @@
-package eu.dnetlib.dhp.sx.graph.ebi
+package eu.dnetllib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
 import eu.dnetlib.dhp.schema.oaf.Result
-import eu.dnetlib.dhp.sx.graph.bio.pubmed.{PMArticle, PMAuthor, PMJournal, PMParser, PubMedToOaf}
 import eu.dnetlib.dhp.utils.ISLookupClientFactory
+import eu.dnetllib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal, PMParser, PubMedToOaf}
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}
@@ -24,24 +24,24 @@ import scala.xml.pull.XMLEventReader
 object SparkCreateBaselineDataFrame {
 
 
-  def requestBaseLineUpdatePage(maxFile:String):List[(String,String)] = {
-    val data =requestPage("https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/")
+  def requestBaseLineUpdatePage(maxFile: String): List[(String, String)] = {
+    val data = requestPage("https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/")
 
-    val result =data.lines.filter(l => l.startsWith("<a href=")).map{l =>
+    val result = data.lines.filter(l => l.startsWith("<a href=")).map { l =>
       val end = l.lastIndexOf("\">")
       val start = l.indexOf("<a href=\"")
 
-      if (start>= 0 && end >start)
-        l.substring(start+9, (end-start))
+      if (start >= 0 && end > start)
+        l.substring(start + 9, (end - start))
       else
         ""
-    }.filter(s =>s.endsWith(".gz") ).filter(s => s > maxFile).map(s => (s,s"https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/$s")).toList
+    }.filter(s => s.endsWith(".gz")).filter(s => s > maxFile).map(s => (s, s"https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/$s")).toList
 
     result
   }
 
 
-  def downloadBaselinePart(url:String):InputStream = {
+  def downloadBaselinePart(url: String): InputStream = {
     val r = new HttpGet(url)
     val timeout = 60; // seconds
     val config = RequestConfig.custom()
@@ -55,7 +55,7 @@ object SparkCreateBaselineDataFrame {
 
   }
 
-  def requestPage(url:String):String = {
+  def requestPage(url: String): String = {
     val r = new HttpGet(url)
     val timeout = 60; // seconds
     val config = RequestConfig.custom()
@@ -90,25 +90,21 @@ object SparkCreateBaselineDataFrame {
   }
 
 
-
-
-
-
-  def downloadBaseLineUpdate(baselinePath:String, hdfsServerUri:String ):Unit = {
+  def downloadBaseLineUpdate(baselinePath: String, hdfsServerUri: String): Unit = {
 
 
     val conf = new Configuration
     conf.set("fs.defaultFS", hdfsServerUri)
-    val fs =  FileSystem.get(conf)
+    val fs = FileSystem.get(conf)
     val p = new Path(baselinePath)
-    val files = fs.listFiles(p,false)
+    val files = fs.listFiles(p, false)
     var max_file = ""
     while (files.hasNext) {
       val c = files.next()
       val data = c.getPath.toString
-      val fileName = data.substring(data.lastIndexOf("/")+1)
+      val fileName = data.substring(data.lastIndexOf("/") + 1)
 
-      if (fileName> max_file)
+      if (fileName > max_file)
         max_file = fileName
     }
 
@@ -119,7 +115,7 @@ object SparkCreateBaselineDataFrame {
       val fsDataOutputStream: FSDataOutputStream = fs.create(hdfsWritePath, true)
       val i = downloadBaselinePart(u._2)
       val buffer = Array.fill[Byte](1024)(0)
-      while(i.read(buffer)>0) {
+      while (i.read(buffer) > 0) {
         fsDataOutputStream.write(buffer)
       }
       i.close()
@@ -134,11 +130,11 @@ object SparkCreateBaselineDataFrame {
     override def zero: PMArticle = new PMArticle
 
     override def reduce(b: PMArticle, a: (String, PMArticle)): PMArticle = {
-      if (b != null && b.getPmid!= null)   b  else a._2
+      if (b != null && b.getPmid != null) b else a._2
     }
 
     override def merge(b1: PMArticle, b2: PMArticle): PMArticle = {
-      if (b1 != null && b1.getPmid!= null)    b1   else     b2
+      if (b1 != null && b1.getPmid != null) b1 else b2
 
     }
 
@@ -153,7 +149,7 @@ object SparkCreateBaselineDataFrame {
   def main(args: Array[String]): Unit = {
     val conf: SparkConf = new SparkConf()
     val log: Logger = LoggerFactory.getLogger(getClass)
-    val parser = new ArgumentApplicationParser(IOUtils.toString(SparkEBILinksToOaf.getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/ebi/baseline_to_oaf_params.json")))
+    val parser = new ArgumentApplicationParser(IOUtils.toString(SparkEBILinksToOaf.getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json")))
     parser.parseArgument(args)
     val isLookupUrl: String = parser.get("isLookupUrl")
     log.info("isLookupUrl: {}", isLookupUrl)
@@ -175,32 +171,32 @@ object SparkCreateBaselineDataFrame {
         .config(conf)
         .appName(SparkEBILinksToOaf.getClass.getSimpleName)
         .master(parser.get("master")).getOrCreate()
-    import spark.implicits._
 
     val sc = spark.sparkContext
+    import spark.implicits._
 
-    implicit  val PMEncoder: Encoder[PMArticle] = Encoders.kryo(classOf[PMArticle])
-    implicit  val PMJEncoder: Encoder[PMJournal] = Encoders.kryo(classOf[PMJournal])
-    implicit  val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
-    implicit  val resultEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
+    implicit val PMEncoder: Encoder[PMArticle] = Encoders.kryo(classOf[PMArticle])
+    implicit val PMJEncoder: Encoder[PMJournal] = Encoders.kryo(classOf[PMJournal])
+    implicit val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
+    implicit val resultEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
 
     downloadBaseLineUpdate(s"$workingPath/baseline", hdfsServerUri)
 
-    val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline",2000)
-    val ds:Dataset[PMArticle] = spark.createDataset(k.filter(i => i._1.endsWith(".gz")).flatMap(i =>{
+    val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline_ftp", 2000)
+    val ds: Dataset[PMArticle] = spark.createDataset(k.filter(i => i._1.endsWith(".gz")).flatMap(i => {
       val xml = new XMLEventReader(Source.fromBytes(i._2.getBytes()))
       new PMParser(xml)
 
-    } ))
+    }))
 
-    ds.map(p => (p.getPmid,p))(Encoders.tuple(Encoders.STRING, PMEncoder)).groupByKey(_._1)
+    ds.map(p => (p.getPmid, p))(Encoders.tuple(Encoders.STRING, PMEncoder)).groupByKey(_._1)
       .agg(pmArticleAggregator.toColumn)
       .map(p => p._2).write.mode(SaveMode.Overwrite).save(s"$workingPath/baseline_dataset")
 
     val exported_dataset = spark.read.load(s"$workingPath/baseline_dataset").as[PMArticle]
     exported_dataset
       .map(a => PubMedToOaf.convert(a, vocabularies)).as[Result]
-      .filter(p => p!= null)
+      .filter(p => p != null)
       .write.mode(SaveMode.Overwrite).save(targetPath)
   }
 }
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkDownloadEBILinks.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkDownloadEBILinks.scala
similarity index 70%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkDownloadEBILinks.scala
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkDownloadEBILinks.scala
index e940fdff0..578db1ea9 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkDownloadEBILinks.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkDownloadEBILinks.scala
@@ -1,8 +1,8 @@
-package eu.dnetlib.dhp.sx.graph.ebi
+package eu.dnetllib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF.EBILinkItem
-import eu.dnetlib.dhp.sx.graph.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
+import eu.dnetllib.dhp.sx.bio.BioDBToOAF.EBILinkItem
+import eu.dnetllib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
 import org.apache.commons.io.IOUtils
 import org.apache.http.client.config.RequestConfig
 import org.apache.http.client.methods.HttpGet
@@ -14,15 +14,15 @@ import org.slf4j.{Logger, LoggerFactory}
 
 object SparkDownloadEBILinks {
 
-  def createEBILinks(pmid:Long):EBILinkItem = {
+  def createEBILinks(pmid: Long): EBILinkItem = {
 
     val res = requestLinks(pmid)
-    if (res!=null)
+    if (res != null)
       return EBILinkItem(pmid, res)
     null
   }
 
-  def requestPage(url:String):String = {
+  def requestPage(url: String): String = {
     val r = new HttpGet(url)
     val timeout = 60; // seconds
     val config = RequestConfig.custom()
@@ -56,10 +56,11 @@ object SparkDownloadEBILinks {
     }
   }
 
-  def requestLinks(PMID:Long):String = {
+  def requestLinks(PMID: Long): String = {
     requestPage(s"https://www.ebi.ac.uk/europepmc/webservices/rest/MED/$PMID/datalinks?format=json")
 
   }
+
   def main(args: Array[String]): Unit = {
 
     val log: Logger = LoggerFactory.getLogger(getClass)
@@ -76,9 +77,9 @@ object SparkDownloadEBILinks {
 
     import spark.implicits._
 
-    implicit  val PMEncoder: Encoder[PMArticle] = Encoders.kryo(classOf[PMArticle])
-    implicit  val PMJEncoder: Encoder[PMJournal] = Encoders.kryo(classOf[PMJournal])
-    implicit  val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
+    implicit val PMEncoder: Encoder[PMArticle] = Encoders.kryo(classOf[PMArticle])
+    implicit val PMJEncoder: Encoder[PMJournal] = Encoders.kryo(classOf[PMJournal])
+    implicit val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
 
     val sourcePath = parser.get("sourcePath")
     log.info(s"sourcePath  -> $sourcePath")
@@ -86,29 +87,29 @@ object SparkDownloadEBILinks {
     log.info(s"workingPath  -> $workingPath")
 
     log.info("Getting max pubmedId where the links have been requested")
-    val links:Dataset[EBILinkItem] = spark.read.load(s"$sourcePath/ebi_links_dataset").as[EBILinkItem]
-    val lastPMIDRequested =links.map(l => l.id).select(max("value")).first.getLong(0)
+    val links: Dataset[EBILinkItem] = spark.read.load(s"$sourcePath/ebi_links_dataset").as[EBILinkItem]
+    val lastPMIDRequested = links.map(l => l.id).select(max("value")).first.getLong(0)
 
     log.info("Retrieving PMID to request links")
     val pubmed = spark.read.load(s"$sourcePath/baseline_dataset").as[PMArticle]
     pubmed.map(p => p.getPmid.toLong).where(s"value > $lastPMIDRequested").write.mode(SaveMode.Overwrite).save(s"$workingPath/id_to_request")
 
-    val pmidToReq:Dataset[Long] = spark.read.load(s"$workingPath/id_to_request").as[Long]
+    val pmidToReq: Dataset[Long] = spark.read.load(s"$workingPath/id_to_request").as[Long]
 
     val total = pmidToReq.count()
 
-    spark.createDataset(pmidToReq.rdd.repartition((total/MAX_ITEM_PER_PARTITION).toInt).map(pmid =>createEBILinks(pmid)).filter(l => l!= null)).write.mode(SaveMode.Overwrite).save(s"$workingPath/links_update")
+    spark.createDataset(pmidToReq.rdd.repartition((total / MAX_ITEM_PER_PARTITION).toInt).map(pmid => createEBILinks(pmid)).filter(l => l != null)).write.mode(SaveMode.Overwrite).save(s"$workingPath/links_update")
 
-    val updates:Dataset[EBILinkItem] =spark.read.load(s"$workingPath/links_update").as[EBILinkItem]
+    val updates: Dataset[EBILinkItem] = spark.read.load(s"$workingPath/links_update").as[EBILinkItem]
 
     links.union(updates).groupByKey(_.id)
-      .reduceGroups{(x,y) =>
-        if (x == null || x.links ==null)
+      .reduceGroups { (x, y) =>
+        if (x == null || x.links == null)
           y
-        if (y ==null || y.links ==null)
+        if (y == null || y.links == null)
           x
         if (x.links.length > y.links.length)
-            x
+          x
         else
           y
       }.map(_._2).write.mode(SaveMode.Overwrite).save(s"$workingPath/links_final")
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkEBILinksToOaf.scala
similarity index 67%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkEBILinksToOaf.scala
index 1924d919e..0db469769 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/ebi/SparkEBILinksToOaf.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkEBILinksToOaf.scala
@@ -1,15 +1,14 @@
-package eu.dnetlib.dhp.sx.graph.ebi
+package eu.dnetllib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.oaf.Oaf
-import eu.dnetlib.dhp.sx.graph.bio
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF.EBILinkItem
+import eu.dnetllib.dhp.sx.bio.BioDBToOAF
+import eu.dnetllib.dhp.sx.bio.BioDBToOAF.EBILinkItem
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
+
 object SparkEBILinksToOaf {
 
   def main(args: Array[String]): Unit = {
@@ -24,17 +23,17 @@ object SparkEBILinksToOaf {
         .appName(SparkEBILinksToOaf.getClass.getSimpleName)
         .master(parser.get("master")).getOrCreate()
 
+
+    import spark.implicits._
     val sourcePath = parser.get("sourcePath")
     log.info(s"sourcePath  -> $sourcePath")
     val targetPath = parser.get("targetPath")
     log.info(s"targetPath  -> $targetPath")
+    implicit val PMEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
 
-    import spark.implicits._
-    implicit  val PMEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
+    val ebLinks: Dataset[EBILinkItem] = spark.read.load(s"${sourcePath}_dataset").as[EBILinkItem].filter(l => l.links != null)
 
-    val ebLinks:Dataset[EBILinkItem] = spark.read.load(s"${sourcePath}_dataset").as[EBILinkItem].filter(l => l.links!= null)
-
-    ebLinks.flatMap(j =>BioDBToOAF.parse_ebi_links(j.links))
+    ebLinks.flatMap(j => BioDBToOAF.parse_ebi_links(j.links))
       .filter(p => BioDBToOAF.EBITargetLinksFilter(p))
       .flatMap(p => BioDBToOAF.convertEBILinksToOaf(p))
       .write.mode(SaveMode.Overwrite).save(targetPath)
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMArticle.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMArticle.java
similarity index 97%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMArticle.java
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMArticle.java
index 211cbcffb..305bb89be 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMArticle.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMArticle.java
@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetllib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 import java.util.ArrayList;
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMAuthor.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMAuthor.java
similarity index 92%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMAuthor.java
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMAuthor.java
index ba69998c5..c89929981 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMAuthor.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMAuthor.java
@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetllib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMGrant.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMGrant.java
similarity index 93%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMGrant.java
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMGrant.java
index 0c3fd4601..7df5dd5f2 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMGrant.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMGrant.java
@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetllib.dhp.sx.bio.pubmed;
 
 public class PMGrant {
 
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMJournal.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMJournal.java
similarity index 94%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMJournal.java
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMJournal.java
index d251354d4..6065416f8 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMJournal.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMJournal.java
@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetllib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMParser.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMParser.scala
similarity index 99%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMParser.scala
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMParser.scala
index 8744bdfb4..8fa226b7d 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMParser.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMParser.scala
@@ -1,4 +1,4 @@
-package eu.dnetlib.dhp.sx.graph.bio.pubmed
+package eu.dnetllib.dhp.sx.bio.pubmed
 
 import scala.xml.MetaData
 import scala.xml.pull.{EvElemEnd, EvElemStart, EvText, XMLEventReader}
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMSubject.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMSubject.java
similarity index 94%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMSubject.java
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMSubject.java
index 354b2cbe5..e6ab61b87 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PMSubject.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PMSubject.java
@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetllib.dhp.sx.bio.pubmed;
 
 public class PMSubject {
 	private String value;
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PubMedToOaf.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PubMedToOaf.scala
similarity index 93%
rename from dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PubMedToOaf.scala
rename to dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PubMedToOaf.scala
index 202eb7b14..a1777a230 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/PubMedToOaf.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/pubmed/PubMedToOaf.scala
@@ -1,11 +1,12 @@
-package eu.dnetlib.dhp.sx.graph.bio.pubmed
+package eu.dnetllib.dhp.sx.bio.pubmed
+
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
 import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf._
 import eu.dnetlib.dhp.schema.oaf.utils.{GraphCleaningFunctions, IdentifierFactory, OafMapperUtils, PidType}
+import eu.dnetlib.dhp.schema.oaf._
+import scala.collection.JavaConverters._
 
 import java.util.regex.Pattern
-import scala.collection.JavaConverters._
 
 object PubMedToOaf {
 
@@ -15,7 +16,7 @@ object PubMedToOaf {
     "doi" -> "https://dx.doi.org/"
   )
 
-  def cleanDoi(doi:String):String = {
+  def cleanDoi(doi: String): String = {
 
     val regex = "^10.\\d{4,9}\\/[\\[\\]\\-\\<\\>._;()\\/:A-Z0-9]+$"
 
@@ -71,14 +72,14 @@ object PubMedToOaf {
     if (article.getPublicationTypes == null)
       return null
     val i = new Instance
-    var pidList: List[StructuredProperty] = List(OafMapperUtils.structuredProperty(article.getPmid, PidType.pmid.toString, PidType.pmid.toString, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, dataInfo))
+    val pidList: List[StructuredProperty] = List(OafMapperUtils.structuredProperty(article.getPmid, PidType.pmid.toString, PidType.pmid.toString, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, dataInfo))
     if (pidList == null)
       return null
 
-    var alternateIdentifier :StructuredProperty = null
+    var alternateIdentifier: StructuredProperty = null
     if (article.getDoi != null) {
       val normalizedPid = cleanDoi(article.getDoi)
-      if (normalizedPid!= null)
+      if (normalizedPid != null)
         alternateIdentifier = OafMapperUtils.structuredProperty(normalizedPid, PidType.doi.toString, PidType.doi.toString, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, dataInfo)
     }
 
@@ -102,10 +103,10 @@ object PubMedToOaf {
       return result
     result.setDataInfo(dataInfo)
     i.setPid(pidList.asJava)
-    if (alternateIdentifier!= null)
+    if (alternateIdentifier != null)
       i.setAlternateIdentifier(List(alternateIdentifier).asJava)
     result.setInstance(List(i).asJava)
-    i.getPid.asScala.filter(p => "pmid".equalsIgnoreCase(p.getQualifier.getClassid)).map(p => p.getValue)(collection breakOut)
+    i.getPid.asScala.filter(p => "pmid".equalsIgnoreCase(p.getQualifier.getClassid)).map(p => p.getValue)(collection.breakOut)
     val urlLists: List[String] = pidList
       .map(s => (urlMap.getOrElse(s.getQualifier.getClassid, ""), s.getValue))
       .filter(t => t._1.nonEmpty)
@@ -136,7 +137,7 @@ object PubMedToOaf {
     }
 
 
-    val subjects: List[StructuredProperty] = article.getSubjects.asScala.map(s => OafMapperUtils.structuredProperty(s.getValue, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, dataInfo))(collection breakOut)
+    val subjects: List[StructuredProperty] = article.getSubjects.asScala.map(s => OafMapperUtils.structuredProperty(s.getValue, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, dataInfo))(collection.breakOut)
     if (subjects != null)
       result.setSubject(subjects.asJava)
 
@@ -148,7 +149,7 @@ object PubMedToOaf {
       author.setFullname(a.getFullName)
       author.setRank(index + 1)
       author
-    }(collection breakOut)
+    }(collection.breakOut)
 
 
     if (authors != null && authors.nonEmpty)
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/baseline_to_oaf_params.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/baseline_to_oaf_params.json
rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_download_update.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/ebi_download_update.json
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_download_update.json
rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/ebi_download_update.json
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_to_df_params.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/ebi_to_df_params.json
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/ebi/ebi_to_df_params.json
rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/ebi_to_df_params.json
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/pubmed/oozie_app/config-default.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/config-default.xml
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/pubmed/oozie_app/config-default.xml
rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/config-default.xml
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/pubmed/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
similarity index 78%
rename from dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/pubmed/oozie_app/workflow.xml
rename to dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
index 914d1c2c7..f5a98ba5e 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/pubmed/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
@@ -1,13 +1,9 @@
-    <workflow-app name="Transform_Pubmed_Workflow" xmlns="uri:oozie:workflow:0.5">
+    <workflow-app name="Download_Transform_Pubmed_Workflow" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
             <name>baselineWorkingPath</name>
             <description>the Baseline Working Path</description>
         </property>
-        <property>
-            <name>targetPath</name>
-            <description>the Target Path</description>
-        </property>
         <property>
             <name>isLookupUrl</name>
             <description>The IS lookUp service endopoint</description>
@@ -24,8 +20,8 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Convert Baseline to Dataset</name>
-            <class>eu.dnetlib.dhp.sx.graph.ebi.SparkCreateBaselineDataFrame</class>
+            <name>Convert Baseline to OAF Dataset</name>
+            <class>eu.dnetllib.dhp.sx.bio.ebi.SparkCreateBaselineDataFrame</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -38,9 +34,10 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
             <arg>--workingPath</arg><arg>${baselineWorkingPath}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}</arg>
+            <arg>--targetPath</arg><arg>${baselineWorkingPath}/transformed</arg>
             <arg>--master</arg><arg>yarn</arg>
             <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
+            <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/BioScholixTest.scala b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetllib/dhp/sx/bio/BioScholixTest.scala
similarity index 92%
rename from dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/BioScholixTest.scala
rename to dhp-workflows/dhp-aggregation/src/test/java/eu/dnetllib/dhp/sx/bio/BioScholixTest.scala
index 87279eb21..c072f149c 100644
--- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/sx/graph/bio/pubmed/BioScholixTest.scala
+++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetllib/dhp/sx/bio/BioScholixTest.scala
@@ -1,13 +1,10 @@
-package eu.dnetlib.dhp.sx.graph.bio.pubmed
+package eu.dnetllib.dhp.sx.bio
 
 import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
-import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.utils.{CleaningFunctions, OafMapperUtils, PidType}
+import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest
 import eu.dnetlib.dhp.schema.oaf.{Oaf, Relation, Result}
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF.ScholixResolved
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF
-import eu.dnetlib.dhp.sx.graph.bio.pubmed.PubMedToOaf.dataInfo
-import eu.dnetlib.dhp.sx.graph.ebi.SparkDownloadEBILinks
+import eu.dnetllib.dhp.sx.bio.BioDBToOAF.ScholixResolved
+import eu.dnetllib.dhp.sx.bio.pubmed.{PMArticle, PMParser, PubMedToOaf}
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.parse
@@ -55,7 +52,7 @@ class BioScholixTest extends AbstractVocabularyTest{
 
   @Test
   def testEBIData() = {
-    val inputXML = Source.fromInputStream(getClass.getResourceAsStream("pubmed.xml")).mkString
+    val inputXML = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")).mkString
     val xml = new XMLEventReader(Source.fromBytes(inputXML.getBytes()))
     new PMParser(xml).foreach(s =>println(mapper.writeValueAsString(s)))
   }
@@ -65,7 +62,7 @@ class BioScholixTest extends AbstractVocabularyTest{
   def testPubmedToOaf(): Unit = {
     assertNotNull(vocabularies)
     assertTrue(vocabularies.vocabularyExists("dnet:publication_resource"))
-    val records:String =Source.fromInputStream(getClass.getResourceAsStream("pubmed_dump")).mkString
+    val records:String =Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed_dump")).mkString
     val r:List[Oaf] = records.lines.toList.map(s=>mapper.readValue(s, classOf[PMArticle])).map(a => PubMedToOaf.convert(a, vocabularies))
     assertEquals(10, r.size)
     assertTrue(r.map(p => p.asInstanceOf[Result]).flatMap(p => p.getInstance().asScala.map(i => i.getInstancetype.getClassid)).exists(p => "0037".equalsIgnoreCase(p)))
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/crossref_links b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/crossref_links
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/crossref_links
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/crossref_links
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/ebi_links.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/ebi_links.gz
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/ebi_links.gz
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/ebi_links.gz
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/ls_result b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/ls_result
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/ls_result
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/ls_result
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pdb_dump b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pdb_dump
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pdb_dump
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pdb_dump
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/pubmed.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/pubmed.xml
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/pubmed_dump b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed_dump
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/pubmed_dump
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed_dump
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/scholix_resolved b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/scholix_resolved
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/scholix_resolved
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/scholix_resolved
diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/uniprot_dump b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/uniprot_dump
similarity index 100%
rename from dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/uniprot_dump
rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/uniprot_dump
diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java
index 0d7c74475..23e97a97a 100644
--- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java
+++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java
@@ -69,7 +69,7 @@ public class PropagationConstant {
 					PROPAGATION_DATA_INFO_TYPE,
 					PROPAGATION_COUNTRY_INSTREPO_CLASS_ID,
 					PROPAGATION_COUNTRY_INSTREPO_CLASS_NAME,
-						ModelConstants.DNET_PROVENANCE_ACTIONS));
+					ModelConstants.DNET_PROVENANCE_ACTIONS));
 		return nc;
 	}
 
@@ -84,7 +84,8 @@ public class PropagationConstant {
 		return di;
 	}
 
-	public static Qualifier getQualifier(String inference_class_id, String inference_class_name, String qualifierSchema) {
+	public static Qualifier getQualifier(String inference_class_id, String inference_class_name,
+		String qualifierSchema) {
 		Qualifier pa = new Qualifier();
 		pa.setClassid(inference_class_id);
 		pa.setClassname(inference_class_name);
@@ -108,7 +109,11 @@ public class PropagationConstant {
 		r.setRelClass(rel_class);
 		r.setRelType(rel_type);
 		r.setSubRelType(subrel_type);
-		r.setDataInfo(getDataInfo(inference_provenance, inference_class_id, inference_class_name, ModelConstants.DNET_PROVENANCE_ACTIONS));
+		r
+			.setDataInfo(
+				getDataInfo(
+					inference_provenance, inference_class_id, inference_class_name,
+					ModelConstants.DNET_PROVENANCE_ACTIONS));
 		return r;
 	}
 
diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/SparkOrcidToResultFromSemRelJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/SparkOrcidToResultFromSemRelJob.java
index 68949b900..a38b4da2e 100644
--- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/SparkOrcidToResultFromSemRelJob.java
+++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/SparkOrcidToResultFromSemRelJob.java
@@ -173,14 +173,17 @@ public class SparkOrcidToResultFromSemRelJob {
 		if (toaddpid) {
 			StructuredProperty p = new StructuredProperty();
 			p.setValue(autoritative_author.getOrcid());
-			p.setQualifier(getQualifier(ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, ModelConstants.DNET_PID_TYPES));
+			p
+				.setQualifier(
+					getQualifier(
+						ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, ModelConstants.DNET_PID_TYPES));
 			p
 				.setDataInfo(
 					getDataInfo(
 						PROPAGATION_DATA_INFO_TYPE,
 						PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_ID,
 						PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_NAME,
-							ModelConstants.DNET_PROVENANCE_ACTIONS));
+						ModelConstants.DNET_PROVENANCE_ACTIONS));
 
 			Optional<List<StructuredProperty>> authorPid = Optional.ofNullable(author.getPid());
 			if (authorPid.isPresent()) {
diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java
index 1289ff644..50df08f8c 100644
--- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java
+++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java
@@ -10,7 +10,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import eu.dnetlib.dhp.schema.common.ModelConstants;
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.function.MapFunction;
@@ -22,6 +21,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.oaf.Context;
 import eu.dnetlib.dhp.schema.oaf.Result;
 import scala.Tuple2;
@@ -130,7 +130,7 @@ public class SparkResultToCommunityFromOrganizationJob {
 											PROPAGATION_DATA_INFO_TYPE,
 											PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_ID,
 											PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME,
-												ModelConstants.DNET_PROVENANCE_ACTIONS)));
+											ModelConstants.DNET_PROVENANCE_ACTIONS)));
 						propagatedContexts.add(newContext);
 					}
 				}
diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/SparkResultToCommunityThroughSemRelJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/SparkResultToCommunityThroughSemRelJob.java
index 7f76ead94..f31a26230 100644
--- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/SparkResultToCommunityThroughSemRelJob.java
+++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/SparkResultToCommunityThroughSemRelJob.java
@@ -7,7 +7,6 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
 import java.util.*;
 import java.util.stream.Collectors;
 
-import eu.dnetlib.dhp.schema.common.ModelConstants;
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.function.MapFunction;
@@ -20,6 +19,7 @@ import org.slf4j.LoggerFactory;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.oaf.*;
 import scala.Tuple2;
 
@@ -126,7 +126,7 @@ public class SparkResultToCommunityThroughSemRelJob {
 													PROPAGATION_DATA_INFO_TYPE,
 													PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_ID,
 													PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_NAME,
-														ModelConstants.DNET_PROVENANCE_ACTIONS)));
+													ModelConstants.DNET_PROVENANCE_ACTIONS)));
 								return newContext;
 							}
 							return null;
diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/pangaea/SparkGeneratePanagaeaDataset.scala b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/pangaea/SparkGeneratePanagaeaDataset.scala
index bf726cf59..79c75d6df 100644
--- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/pangaea/SparkGeneratePanagaeaDataset.scala
+++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/pangaea/SparkGeneratePanagaeaDataset.scala
@@ -1,7 +1,6 @@
 package eu.dnetlib.dhp.sx.graph.pangaea
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.sx.graph.ebi.SparkEBILinksToOaf
 import org.apache.spark.rdd.RDD
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java
index 1c7dce3f2..64935e79d 100644
--- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java
+++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java
@@ -84,13 +84,15 @@ public class IndexRecordTransformerTest {
 
 	@Test
 	public void testForEOSCFutureTraining() throws IOException, TransformerException {
-		final String record = IOUtils.toString(getClass().getResourceAsStream("eosc-future/training-notebooks-seadatanet.xml"));
+		final String record = IOUtils
+			.toString(getClass().getResourceAsStream("eosc-future/training-notebooks-seadatanet.xml"));
 		testRecordTransformation(record);
 	}
 
 	@Test
 	public void testForEOSCFutureAirQualityCopernicus() throws IOException, TransformerException {
-		final String record = IOUtils.toString(getClass().getResourceAsStream("eosc-future/air-quality-copernicus.xml"));
+		final String record = IOUtils
+			.toString(getClass().getResourceAsStream("eosc-future/air-quality-copernicus.xml"));
 		testRecordTransformation(record);
 	}
 
@@ -102,12 +104,11 @@ public class IndexRecordTransformerTest {
 
 	@Test
 	public void testForEOSCFutureB2SharePlotRelatedORP() throws IOException, TransformerException {
-		final String record = IOUtils.toString(getClass().getResourceAsStream("eosc-future/b2share-plot-related-orp.xml"));
+		final String record = IOUtils
+			.toString(getClass().getResourceAsStream("eosc-future/b2share-plot-related-orp.xml"));
 		testRecordTransformation(record);
 	}
 
-
-
 	private void testRecordTransformation(final String record) throws IOException, TransformerException {
 		final String fields = IOUtils.toString(getClass().getResourceAsStream("fields.xml"));
 		final String xslt = IOUtils.toString(getClass().getResourceAsStream("layoutToRecordTransformer.xsl"));

From 511da98d0ce3c4f7c046799d18f7398367c755fa Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Tue, 12 Oct 2021 11:47:49 +0200
Subject: [PATCH 07/12] - fixed bug on download pmc Article - removed unused
 line of code in SparkCreateActionset

---
 .../dhp/actionmanager/scholix/SparkCreateActionset.scala  | 4 ----
 .../dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala     | 8 ++------
 .../eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml   | 8 ++++++--
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/scholix/SparkCreateActionset.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/scholix/SparkCreateActionset.scala
index b78f411ee..7a87861db 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/scholix/SparkCreateActionset.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/scholix/SparkCreateActionset.scala
@@ -60,14 +60,10 @@ object SparkCreateActionset {
 
     val entities: Dataset[(String, Result)] = spark.read.load(s"$sourcePath/entities/*").as[Result].map(p => (p.getId, p))(Encoders.tuple(Encoders.STRING, resultEncoders))
 
-
-    entities.filter(r => r.isInstanceOf[Result]).map(r => r.asInstanceOf[Result])
     entities
       .joinWith(idRelation, entities("_1").equalTo(idRelation("value")))
       .map(p => p._1._2)
       .write.mode(SaveMode.Append).save(s"$workingDirFolder/actionSetOaf")
-
-
   }
 
 }
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
index 97b3cdc99..2fc9623a8 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
@@ -114,11 +114,7 @@ object SparkCreateBaselineDataFrame {
       val hdfsWritePath: Path = new Path(s"$baselinePath/${u._1}")
       val fsDataOutputStream: FSDataOutputStream = fs.create(hdfsWritePath, true)
       val i = downloadBaselinePart(u._2)
-      val buffer = Array.fill[Byte](1024)(0)
-      while (i.read(buffer) > 0) {
-        fsDataOutputStream.write(buffer)
-      }
-      i.close()
+      IOUtils.copy(i, fsDataOutputStream)
       println(s"Downloaded ${u._2} into $baselinePath/${u._1}")
       fsDataOutputStream.close()
     }
@@ -182,7 +178,7 @@ object SparkCreateBaselineDataFrame {
 
     downloadBaseLineUpdate(s"$workingPath/baseline", hdfsServerUri)
 
-    val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline_ftp", 2000)
+    val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline", 2000)
     val ds: Dataset[PMArticle] = spark.createDataset(k.filter(i => i._1.endsWith(".gz")).flatMap(i => {
       val xml = new XMLEventReader(Source.fromBytes(i._2.getBytes()))
       new PMParser(xml)
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
index f5a98ba5e..4ed6dd8bf 100644
--- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
@@ -8,6 +8,10 @@
             <name>isLookupUrl</name>
             <description>The IS lookUp service endopoint</description>
         </property>
+        <property>
+            <name>targetPath</name>
+            <description>The target path</description>
+        </property>
     </parameters>
 
     <start to="ConvertDataset"/>
@@ -22,7 +26,7 @@
             <mode>cluster</mode>
             <name>Convert Baseline to OAF Dataset</name>
             <class>eu.dnetllib.dhp.sx.bio.ebi.SparkCreateBaselineDataFrame</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
@@ -34,7 +38,7 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
             <arg>--workingPath</arg><arg>${baselineWorkingPath}</arg>
-            <arg>--targetPath</arg><arg>${baselineWorkingPath}/transformed</arg>
+            <arg>--targetPath</arg><arg>${targetPath}</arg>
             <arg>--master</arg><arg>yarn</arg>
             <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
             <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>

From 914b3e92cb2301fb86a8b5c1942cc01a9cb90d0f Mon Sep 17 00:00:00 2001
From: Claudio Atzori <claudio.atzori@isti.cnr.it>
Date: Tue, 12 Oct 2021 12:00:45 +0200
Subject: [PATCH 08/12] updating graph schema module dependency to version
 2.8.20 to include organization parent/child relation constants

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 61b0ad873..3e8d6bc19 100644
--- a/pom.xml
+++ b/pom.xml
@@ -753,7 +753,7 @@
 		<mockito-core.version>3.3.3</mockito-core.version>
 		<mongodb.driver.version>3.4.2</mongodb.driver.version>
 		<vtd.version>[2.12,3.0)</vtd.version>
-		<dhp-schemas.version>[2.7.18]</dhp-schemas.version>
+		<dhp-schemas.version>[2.8.20]</dhp-schemas.version>
 		<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
 		<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
 		<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>

From 7387416e909bad95b95b2303f5b6bdff76e47749 Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@isti.cnr.it>
Date: Tue, 12 Oct 2021 12:36:30 +0200
Subject: [PATCH 09/12] added params skip update to direct transform in OAF,
 this should be set to true in production

---
 .../ebi/SparkCreateBaselineDataFrame.scala    | 28 ++++++++++---------
 .../sx/bio/ebi/baseline_to_oaf_params.json    |  1 +
 .../dhp/sx/bio/pubmed/oozie_app/workflow.xml  |  6 ++++
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
index 2fc9623a8..17bf3fa6b 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetllib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala
@@ -32,7 +32,7 @@ object SparkCreateBaselineDataFrame {
       val start = l.indexOf("<a href=\"")
 
       if (start >= 0 && end > start)
-        l.substring(start + 9, (end - start))
+        l.substring(start + 9, end - start)
       else
         ""
     }.filter(s => s.endsWith(".gz")).filter(s => s > maxFile).map(s => (s, s"https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/$s")).toList
@@ -158,6 +158,9 @@ object SparkCreateBaselineDataFrame {
     val hdfsServerUri = parser.get("hdfsServerUri")
     log.info("hdfsServerUri: {}", targetPath)
 
+    val skipUpdate = parser.get("skipUpdate")
+    log.info("skipUpdate: {}", skipUpdate)
+
 
     val isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl)
     val vocabularies = VocabularyGroup.loadVocsFromIS(isLookupService)
@@ -176,18 +179,17 @@ object SparkCreateBaselineDataFrame {
     implicit val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
     implicit val resultEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
 
-    downloadBaseLineUpdate(s"$workingPath/baseline", hdfsServerUri)
-
-    val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline", 2000)
-    val ds: Dataset[PMArticle] = spark.createDataset(k.filter(i => i._1.endsWith(".gz")).flatMap(i => {
-      val xml = new XMLEventReader(Source.fromBytes(i._2.getBytes()))
-      new PMParser(xml)
-
-    }))
-
-    ds.map(p => (p.getPmid, p))(Encoders.tuple(Encoders.STRING, PMEncoder)).groupByKey(_._1)
-      .agg(pmArticleAggregator.toColumn)
-      .map(p => p._2).write.mode(SaveMode.Overwrite).save(s"$workingPath/baseline_dataset")
+    if (!"true".equalsIgnoreCase(skipUpdate)) {
+      downloadBaseLineUpdate(s"$workingPath/baseline", hdfsServerUri)
+      val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline", 2000)
+      val ds: Dataset[PMArticle] = spark.createDataset(k.filter(i => i._1.endsWith(".gz")).flatMap(i => {
+        val xml = new XMLEventReader(Source.fromBytes(i._2.getBytes()))
+        new PMParser(xml)
+      }))
+      ds.map(p => (p.getPmid, p))(Encoders.tuple(Encoders.STRING, PMEncoder)).groupByKey(_._1)
+        .agg(pmArticleAggregator.toColumn)
+        .map(p => p._2).write.mode(SaveMode.Overwrite).save(s"$workingPath/baseline_dataset")
+    }
 
     val exported_dataset = spark.read.load(s"$workingPath/baseline_dataset").as[PMArticle]
     exported_dataset
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json
index 4bee770bd..8dc8a2aae 100644
--- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json
@@ -3,5 +3,6 @@
   {"paramName":"i",   "paramLongName":"isLookupUrl",    "paramDescription": "isLookupUrl",                              "paramRequired": true},
   {"paramName":"w",   "paramLongName":"workingPath",    "paramDescription": "the path of the sequencial file to read",  "paramRequired": true},
   {"paramName":"t",   "paramLongName":"targetPath",     "paramDescription": "the oaf path ",                            "paramRequired": true},
+  {"paramName":"s",   "paramLongName":"skipUpdate",     "paramDescription": "skip update ",                             "paramRequired": false},
   {"paramName":"h",   "paramLongName":"hdfsServerUri",  "paramDescription": "the  working path ",                       "paramRequired": true}
 ]
\ No newline at end of file
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
index 4ed6dd8bf..21fd2d153 100644
--- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml
@@ -12,6 +12,11 @@
             <name>targetPath</name>
             <description>The target path</description>
         </property>
+        <property>
+            <name>skipUpdate</name>
+            <value>false</value>
+            <description>The request block size</description>
+        </property>
     </parameters>
 
     <start to="ConvertDataset"/>
@@ -42,6 +47,7 @@
             <arg>--master</arg><arg>yarn</arg>
             <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
             <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
+            <arg>--skipUpdate</arg><arg>${skipUpdate}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>

From 16b28494a940e5be42874ba5ef1e91778d6bd2bd Mon Sep 17 00:00:00 2001
From: "miriam.baglioni" <miriam.baglioni@isti.cnr.it>
Date: Wed, 13 Oct 2021 11:34:24 +0200
Subject: [PATCH 10/12] added new parameter in the doiboost process workflow to
 specify a folder for the process of MAG dataset

---
 .../dnetlib/dhp/doiboost/process/oozie_app/workflow.xml   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
index eb82c3a7d..282c06f4e 100644
--- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
@@ -48,7 +48,11 @@
         <!--    MAG Parameters    -->
         <property>
             <name>inputPathMAG</name>
-            <description>the MAG working path</description>
+            <description>the MAG input path</description>
+        </property>
+        <property>
+            <name>workingPathMAG</name>
+            <description>the MAG working path where to store the intermediate process results</description>
         </property>
 
 
@@ -138,7 +142,7 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${inputPathMAG}/dataset</arg>
-            <arg>--workingPath</arg><arg>${inputPathMAG}/process_p</arg>
+            <arg>--workingPath</arg><arg>${workingPathMAG}</arg>
             <arg>--targetPath</arg><arg>${workingPath}</arg>
             <arg>--master</arg><arg>yarn-cluster</arg>
         </spark>

From 5d9cc2452dc76452fbb45f326315ec5061f2feae Mon Sep 17 00:00:00 2001
From: "miriam.baglioni" <miriam.baglioni@isti.cnr.it>
Date: Wed, 13 Oct 2021 15:33:50 +0200
Subject: [PATCH 11/12] changed the working path parameter value as dependant
 from the dnet-workflow working dir parameter

---
 .../eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml  | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
index 282c06f4e..29a12f4df 100644
--- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml
@@ -50,10 +50,6 @@
             <name>inputPathMAG</name>
             <description>the MAG input path</description>
         </property>
-        <property>
-            <name>workingPathMAG</name>
-            <description>the MAG working path where to store the intermediate process results</description>
-        </property>
 
 
         <!--    UnpayWall Parameters    -->
@@ -142,7 +138,7 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${inputPathMAG}/dataset</arg>
-            <arg>--workingPath</arg><arg>${workingPathMAG}</arg>
+            <arg>--workingPath</arg><arg>${workingPath}/MAG</arg>
             <arg>--targetPath</arg><arg>${workingPath}</arg>
             <arg>--master</arg><arg>yarn-cluster</arg>
         </spark>

From b292e4a700eed70b8a94542245610b88f53e9ac3 Mon Sep 17 00:00:00 2001
From: Claudio Atzori <claudio.atzori@isti.cnr.it>
Date: Wed, 13 Oct 2021 17:31:53 +0200
Subject: [PATCH 12/12] [stats wf] added extra logging in the context data
 retrieval phase

---
 .../eu/dnetlib/dhp/oa/graph/stats/oozie_app/contexts.sh     | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/contexts.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/contexts.sh
index 6d42ab13d..c60d67b1a 100644
--- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/contexts.sh
+++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/contexts.sh
@@ -11,9 +11,13 @@ TARGET_DB=$2
 
 TMP=/tmp/stats-update-`tr -dc A-Za-z0-9 </dev/urandom | head -c 6`
 
-echo "Downloading context data"
+echo "Downloading context ids"
 curl -L ${CONTEXT_API}/contexts/?type=ri,community -H "accept: application/json" | /usr/local/sbin/jq -r '.[] | "\(.id),\(.label)"' > contexts.csv
+
+echo "Downloading categories data"
 cat contexts.csv | cut -d , -f1 | xargs -I {} curl -L ${CONTEXT_API}/context/{}/?all=true | /usr/local/sbin/jq -r '.[]|"\(.id|split(":")[0]),\(.id),\(.label)"' > categories.csv
+
+echo "Downloading concepts data"
 cat categories.csv | cut -d , -f2 | sed 's/:/%3A/g'| xargs -I {} curl -L ${CONTEXT_API}/context/category/{}/?all=true | /usr/local/sbin/jq -r '.[]|"\(.id|split("::")[0])::\(.id|split("::")[1]),\(.id),\(.label)"' > concepts.csv
 cat contexts.csv | sed 's/^\(.*\),\(.*\)/\1,\1::other,\2/' >> categories.csv
 cat categories.csv | sed 's/^.*,\(.*\),\(.*\)/\1,\1::other,\2/' >> concepts.csv