mergin with branch beta

dhp-common/src/main/java/eu/dnetlib/dhp/common/HdfsSupport.java

@@ -28,7 +28,7 @@ public class HdfsSupport {
 	 * @param configuration Configuration of hadoop env
 	 */
 	public static boolean exists(String path, Configuration configuration) {
-		logger.info("Removing path: {}", path);
+		logger.info("Checking existence for path: {}", path);
 		return rethrowAsRuntimeException(
 			() -> {
 				Path f = new Path(path);

dhp-common/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java

@@ -85,6 +85,13 @@ public class MakeTarArchive implements Serializable {
 		String p_string = p.toString();
 		if (!p_string.endsWith("_SUCCESS")) {
 			String name = p_string.substring(p_string.lastIndexOf("/") + 1);
+			if (name.startsWith("part-") & name.length() > 10) {
+				String tmp = name.substring(0, 10);
+				if (name.contains(".")) {
+					tmp += name.substring(name.indexOf("."));
+				}
+				name = tmp;
+			}
 			TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
 			entry.setSize(fileStatus.getLen());
 			current_size += fileStatus.getLen();

dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java

@@ -4,19 +4,19 @@ package eu.dnetlib.dhp.utils;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
-import java.util.List;
+import java.util.*;
-import java.util.Map;
+import java.util.stream.Collectors;
-import java.util.Properties;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
 
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Base64OutputStream;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.SaveMode;
 import org.slf4j.Logger;
@@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Maps;
 import com.jayway.jsonpath.JsonPath;
 
+import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
 import net.minidev.json.JSONArray;
 import scala.collection.JavaConverters;
 import scala.collection.Seq;
@@ -52,10 +54,56 @@ public class DHPUtils {
 		}
 	}
 
+	/**
+	 * Retrieves from the metadata store manager application the list of paths associated with mdstores characterized
+	 * by he given format, layout, interpretation
+	 * @param mdstoreManagerUrl the URL of the mdstore manager service
+	 * @param format the mdstore format
+	 * @param layout the mdstore layout
+	 * @param interpretation the mdstore interpretation
+	 * @param includeEmpty include Empty mdstores
+	 * @return the set of hdfs paths
+	 * @throws IOException in case of HTTP communication issues
+	 */
+	public static Set<String> mdstorePaths(final String mdstoreManagerUrl,
+		final String format,
+		final String layout,
+		final String interpretation,
+		boolean includeEmpty) throws IOException {
+		final String url = mdstoreManagerUrl + "/mdstores/";
+		final ObjectMapper objectMapper = new ObjectMapper();
+
+		final HttpGet req = new HttpGet(url);
+
+		try (final CloseableHttpClient client = HttpClients.createDefault()) {
+			try (final CloseableHttpResponse response = client.execute(req)) {
+				final String json = IOUtils.toString(response.getEntity().getContent());
+				final MDStoreWithInfo[] mdstores = objectMapper.readValue(json, MDStoreWithInfo[].class);
+				return Arrays
+					.stream(mdstores)
+					.filter(md -> md.getFormat().equalsIgnoreCase(format))
+					.filter(md -> md.getLayout().equalsIgnoreCase(layout))
+					.filter(md -> md.getInterpretation().equalsIgnoreCase(interpretation))
+					.filter(md -> StringUtils.isNotBlank(md.getHdfsPath()))
+					.filter(md -> StringUtils.isNotBlank(md.getCurrentVersion()))
+					.filter(md -> includeEmpty || md.getSize() > 0)
+					.map(md -> md.getHdfsPath() + "/" + md.getCurrentVersion() + "/store")
+					.collect(Collectors.toSet());
+			}
+		}
+	}
+
 	public static String generateIdentifier(final String originalId, final String nsPrefix) {
 		return String.format("%s::%s", nsPrefix, DHPUtils.md5(originalId));
 	}
 
+	public static String generateUnresolvedIdentifier(final String pid, final String pidType) {
+
+		final String cleanedPid = CleaningFunctions.normalizePidValue(pidType, pid);
+
+		return String.format("unresolved::%s::%s", cleanedPid, pidType.toLowerCase().trim());
+	}
+
 	public static String getJPathString(final String jsonPath, final String json) {
 		try {
 			Object o = JsonPath.read(json, jsonPath);

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/ExportActionSetJobNode.scala

@@ -1,41 +0,0 @@
-package eu.dnetlib.dhp.actionmanager.datacite
-
-import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.Oaf
-import org.apache.hadoop.io.Text
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.hadoop.mapred.SequenceFileOutputFormat
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.io.Source
-
-object ExportActionSetJobNode {
-
-  val log: Logger = LoggerFactory.getLogger(ExportActionSetJobNode.getClass)
-
-  def main(args: Array[String]): Unit = {
-    val conf = new SparkConf
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/exportDataset_parameters.json")).mkString)
-    parser.parseArgument(args)
-    val master = parser.get("master")
-    val sourcePath = parser.get("sourcePath")
-    val targetPath = parser.get("targetPath")
-
-    val spark: SparkSession = SparkSession.builder().config(conf)
-      .appName(ExportActionSetJobNode.getClass.getSimpleName)
-      .master(master)
-      .getOrCreate()
-    implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
-    implicit val tEncoder:Encoder[(String,String)] = Encoders.tuple(Encoders.STRING,Encoders.STRING)
-
-    spark.read.load(sourcePath).as[Oaf]
-      .map(o =>DataciteToOAFTransformation.toActionSet(o))
-      .filter(o => o!= null)
-      .rdd.map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$targetPath", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text,Text]], classOf[GzipCodec])
-
-
-  }
-
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/FilterCrossrefEntitiesSpark.scala

@@ -1,46 +0,0 @@
-package eu.dnetlib.dhp.actionmanager.datacite
-
-import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
-import eu.dnetlib.dhp.schema.mdstore.MetadataRecord
-import eu.dnetlib.dhp.schema.oaf.{Oaf, Result}
-import eu.dnetlib.dhp.utils.ISLookupClientFactory
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.io.Source
-
-object FilterCrossrefEntitiesSpark {
-
-  val log: Logger = LoggerFactory.getLogger(getClass.getClass)
-
-  def main(args: Array[String]): Unit = {
-    val conf = new SparkConf
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/filter_crossref_param.json")).mkString)
-    parser.parseArgument(args)
-    val master = parser.get("master")
-    val sourcePath = parser.get("sourcePath")
-    log.info("sourcePath: {}", sourcePath)
-    val targetPath = parser.get("targetPath")
-    log.info("targetPath: {}", targetPath)
-
-
-
-    val spark: SparkSession = SparkSession.builder().config(conf)
-      .appName(getClass.getSimpleName)
-      .master(master)
-      .getOrCreate()
-
-
-
-    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
-    implicit val resEncoder: Encoder[Result] = Encoders.kryo[Result]
-
-    val d:Dataset[Oaf]= spark.read.load(sourcePath).as[Oaf]
-
-    d.filter(r => r.isInstanceOf[Result]).map(r => r.asInstanceOf[Result]).write.mode(SaveMode.Overwrite).save(targetPath)
-
-  }
-
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectionUtils.scala

@@ -0,0 +1,49 @@
+package eu.dnetlib.dhp.collection
+
+import eu.dnetlib.dhp.schema.common.ModelSupport
+import eu.dnetlib.dhp.schema.oaf.{Oaf, OafEntity, Relation}
+
+object CollectionUtils {
+
+  /**
+   * This method in pipeline to the transformation phase,
+   * generates relations in both verse, typically it should be a phase of flatMap
+   *
+   * @param i input OAF
+   * @return
+   * If the input OAF is an entity -> List(i)
+   * If the input OAF is a relation -> List(relation, inverseRelation)
+   *
+   */
+
+  def fixRelations(i: Oaf): List[Oaf] = {
+    if (i.isInstanceOf[OafEntity])
+      return List(i)
+    else {
+      val r: Relation = i.asInstanceOf[Relation]
+      val currentRel = ModelSupport.findRelation(r.getRelClass)
+      if (currentRel != null) {
+
+        // Cleaning relation
+        r.setRelType(currentRel.getRelType)
+        r.setSubRelType(currentRel.getSubReltype)
+        r.setRelClass(currentRel.getRelClass)
+        val inverse = new Relation
+        inverse.setSource(r.getTarget)
+        inverse.setTarget(r.getSource)
+        inverse.setRelType(currentRel.getRelType)
+        inverse.setSubRelType(currentRel.getSubReltype)
+        inverse.setRelClass(currentRel.getInverseRelClass)
+        inverse.setCollectedfrom(r.getCollectedfrom)
+        inverse.setDataInfo(r.getDataInfo)
+        inverse.setProperties(r.getProperties)
+        inverse.setLastupdatetimestamp(r.getLastupdatetimestamp)
+        inverse.setValidated(r.getValidated)
+        inverse.setValidationDate(r.getValidationDate)
+        return List(r, inverse)
+      }
+    }
+    List()
+  }
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/AbstractRestClient.scala

@@ -1,12 +1,10 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
 import org.apache.commons.io.IOUtils
 import org.apache.http.client.config.RequestConfig
-import org.apache.http.client.methods.{HttpGet, HttpPost, HttpRequestBase, HttpUriRequest}
+import org.apache.http.client.methods.{HttpGet, HttpPost, HttpUriRequest}
 import org.apache.http.entity.StringEntity
-import org.apache.http.impl.client.{HttpClientBuilder, HttpClients}
+import org.apache.http.impl.client.HttpClientBuilder
-
-import java.io.IOException
 
 
 abstract class AbstractRestClient extends Iterator[String] {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/DataciteAPIImporter.scala

@@ -1,7 +1,7 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
-import org.json4s.{DefaultFormats, JValue}
 import org.json4s.jackson.JsonMethods.{compact, parse, render}
+import org.json4s.{DefaultFormats, JValue}
 
 class DataciteAPIImporter(timestamp: Long = 0, blocks: Long = 10, until:Long = -1) extends AbstractRestClient {
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala

@@ -1,4 +1,4 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
@@ -325,8 +325,9 @@ object DataciteToOAFTransformation {
       val grantId = m.matcher(awardUri).replaceAll("$2")
       val targetId = s"$p${DHPUtils.md5(grantId)}"
       List(
-        generateRelation(sourceId, targetId, "isProducedBy", DATACITE_COLLECTED_FROM, dataInfo),
+        generateRelation(sourceId, targetId, "isProducedBy", DATACITE_COLLECTED_FROM, dataInfo)
-        generateRelation(targetId, sourceId, "produces", DATACITE_COLLECTED_FROM, dataInfo)
+// REMOVED INVERSE RELATION since there is a specific method that should generate later
+//        generateRelation(targetId, sourceId, "produces", DATACITE_COLLECTED_FROM, dataInfo)
       )
     }
     else
@@ -580,11 +581,11 @@ object DataciteToOAFTransformation {
         rel.setProperties(List(dateProps).asJava)
 
         rel.setSource(id)
-        rel.setTarget(s"unresolved::${r.relatedIdentifier}::${r.relatedIdentifierType}")
+        rel.setTarget(DHPUtils.generateUnresolvedIdentifier(r.relatedIdentifier,r.relatedIdentifierType))
         rel.setCollectedfrom(List(DATACITE_COLLECTED_FROM).asJava)
-        rel.getCollectedfrom.asScala.map(c => c.getValue)(collection.breakOut)
+        rel.getCollectedfrom.asScala.map(c => c.getValue).toList
         rel
-      })(collection breakOut)
+      }).toList
   }
 
   def generateDataInfo(trust: String): DataInfo = {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/GenerateDataciteDatasetSpark.scala

@@ -1,9 +1,14 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.collection.CollectionUtils.fixRelations
+import eu.dnetlib.dhp.common.Constants.MDSTORE_DATA_PATH
+import eu.dnetlib.dhp.common.Constants.MDSTORE_SIZE_PATH
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
-import eu.dnetlib.dhp.schema.mdstore.MetadataRecord
+import eu.dnetlib.dhp.schema.mdstore.{MDStoreVersion, MetadataRecord}
 import eu.dnetlib.dhp.schema.oaf.Oaf
+import eu.dnetlib.dhp.utils.DHPUtils.writeHdfsFile
 import eu.dnetlib.dhp.utils.ISLookupClientFactory
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
@@ -17,11 +22,10 @@ object GenerateDataciteDatasetSpark {
 
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/generate_dataset_params.json")).mkString)
+    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/datacite/generate_dataset_params.json")).mkString)
     parser.parseArgument(args)
     val master = parser.get("master")
     val sourcePath = parser.get("sourcePath")
-    val targetPath = parser.get("targetPath")
     val exportLinks = "true".equalsIgnoreCase(parser.get("exportLinks"))
     val isLookupUrl: String = parser.get("isLookupUrl")
     log.info("isLookupUrl: {}", isLookupUrl)
@@ -33,16 +37,28 @@ object GenerateDataciteDatasetSpark {
       .master(master)
       .getOrCreate()
 
+    import spark.implicits._
+
     implicit val mrEncoder: Encoder[MetadataRecord] = Encoders.kryo[MetadataRecord]
 
     implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
 
-    import spark.implicits._
+    val mdstoreOutputVersion = parser.get("mdstoreOutputVersion")
+    val mapper = new ObjectMapper()
+    val cleanedMdStoreVersion = mapper.readValue(mdstoreOutputVersion, classOf[MDStoreVersion])
+    val outputBasePath = cleanedMdStoreVersion.getHdfsPath
+
+    log.info("outputBasePath: {}", outputBasePath)
+    val targetPath = s"$outputBasePath/$MDSTORE_DATA_PATH"
 
     spark.read.load(sourcePath).as[DataciteType]
       .filter(d => d.isActive)
       .flatMap(d => DataciteToOAFTransformation.generateOAF(d.json, d.timestamp, d.timestamp, vocabularies, exportLinks))
       .filter(d => d != null)
+      .flatMap(i => fixRelations(i)).filter(i => i != null)
       .write.mode(SaveMode.Overwrite).save(targetPath)
+
+    val total_items = spark.read.load(targetPath).as[Oaf].count()
+    writeHdfsFile(spark.sparkContext.hadoopConfiguration, s"$total_items", outputBasePath + MDSTORE_SIZE_PATH)
   }
-}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/ImportDatacite.scala

@@ -1,6 +1,5 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
-import eu.dnetlib.dhp.actionmanager.datacite.DataciteToOAFTransformation.df_it
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, LocalFileSystem, Path}
@@ -9,14 +8,14 @@ import org.apache.hadoop.io.{IntWritable, SequenceFile, Text}
 import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.expressions.Aggregator
+import org.apache.spark.sql.functions.max
 import org.apache.spark.sql.{Dataset, Encoder, SaveMode, SparkSession}
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
-import org.apache.spark.sql.functions.max
 import org.slf4j.{Logger, LoggerFactory}
 
-import java.time.format.DateTimeFormatter._
+import java.time.format.DateTimeFormatter.ISO_DATE_TIME
-import java.time.{LocalDate, LocalDateTime, ZoneOffset}
+import java.time.{LocalDateTime, ZoneOffset}
 import scala.io.Source
 
 object ImportDatacite {
@@ -138,11 +137,11 @@ object ImportDatacite {
     }
   }
 
-  private def writeSequenceFile(hdfsTargetPath: Path, timestamp: Long, conf: Configuration, bs:Int): Long = {
+  private def writeSequenceFile(hdfsTargetPath: Path, timestamp: Long, conf: Configuration, bs: Int): Long = {
-    var from:Long = timestamp * 1000
+    var from: Long = timestamp * 1000
-    val delta:Long = 100000000L
+    val delta: Long = 100000000L
     var client: DataciteAPIImporter = null
-    val now :Long =System.currentTimeMillis()
+    val now: Long = System.currentTimeMillis()
     var i = 0
     try {
       val writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(hdfsTargetPath), SequenceFile.Writer.keyClass(classOf[IntWritable]), SequenceFile.Writer.valueClass(classOf[Text]))
@@ -168,7 +167,7 @@ object ImportDatacite {
               start = System.currentTimeMillis
             }
           }
-          println(s"updating from value: $from  -> ${from+delta}")
+          println(s"updating from value: $from  -> ${from + delta}")
           from = from + delta
         }
       } catch {
@@ -183,4 +182,4 @@ object ImportDatacite {
     i
   }
 
-}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/SparkDownloadUpdateDatacite.scala

@@ -1,18 +1,14 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
-
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.oaf.{Oaf, Result}
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.LocalFileSystem
-import org.apache.hadoop.hdfs.DistributedFileSystem
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Encoder, Encoders, SparkSession}
 import org.apache.spark.sql.functions.max
+import org.apache.spark.sql.{Encoder, Encoders, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
 
 import java.text.SimpleDateFormat
-import java.util.{Date, Locale}
+import java.util.Locale
 import scala.io.Source
 
 object SparkDownloadUpdateDatacite {
@@ -21,7 +17,7 @@ object SparkDownloadUpdateDatacite {
   def main(args: Array[String]): Unit = {
 
     val conf = new SparkConf
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/generate_dataset_params.json")).mkString)
+    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/datacite/generate_dataset_params.json")).mkString)
     parser.parseArgument(args)
     val master = parser.get("master")
     val sourcePath = parser.get("sourcePath")
@@ -42,9 +38,9 @@ object SparkDownloadUpdateDatacite {
     import spark.implicits._
 
 
-    val maxDate:String = spark.read.load(workingPath).as[Oaf].filter(s => s.isInstanceOf[Result]).map(r => r.asInstanceOf[Result].getDateofcollection).select(max("value")).first().getString(0)
+    val maxDate: String = spark.read.load(workingPath).as[Oaf].filter(s => s.isInstanceOf[Result]).map(r => r.asInstanceOf[Result].getDateofcollection).select(max("value")).first().getString(0)
     val ISO8601FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US)
-    val string_to_date =ISO8601FORMAT.parse(maxDate)
+    val string_to_date = ISO8601FORMAT.parse(maxDate)
     val ts = string_to_date.getTime
 
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala

@@ -1,4 +1,4 @@
-package eu.dnetllib.dhp.sx.bio
+package eu.dnetlib.dhp.sx.bio
 
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.{GraphCleaningFunctions, OafMapperUtils}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/SparkTransformBioDatabaseToOAF.scala

@@ -1,8 +1,9 @@
-package eu.dnetllib.dhp.sx.bio
+package eu.dnetlib.dhp.sx.bio
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.oaf.Oaf
-import eu.dnetllib.dhp.sx.bio.BioDBToOAF.ScholixResolved
+import BioDBToOAF.ScholixResolved
+import eu.dnetlib.dhp.collection.CollectionUtils
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
@@ -13,7 +14,7 @@ object SparkTransformBioDatabaseToOAF {
   def main(args: Array[String]): Unit = {
     val conf: SparkConf = new SparkConf()
     val log: Logger = LoggerFactory.getLogger(getClass)
-    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/bio_to_oaf_params.json")))
+    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/bio/ebi/bio_to_oaf_params.json")))
     parser.parseArgument(args)
     val database: String = parser.get("database")
     log.info("database: {}", database)
@@ -35,13 +36,13 @@ object SparkTransformBioDatabaseToOAF {
     import spark.implicits._
     database.toUpperCase() match {
       case "UNIPROT" =>
-        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.uniprotToOAF(i))).write.mode(SaveMode.Overwrite).save(targetPath)
+        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.uniprotToOAF(i))).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
       case "PDB" =>
-        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.pdbTOOaf(i))).write.mode(SaveMode.Overwrite).save(targetPath)
+        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.pdbTOOaf(i))).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
       case "SCHOLIX" =>
-        spark.read.load(dbPath).as[ScholixResolved].map(i => BioDBToOAF.scholixResolvedToOAF(i)).write.mode(SaveMode.Overwrite).save(targetPath)
+        spark.read.load(dbPath).as[ScholixResolved].map(i => BioDBToOAF.scholixResolvedToOAF(i)).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
       case "CROSSREF_LINKS" =>
-        spark.createDataset(sc.textFile(dbPath).map(i => BioDBToOAF.crossrefLinksToOaf(i))).write.mode(SaveMode.Overwrite).save(targetPath)
+        spark.createDataset(sc.textFile(dbPath).map(i => BioDBToOAF.crossrefLinksToOaf(i))).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
     }
   }
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala

@@ -1,10 +1,10 @@
-package eu.dnetllib.dhp.sx.bio.ebi
+package eu.dnetlib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
 import eu.dnetlib.dhp.schema.oaf.Result
+import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal, PMParser, PubMedToOaf}
 import eu.dnetlib.dhp.utils.ISLookupClientFactory
-import eu.dnetllib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal, PMParser, PubMedToOaf}
 import org.apache.commons.io.IOUtils
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/ebi/SparkDownloadEBILinks.scala

@@ -1,8 +1,9 @@
-package eu.dnetllib.dhp.sx.bio.ebi
+package eu.dnetlib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetllib.dhp.sx.bio.BioDBToOAF.EBILinkItem
+import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
-import eu.dnetllib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF.EBILinkItem
+import eu.dnetlib.dhp.sx.bio.pubmed.PMJournal
 import org.apache.commons.io.IOUtils
 import org.apache.http.client.config.RequestConfig
 import org.apache.http.client.methods.HttpGet

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/ebi/SparkEBILinksToOaf.scala

@@ -1,9 +1,11 @@
-package eu.dnetllib.dhp.sx.bio.ebi
+package eu.dnetlib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.oaf.Oaf
-import eu.dnetllib.dhp.sx.bio.BioDBToOAF
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF
-import eu.dnetllib.dhp.sx.bio.BioDBToOAF.EBILinkItem
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF.EBILinkItem
+import BioDBToOAF.EBILinkItem
+import eu.dnetlib.dhp.collection.CollectionUtils
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql._
@@ -36,6 +38,7 @@ object SparkEBILinksToOaf {
     ebLinks.flatMap(j => BioDBToOAF.parse_ebi_links(j.links))
       .filter(p => BioDBToOAF.EBITargetLinksFilter(p))
       .flatMap(p => BioDBToOAF.convertEBILinksToOaf(p))
+      .flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null)
       .write.mode(SaveMode.Overwrite).save(targetPath)
   }
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMArticle.java

@@ -1,5 +1,5 @@
 
-package eu.dnetllib.dhp.sx.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 import java.util.ArrayList;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMAuthor.java

@@ -1,5 +1,5 @@
 
-package eu.dnetllib.dhp.sx.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMGrant.java

@@ -1,5 +1,5 @@
 
-package eu.dnetllib.dhp.sx.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 public class PMGrant {
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMJournal.java

@@ -1,5 +1,5 @@
 
-package eu.dnetllib.dhp.sx.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMParser.scala

@@ -1,4 +1,4 @@
-package eu.dnetllib.dhp.sx.bio.pubmed
+package eu.dnetlib.dhp.sx.bio.pubmed
 
 import scala.xml.MetaData
 import scala.xml.pull.{EvElemEnd, EvElemStart, EvText, XMLEventReader}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMSubject.java

@@ -1,5 +1,5 @@
 
-package eu.dnetllib.dhp.sx.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 public class PMSubject {
 	private String value;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala

@@ -1,4 +1,4 @@
-package eu.dnetllib.dhp.sx.bio.pubmed
+package eu.dnetlib.dhp.sx.bio.pubmed
 
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
 import eu.dnetlib.dhp.schema.common.ModelConstants

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/oozie_app/workflow.xml

@@ -1,81 +0,0 @@
-<workflow-app name="Import_Datacite_and_transform_to_OAF" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>mainPath</name>
-            <description>the working path of Datacite stores</description>
-        </property>
-        <property>
-            <name>isLookupUrl</name>
-            <description>The IS lookUp service endopoint</description>
-        </property>
-        <property>
-            <name>blocksize</name>
-            <value>100</value>
-            <description>The request block size</description>
-        </property>
-
-    </parameters>
-
-    <start to="ImportDatacite"/>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-
-    <action name="ImportDatacite">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>ImportDatacite</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.ImportDatacite</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--targetPath</arg><arg>${mainPath}/datacite_update</arg>
-            <arg>--dataciteDumpPath</arg><arg>${mainPath}/datacite_dump</arg>
-            <arg>--namenode</arg><arg>${nameNode}</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-            <arg>--blocksize</arg><arg>${blocksize}</arg>
-        </spark>
-        <ok to="TransformJob"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="TransformJob">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>TransformJob</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.GenerateDataciteDatasetSpark</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${mainPath}/datacite_dump</arg>
-            <arg>--targetPath</arg><arg>${mainPath}/datacite_oaf</arg>
-            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
-            <arg>--exportLinks</arg><arg>false</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-    <end name="End"/>
-</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/scholix/oozie_app/workflow.xml

@@ -1,84 +0,0 @@
-<workflow-app name="Generate_Datacite_and_Crossref_dump_for_Scholexplorer" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>datacitePath</name>
-            <description>the path of Datacite spark dataset</description>
-        </property>
-        <property>
-            <name>isLookupUrl</name>
-            <description>The IS lookUp service endopoint</description>
-        </property>
-        <property>
-            <name>crossrefPath</name>
-            <description>the path of Crossref spark dataset</description>
-        </property>
-
-        <property>
-            <name>targetPath</name>
-            <description>the path of Crossref spark dataset</description>
-        </property>
-
-    </parameters>
-
-    <start to="ImportDatacite"/>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-
-    <action name="ImportDatacite">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>ImportDatacite</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.GenerateDataciteDatasetSpark</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${datacitePath}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/datacite_oaf</arg>
-            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
-            <arg>--exportLinks</arg><arg>true</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="FilterCrossrefEntities"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="FilterCrossrefEntities">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>FilterCrossrefEntities</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.FilterCrossrefEntitiesSpark</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${crossrefPath}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/crossref_oaf</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-    <end name="End"/>
-</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/collection/oozie_app/workflow.xml

@@ -1,46 +1,52 @@
-<workflow-app name="Datacite_to_ActionSet_Workflow" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="Collect_Datacite" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
-            <name>sourcePath</name>
+            <name>mainPath</name>
             <description>the working path of Datacite stores</description>
         </property>
         <property>
-            <name>outputPath</name>
+            <name>isLookupUrl</name>
-            <description>the path of Datacite ActionSet</description>
+            <description>The IS lookUp service endopoint</description>
         </property>
+        <property>
+            <name>blocksize</name>
+            <value>100</value>
+            <description>The request block size</description>
+        </property>
+
     </parameters>
 
-    <start to="ExportDataset"/>
+    <start to="ImportDatacite"/>
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
 
-    <action name="ExportDataset">
+    <action name="ImportDatacite">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn-cluster</master>
             <mode>cluster</mode>
-            <name>ExportDataset</name>
+            <name>ImportDatacite</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.ExportActionSetJobNode</class>
+            <class>eu.dnetlib.dhp.datacite.ImportDatacite</class>
             <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
-            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+            <arg>--targetPath</arg><arg>${mainPath}/datacite_update</arg>
-            <arg>--targetPath</arg><arg>${outputPath}</arg>
+            <arg>--dataciteDumpPath</arg><arg>${mainPath}/datacite_dump</arg>
+            <arg>--namenode</arg><arg>${nameNode}</arg>
             <arg>--master</arg><arg>yarn-cluster</arg>
+            <arg>--blocksize</arg><arg>${blocksize}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
     </action>
-
     <end name="End"/>
 </workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/generate_dataset_params.json

@@ -7,8 +7,8 @@
   },
 
   {
-    "paramName": "t",
+    "paramName": "mo",
-    "paramLongName": "targetPath",
+    "paramLongName": "mdstoreOutputVersion",
     "paramDescription": "the target mdstore path",
     "paramRequired": true
   },

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/transformation/oozie_app/workflow.xml

@@ -0,0 +1,126 @@
+<workflow-app name="transform_Datacite" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>mainPath</name>
+            <description>the working path of Datacite stores</description>
+        </property>
+        <property>
+            <name>isLookupUrl</name>
+            <description>The IS lookUp service endopoint</description>
+        </property>
+        <property>
+            <name>mdStoreOutputId</name>
+            <description>the identifier of the cleaned MDStore</description>
+        </property>
+        <property>
+            <name>mdStoreManagerURI</name>
+            <description>the path of the cleaned mdstore</description>
+        </property>
+    </parameters>
+
+    <start to="StartTransaction"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <action name="StartTransaction">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>NEW_VERSION</arg>
+            <arg>--mdStoreID</arg><arg>${mdStoreOutputId}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            <capture-output/>
+        </java>
+        <ok to="TransformJob"/>
+        <error to="EndReadRollBack"/>
+    </action>
+
+    <action name="TransformJob">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>TransformJob</name>
+            <class>eu.dnetlib.dhp.datacite.GenerateDataciteDatasetSpark</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.sql.shuffle.partitions=3840
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${mainPath}/datacite_dump</arg>
+            <arg>--mdstoreOutputVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
+            <arg>--exportLinks</arg><arg>true</arg>
+            <arg>--master</arg><arg>yarn-cluster</arg>
+        </spark>
+        <ok to="CommitVersion"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="CommitVersion">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>COMMIT</arg>
+            <arg>--namenode</arg><arg>${nameNode}</arg>
+            <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+        </java>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="EndReadRollBack">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>READ_UNLOCK</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            <arg>--readMDStoreId</arg><arg>${wf:actionData('BeginRead')['mdStoreReadLockVersion']}</arg>
+            <capture-output/>
+        </java>
+        <ok to="RollBack"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="RollBack">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>ROLLBACK</arg>
+            <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+        </java>
+        <ok to="Kill"/>
+        <error to="Kill"/>
+    </action>
+
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/db/oozie_app/workflow.xml

@@ -0,0 +1,51 @@
+    <workflow-app name="Transform_BioEntity_Workflow" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>sourcePath</name>
+            <description>the PDB Database Working Path</description>
+        </property>
+        <property>
+            <name>database</name>
+            <description>the PDB Database Working Path</description>
+        </property>
+
+        <property>
+            <name>targetPath</name>
+            <description>the Target Working dir path</description>
+        </property>
+    </parameters>
+
+    <start to="ConvertDB"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <action name="ConvertDB">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Convert Bio DB to OAF Dataset</name>
+            <class>eu.dnetlib.dhp.sx.bio.SparkTransformBioDatabaseToOAF</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.shuffle.partitions=2000
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--master</arg><arg>yarn</arg>
+            <arg>--dbPath</arg><arg>${sourcePath}</arg>
+            <arg>--database</arg><arg>${database}</arg>
+            <arg>--targetPath</arg><arg>${targetPath}</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+    <end name="End"/>
+
+</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/oozie_app/workflow.xml

@@ -52,7 +52,7 @@
             <master>yarn-cluster</master>
             <mode>cluster</mode>
             <name>Incremental Download EBI Links</name>
-            <class>eu.dnetllib.dhp.sx.bio.ebi.SparkDownloadEBILinks</class>
+            <class>eu.dnetlib.dhp.sx.bio.ebi.SparkDownloadEBILinks</class>
             <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
@@ -85,7 +85,7 @@
             <master>yarn-cluster</master>
             <mode>cluster</mode>
             <name>Create OAF DataSet</name>
-            <class>eu.dnetllib.dhp.sx.bio.ebi.SparkEBILinksToOaf</class>
+            <class>eu.dnetlib.dhp.sx.bio.ebi.SparkEBILinksToOaf</class>
             <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml

@@ -30,7 +30,7 @@
             <master>yarn</master>
             <mode>cluster</mode>
             <name>Convert Baseline to OAF Dataset</name>
-            <class>eu.dnetllib.dhp.sx.bio.ebi.SparkCreateBaselineDataFrame</class>
+            <class>eu.dnetlib.dhp.sx.bio.ebi.SparkCreateBaselineDataFrame</class>
             <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/datacite/DataciteToOAFTest.scala

@@ -1,8 +1,7 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
 
-import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
-import com.fasterxml.jackson.databind.SerializationFeature
 import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest
 import eu.dnetlib.dhp.schema.oaf.Oaf
 import org.junit.jupiter.api.extension.ExtendWith

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/sx/bio/BioScholixTest.scala

@@ -1,10 +1,10 @@
-package eu.dnetllib.dhp.sx.bio
+package eu.dnetlib.dhp.sx.bio
 
 import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
 import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest
 import eu.dnetlib.dhp.schema.oaf.{Oaf, Relation, Result}
-import eu.dnetllib.dhp.sx.bio.BioDBToOAF.ScholixResolved
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF.ScholixResolved
-import eu.dnetllib.dhp.sx.bio.pubmed.{PMArticle, PMParser, PubMedToOaf}
+import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMParser, PubMedToOaf}
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.parse

dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/ReadBlacklistFromDB.java

@@ -91,8 +91,8 @@ public class ReadBlacklistFromDB implements Closeable {
 
 			String encoding = rs.getString("relationship");
 			RelationInverse ri = ModelSupport.relationInverseMap.get(encoding);
-			direct.setRelClass(ri.getRelation());
+			direct.setRelClass(ri.getRelClass());
-			inverse.setRelClass(ri.getInverse());
+			inverse.setRelClass(ri.getInverseRelClass());
 			direct.setRelType(ri.getRelType());
 			inverse.setRelType(ri.getRelType());
 			direct.setSubRelType(ri.getSubReltype());

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Constants.java

@@ -5,37 +5,40 @@ import java.util.Map;
 
 import com.google.common.collect.Maps;
 
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+
 public class Constants {
 
-	public static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
+	protected static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
-	public static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
+	protected static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
 
 	public static final String INFERRED = "Inferred by OpenAIRE";
+	public static final String CABF2 = "c_abf2";
 
 	public static final String HARVESTED = "Harvested";
 	public static final String DEFAULT_TRUST = "0.9";
 	public static final String USER_CLAIM = "Linked by user";
 
-	public static String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
+	public static final String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
 
-	public static String ZENODO_COMMUNITY_PREFIX = "https://zenodo.org/communities/";
+	public static final String ZENODO_COMMUNITY_PREFIX = "https://zenodo.org/communities/";
 
-	public static String RESEARCH_COMMUNITY = "Research Community";
+	public static final String RESEARCH_COMMUNITY = "Research Community";
 
-	public static String RESEARCH_INFRASTRUCTURE = "Research Infrastructure/Initiative";
+	public static final String RESEARCH_INFRASTRUCTURE = "Research Infrastructure/Initiative";
 
 	static {
-		accessRightsCoarMap.put("OPEN", "c_abf2");
+		accessRightsCoarMap.put(ModelConstants.ACCESS_RIGHT_OPEN, CABF2);
 		accessRightsCoarMap.put("RESTRICTED", "c_16ec");
-		accessRightsCoarMap.put("OPEN SOURCE", "c_abf2");
+		accessRightsCoarMap.put("OPEN SOURCE", CABF2);
-		accessRightsCoarMap.put("CLOSED", "c_14cb");
+		accessRightsCoarMap.put(ModelConstants.ACCESS_RIGHT_CLOSED, "c_14cb");
-		accessRightsCoarMap.put("EMBARGO", "c_f1cf");
+		accessRightsCoarMap.put(ModelConstants.ACCESS_RIGHT_EMBARGO, "c_f1cf");
 	}
 
 	static {
-		coarCodeLabelMap.put("c_abf2", "OPEN");
+		coarCodeLabelMap.put(CABF2, ModelConstants.ACCESS_RIGHT_OPEN);
 		coarCodeLabelMap.put("c_16ec", "RESTRICTED");
-		coarCodeLabelMap.put("c_14cb", "CLOSED");
+		coarCodeLabelMap.put("c_14cb", ModelConstants.ACCESS_RIGHT_CLOSED);
 		coarCodeLabelMap.put("c_f1cf", "EMBARGO");
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/DumpProducts.java

@@ -11,12 +11,14 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.FilterFunction;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SaveMode;
 import org.apache.spark.sql.SparkSession;
 
 import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
+import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
 import eu.dnetlib.dhp.schema.oaf.*;
 
 /**
@@ -37,7 +39,8 @@ public class DumpProducts implements Serializable {
 			isSparkSessionManaged,
 			spark -> {
 				Utils.removeOutputDir(spark, outputPath);
-				execDump(spark, inputPath, outputPath, communityMapPath, inputClazz, outputClazz, dumpType);
+				execDump(
+					spark, inputPath, outputPath, communityMapPath, inputClazz, outputClazz, dumpType);
 			});
 	}
 
@@ -55,7 +58,7 @@ public class DumpProducts implements Serializable {
 		Utils
 			.readPath(spark, inputPath, inputClazz)
 			.map((MapFunction<I, O>) value -> execMap(value, communityMap, dumpType), Encoders.bean(outputClazz))
-			.filter(Objects::nonNull)
+			.filter((FilterFunction<O>) value -> value != null)
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")
@@ -65,7 +68,7 @@ public class DumpProducts implements Serializable {
 
 	private static <I extends OafEntity, O extends eu.dnetlib.dhp.schema.dump.oaf.Result> O execMap(I value,
 		CommunityMap communityMap,
-		String dumpType) {
+		String dumpType) throws NoAvailableEntityTypeException {
 
 		Optional<DataInfo> odInfo = Optional.ofNullable(value.getDataInfo());
 		if (odInfo.isPresent()) {
@@ -89,11 +92,11 @@ public class DumpProducts implements Serializable {
 					return c.getId();
 				}
 				if (c.getId().contains("::") && communities.contains(c.getId().substring(0, c.getId().indexOf("::")))) {
-					return c.getId().substring(0, 3);
+					return c.getId().substring(0, c.getId().indexOf("::"));
 				}
 				return null;
 			}).filter(Objects::nonNull).collect(Collectors.toList());
-			if (toDumpFor.size() == 0) {
+			if (toDumpFor.isEmpty()) {
 				return null;
 			}
 		}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java

@@ -57,16 +57,16 @@ public class MakeTar implements Serializable {
 	public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit)
 		throws IOException {
 
-		RemoteIterator<LocatedFileStatus> dir_iterator = fileSystem.listLocatedStatus(new Path(inputPath));
+		RemoteIterator<LocatedFileStatus> dirIterator = fileSystem.listLocatedStatus(new Path(inputPath));
 
-		while (dir_iterator.hasNext()) {
+		while (dirIterator.hasNext()) {
-			LocatedFileStatus fileStatus = dir_iterator.next();
+			LocatedFileStatus fileStatus = dirIterator.next();
 
 			Path p = fileStatus.getPath();
-			String p_string = p.toString();
+			String pathString = p.toString();
-			String entity = p_string.substring(p_string.lastIndexOf("/") + 1);
+			String entity = pathString.substring(pathString.lastIndexOf("/") + 1);
 
-			MakeTarArchive.tarMaxSize(fileSystem, p_string, outputPath + "/" + entity, entity, gBperSplit);
+			MakeTarArchive.tarMaxSize(fileSystem, pathString, outputPath + "/" + entity, entity, gBperSplit);
 		}
 
 	}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/QueryInformationSystem.java

@@ -18,10 +18,10 @@ public class QueryInformationSystem {
 
 	private ISLookUpService isLookUp;
 
-	private static final String XQUERY = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') "
+	private static final String XQUERY_ALL = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') "
 		+
 		"  where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] " +
-		" and ($x//context/param[./@name = 'status']/text() = 'manager'  or $x//context/param[./@name = 'status']/text() = 'all') "
+		" and ($x//context/param[./@name = 'status']/text() = 'all') "
 		+
 		"  return " +
 		"<community> " +
@@ -29,9 +29,22 @@ public class QueryInformationSystem {
 		"{$x//CONFIGURATION/context/@label}" +
 		"</community>";
 
-	public CommunityMap getCommunityMap()
+	private static final String XQUERY_CI = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') "
+		+
+		"  where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] " +
+		" and  $x//CONFIGURATION/context[./@id=%s]  "
+		+
+		"  return " +
+		"<community> " +
+		"{$x//CONFIGURATION/context/@id}" +
+		"{$x//CONFIGURATION/context/@label}" +
+		"</community>";
+
+	public CommunityMap getCommunityMap(boolean singleCommunity, String communityId)
 		throws ISLookUpException, DocumentException, SAXException {
-		return getMap(isLookUp.quickSearchProfile(XQUERY));
+		if (singleCommunity)
+			return getMap(isLookUp.quickSearchProfile(XQUERY_CI.replace("%s", "'" + communityId + "'")));
+		return getMap(isLookUp.quickSearchProfile(XQUERY_ALL));
 
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java

@@ -7,21 +7,28 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
+import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.dump.oaf.*;
+import eu.dnetlib.dhp.schema.dump.oaf.AccessRight;
+import eu.dnetlib.dhp.schema.dump.oaf.Author;
+import eu.dnetlib.dhp.schema.dump.oaf.Country;
+import eu.dnetlib.dhp.schema.dump.oaf.GeoLocation;
+import eu.dnetlib.dhp.schema.dump.oaf.Instance;
+import eu.dnetlib.dhp.schema.dump.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.dump.oaf.OpenAccessRoute;
+import eu.dnetlib.dhp.schema.dump.oaf.Qualifier;
+import eu.dnetlib.dhp.schema.dump.oaf.Result;
 import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityInstance;
 import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
 import eu.dnetlib.dhp.schema.dump.oaf.community.Context;
 import eu.dnetlib.dhp.schema.dump.oaf.graph.GraphResult;
-import eu.dnetlib.dhp.schema.oaf.DataInfo;
+import eu.dnetlib.dhp.schema.oaf.*;
-import eu.dnetlib.dhp.schema.oaf.Field;
-import eu.dnetlib.dhp.schema.oaf.Journal;
-import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
 
 public class ResultMapper implements Serializable {
 
 	public static <E extends eu.dnetlib.dhp.schema.oaf.OafEntity> Result map(
-		E in, Map<String, String> communityMap, String dumpType) {
+		E in, Map<String, String> communityMap, String dumpType) throws NoAvailableEntityTypeException {
 
 		Result out;
 		if (Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
@@ -33,113 +40,28 @@ public class ResultMapper implements Serializable {
 		eu.dnetlib.dhp.schema.oaf.Result input = (eu.dnetlib.dhp.schema.oaf.Result) in;
 		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> ort = Optional.ofNullable(input.getResulttype());
 		if (ort.isPresent()) {
-			switch (ort.get().getClassid()) {
+			try {
-				case "publication":
-					Optional<Journal> journal = Optional
-						.ofNullable(((eu.dnetlib.dhp.schema.oaf.Publication) input).getJournal());
-					if (journal.isPresent()) {
-						Journal j = journal.get();
-						Container c = new Container();
-						c.setConferencedate(j.getConferencedate());
-						c.setConferenceplace(j.getConferenceplace());
-						c.setEdition(j.getEdition());
-						c.setEp(j.getEp());
-						c.setIss(j.getIss());
-						c.setIssnLinking(j.getIssnLinking());
-						c.setIssnOnline(j.getIssnOnline());
-						c.setIssnPrinted(j.getIssnPrinted());
-						c.setName(j.getName());
-						c.setSp(j.getSp());
-						c.setVol(j.getVol());
-						out.setContainer(c);
-						out.setType(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE.getClassname());
-					}
-					break;
-				case "dataset":
-					eu.dnetlib.dhp.schema.oaf.Dataset id = (eu.dnetlib.dhp.schema.oaf.Dataset) input;
-					Optional.ofNullable(id.getSize()).ifPresent(v -> out.setSize(v.getValue()));
-					Optional.ofNullable(id.getVersion()).ifPresent(v -> out.setVersion(v.getValue()));
 
-					out
+				addTypeSpecificInformation(out, input, ort);
-						.setGeolocation(
+				Optional<List<Measure>> mes = Optional.ofNullable(input.getMeasures());
-							Optional
+				if (mes.isPresent()) {
-								.ofNullable(id.getGeolocation())
+					List<KeyValue> measure = new ArrayList<>();
-								.map(
+					mes
-									igl -> igl
+						.get()
-										.stream()
+						.forEach(
-										.filter(Objects::nonNull)
+							m -> m.getUnit().forEach(u -> measure.add(KeyValue.newInstance(m.getId(), u.getValue()))));
-										.map(gli -> {
+					out.setMeasures(measure);
-											GeoLocation gl = new GeoLocation();
+				}
-											gl.setBox(gli.getBox());
-											gl.setPlace(gli.getPlace());
-											gl.setPoint(gli.getPoint());
-											return gl;
-										})
-										.collect(Collectors.toList()))
-								.orElse(null));
 
-					out.setType(ModelConstants.DATASET_DEFAULT_RESULTTYPE.getClassname());
+				Optional
-					break;
+					.ofNullable(input.getAuthor())
-				case "software":
+					.ifPresent(
+						ats -> out.setAuthor(ats.stream().map(ResultMapper::getAuthor).collect(Collectors.toList())));
 
-					eu.dnetlib.dhp.schema.oaf.Software is = (eu.dnetlib.dhp.schema.oaf.Software) input;
+				// I do not map Access Right UNKNOWN or OTHER
-					Optional
-						.ofNullable(is.getCodeRepositoryUrl())
-						.ifPresent(value -> out.setCodeRepositoryUrl(value.getValue()));
-					Optional
-						.ofNullable(is.getDocumentationUrl())
-						.ifPresent(
-							value -> out
-								.setDocumentationUrl(
-									value
-										.stream()
-										.map(Field::getValue)
-										.collect(Collectors.toList())));
 
-					Optional
+				Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oar = Optional.ofNullable(input.getBestaccessright());
-						.ofNullable(is.getProgrammingLanguage())
+				if (oar.isPresent() && Constants.accessRightsCoarMap.containsKey(oar.get().getClassid())) {
-						.ifPresent(value -> out.setProgrammingLanguage(value.getClassid()));
-
-					out.setType(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE.getClassname());
-					break;
-				case "other":
-
-					eu.dnetlib.dhp.schema.oaf.OtherResearchProduct ir = (eu.dnetlib.dhp.schema.oaf.OtherResearchProduct) input;
-					out
-						.setContactgroup(
-							Optional
-								.ofNullable(ir.getContactgroup())
-								.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
-								.orElse(null));
-
-					out
-						.setContactperson(
-							Optional
-								.ofNullable(ir.getContactperson())
-								.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
-								.orElse(null));
-					out
-						.setTool(
-							Optional
-								.ofNullable(ir.getTool())
-								.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
-								.orElse(null));
-
-					out.setType(ModelConstants.ORP_DEFAULT_RESULTTYPE.getClassname());
-
-					break;
-			}
-
-			Optional
-				.ofNullable(input.getAuthor())
-				.ifPresent(
-					ats -> out.setAuthor(ats.stream().map(ResultMapper::getAuthor).collect(Collectors.toList())));
-
-			// I do not map Access Right UNKNOWN or OTHER
-
-			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oar = Optional.ofNullable(input.getBestaccessright());
-			if (oar.isPresent()) {
-				if (Constants.accessRightsCoarMap.containsKey(oar.get().getClassid())) {
 					String code = Constants.accessRightsCoarMap.get(oar.get().getClassid());
 					out
 						.setBestaccessright(
@@ -149,226 +71,340 @@ public class ResultMapper implements Serializable {
 									Constants.coarCodeLabelMap.get(code),
 									Constants.COAR_ACCESS_RIGHT_SCHEMA));
 				}
-			}
 
-			final List<String> contributorList = new ArrayList<>();
+				final List<String> contributorList = new ArrayList<>();
-			Optional
+				Optional
-				.ofNullable(input.getContributor())
+					.ofNullable(input.getContributor())
-				.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
+					.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
-			out.setContributor(contributorList);
+				out.setContributor(contributorList);
 
-			Optional
+				Optional
-				.ofNullable(input.getCountry())
+					.ofNullable(input.getCountry())
-				.ifPresent(
+					.ifPresent(
-					value -> out
+						value -> out
-						.setCountry(
+							.setCountry(
-							value
+								value
-								.stream()
+									.stream()
-								.map(
+									.map(
-									c -> {
+										c -> {
-										if (c.getClassid().equals((ModelConstants.UNKNOWN))) {
+											if (c.getClassid().equals((ModelConstants.UNKNOWN))) {
-											return null;
+												return null;
-										}
+											}
-										Country country = new Country();
+											Country country = new Country();
-										country.setCode(c.getClassid());
+											country.setCode(c.getClassid());
-										country.setLabel(c.getClassname());
+											country.setLabel(c.getClassname());
-										Optional
+											Optional
-											.ofNullable(c.getDataInfo())
+												.ofNullable(c.getDataInfo())
-											.ifPresent(
+												.ifPresent(
-												provenance -> country
+													provenance -> country
-													.setProvenance(
+														.setProvenance(
-														Provenance
+															Provenance
-															.newInstance(
-																provenance
-																	.getProvenanceaction()
-																	.getClassname(),
-																c.getDataInfo().getTrust())));
-										return country;
-									})
-								.filter(Objects::nonNull)
-								.collect(Collectors.toList())));
-
-			final List<String> coverageList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getCoverage())
-				.ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue())));
-			out.setCoverage(coverageList);
-
-			out.setDateofcollection(input.getDateofcollection());
-
-			final List<String> descriptionList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getDescription())
-				.ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue())));
-			out.setDescription(descriptionList);
-			Optional<Field<String>> oStr = Optional.ofNullable(input.getEmbargoenddate());
-			if (oStr.isPresent()) {
-				out.setEmbargoenddate(oStr.get().getValue());
-			}
-
-			final List<String> formatList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getFormat())
-				.ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue())));
-			out.setFormat(formatList);
-			out.setId(input.getId());
-			out.setOriginalId(input.getOriginalId());
-
-			Optional<List<eu.dnetlib.dhp.schema.oaf.Instance>> oInst = Optional
-				.ofNullable(input.getInstance());
-
-			if (oInst.isPresent()) {
-				if (Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
-					((GraphResult) out)
-						.setInstance(
-							oInst.get().stream().map(ResultMapper::getGraphInstance).collect(Collectors.toList()));
-				} else {
-					((CommunityResult) out)
-						.setInstance(
-							oInst.get().stream().map(ResultMapper::getCommunityInstance).collect(Collectors.toList()));
-				}
-			}
-
-			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oL = Optional.ofNullable(input.getLanguage());
-			if (oL.isPresent()) {
-				eu.dnetlib.dhp.schema.oaf.Qualifier language = oL.get();
-				out.setLanguage(Qualifier.newInstance(language.getClassid(), language.getClassname()));
-			}
-			Optional<Long> oLong = Optional.ofNullable(input.getLastupdatetimestamp());
-			if (oLong.isPresent()) {
-				out.setLastupdatetimestamp(oLong.get());
-			}
-			Optional<List<StructuredProperty>> otitle = Optional.ofNullable(input.getTitle());
-			if (otitle.isPresent()) {
-				List<StructuredProperty> iTitle = otitle
-					.get()
-					.stream()
-					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title"))
-					.collect(Collectors.toList());
-				if (iTitle.size() > 0) {
-					out.setMaintitle(iTitle.get(0).getValue());
-				}
-
-				iTitle = otitle
-					.get()
-					.stream()
-					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle"))
-					.collect(Collectors.toList());
-				if (iTitle.size() > 0) {
-					out.setSubtitle(iTitle.get(0).getValue());
-				}
-
-			}
-
-			List<ControlledField> pids = new ArrayList<>();
-			Optional
-				.ofNullable(input.getPid())
-				.ifPresent(
-					value -> value
-						.stream()
-						.forEach(
-							p -> pids
-								.add(
-									ControlledField
-										.newInstance(p.getQualifier().getClassid(), p.getValue()))));
-			out.setPid(pids);
-			oStr = Optional.ofNullable(input.getDateofacceptance());
-			if (oStr.isPresent()) {
-				out.setPublicationdate(oStr.get().getValue());
-			}
-			oStr = Optional.ofNullable(input.getPublisher());
-			if (oStr.isPresent()) {
-				out.setPublisher(oStr.get().getValue());
-			}
-
-			List<String> sourceList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getSource())
-				.ifPresent(value -> value.stream().forEach(s -> sourceList.add(s.getValue())));
-			// out.setSource(input.getSource().stream().map(s -> s.getValue()).collect(Collectors.toList()));
-			List<Subject> subjectList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getSubject())
-				.ifPresent(
-					value -> value
-						.forEach(s -> subjectList.add(getSubject(s))));
-
-			out.setSubjects(subjectList);
-
-			out.setType(input.getResulttype().getClassid());
-		}
-
-		if (!Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
-			((CommunityResult) out)
-				.setCollectedfrom(
-					input
-						.getCollectedfrom()
-						.stream()
-						.map(cf -> KeyValue.newInstance(cf.getKey(), cf.getValue()))
-						.collect(Collectors.toList()));
-
-			Set<String> communities = communityMap.keySet();
-			List<Context> contextList = Optional
-				.ofNullable(
-					input
-						.getContext())
-				.map(
-					value -> value
-						.stream()
-						.map(c -> {
-							String community_id = c.getId();
-							if (community_id.indexOf("::") > 0) {
-								community_id = community_id.substring(0, community_id.indexOf("::"));
-							}
-							if (communities.contains(community_id)) {
-								Context context = new Context();
-								context.setCode(community_id);
-								context.setLabel(communityMap.get(community_id));
-								Optional<List<DataInfo>> dataInfo = Optional.ofNullable(c.getDataInfo());
-								if (dataInfo.isPresent()) {
-									List<Provenance> provenance = new ArrayList<>();
-									provenance
-										.addAll(
-											dataInfo
-												.get()
-												.stream()
-												.map(
-													di -> Optional
-														.ofNullable(di.getProvenanceaction())
-														.map(
-															provenanceaction -> Provenance
 																.newInstance(
-																	provenanceaction.getClassname(), di.getTrust()))
+																	provenance
-														.orElse(null))
+																		.getProvenanceaction()
-												.filter(Objects::nonNull)
+																		.getClassname(),
-												.collect(Collectors.toSet()));
+																	c.getDataInfo().getTrust())));
+											return country;
+										})
+									.filter(Objects::nonNull)
+									.collect(Collectors.toList())));
 
-									context.setProvenance(getUniqueProvenance(provenance));
+				final List<String> coverageList = new ArrayList<>();
-								}
+				Optional
-								return context;
+					.ofNullable(input.getCoverage())
-							}
+					.ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue())));
-							return null;
+				out.setCoverage(coverageList);
-						})
-						.filter(Objects::nonNull)
-						.collect(Collectors.toList()))
-				.orElse(new ArrayList<>());
 
-			if (contextList.size() > 0) {
+				out.setDateofcollection(input.getDateofcollection());
-				Set<Integer> hashValue = new HashSet<>();
+
-				List<Context> remainigContext = new ArrayList<>();
+				final List<String> descriptionList = new ArrayList<>();
-				contextList.forEach(c -> {
+				Optional
-					if (!hashValue.contains(c.hashCode())) {
+					.ofNullable(input.getDescription())
-						remainigContext.add(c);
+					.ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue())));
-						hashValue.add(c.hashCode());
+				out.setDescription(descriptionList);
+				Optional<Field<String>> oStr = Optional.ofNullable(input.getEmbargoenddate());
+				if (oStr.isPresent()) {
+					out.setEmbargoenddate(oStr.get().getValue());
+				}
+
+				final List<String> formatList = new ArrayList<>();
+				Optional
+					.ofNullable(input.getFormat())
+					.ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue())));
+				out.setFormat(formatList);
+				out.setId(input.getId());
+				out.setOriginalId(input.getOriginalId());
+
+				Optional<List<eu.dnetlib.dhp.schema.oaf.Instance>> oInst = Optional
+					.ofNullable(input.getInstance());
+
+				if (oInst.isPresent()) {
+					if (Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
+						((GraphResult) out)
+							.setInstance(
+								oInst.get().stream().map(ResultMapper::getGraphInstance).collect(Collectors.toList()));
+					} else {
+						((CommunityResult) out)
+							.setInstance(
+								oInst
+									.get()
+									.stream()
+									.map(ResultMapper::getCommunityInstance)
+									.collect(Collectors.toList()));
 					}
-				});
+				}
-				((CommunityResult) out).setContext(remainigContext);
+
+				Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oL = Optional.ofNullable(input.getLanguage());
+				if (oL.isPresent()) {
+					eu.dnetlib.dhp.schema.oaf.Qualifier language = oL.get();
+					out.setLanguage(Qualifier.newInstance(language.getClassid(), language.getClassname()));
+				}
+				Optional<Long> oLong = Optional.ofNullable(input.getLastupdatetimestamp());
+				if (oLong.isPresent()) {
+					out.setLastupdatetimestamp(oLong.get());
+				}
+				Optional<List<StructuredProperty>> otitle = Optional.ofNullable(input.getTitle());
+				if (otitle.isPresent()) {
+					List<StructuredProperty> iTitle = otitle
+						.get()
+						.stream()
+						.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title"))
+						.collect(Collectors.toList());
+					if (!iTitle.isEmpty()) {
+						out.setMaintitle(iTitle.get(0).getValue());
+					}
+
+					iTitle = otitle
+						.get()
+						.stream()
+						.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle"))
+						.collect(Collectors.toList());
+					if (!iTitle.isEmpty()) {
+						out.setSubtitle(iTitle.get(0).getValue());
+					}
+
+				}
+
+				Optional
+					.ofNullable(input.getPid())
+					.ifPresent(
+						value -> out
+							.setPid(
+								value
+									.stream()
+									.map(
+										p -> ControlledField
+											.newInstance(p.getQualifier().getClassid(), p.getValue()))
+									.collect(Collectors.toList())));
+
+				oStr = Optional.ofNullable(input.getDateofacceptance());
+				if (oStr.isPresent()) {
+					out.setPublicationdate(oStr.get().getValue());
+				}
+				oStr = Optional.ofNullable(input.getPublisher());
+				if (oStr.isPresent()) {
+					out.setPublisher(oStr.get().getValue());
+				}
+
+				Optional
+					.ofNullable(input.getSource())
+					.ifPresent(
+						value -> out.setSource(value.stream().map(Field::getValue).collect(Collectors.toList())));
+
+				List<Subject> subjectList = new ArrayList<>();
+				Optional
+					.ofNullable(input.getSubject())
+					.ifPresent(
+						value -> value
+							.forEach(s -> subjectList.add(getSubject(s))));
+
+				out.setSubjects(subjectList);
+
+				out.setType(input.getResulttype().getClassid());
+
+				if (!Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
+					((CommunityResult) out)
+						.setCollectedfrom(
+							input
+								.getCollectedfrom()
+								.stream()
+								.map(cf -> KeyValue.newInstance(cf.getKey(), cf.getValue()))
+								.collect(Collectors.toList()));
+
+					Set<String> communities = communityMap.keySet();
+					List<Context> contextList = Optional
+						.ofNullable(
+							input
+								.getContext())
+						.map(
+							value -> value
+								.stream()
+								.map(c -> {
+									String communityId = c.getId();
+									if (communityId.contains("::")) {
+										communityId = communityId.substring(0, communityId.indexOf("::"));
+									}
+									if (communities.contains(communityId)) {
+										Context context = new Context();
+										context.setCode(communityId);
+										context.setLabel(communityMap.get(communityId));
+										Optional<List<DataInfo>> dataInfo = Optional.ofNullable(c.getDataInfo());
+										if (dataInfo.isPresent()) {
+											List<Provenance> provenance = new ArrayList<>();
+											provenance
+												.addAll(
+													dataInfo
+														.get()
+														.stream()
+														.map(
+															di -> Optional
+																.ofNullable(di.getProvenanceaction())
+																.map(
+																	provenanceaction -> Provenance
+																		.newInstance(
+																			provenanceaction.getClassname(),
+																			di.getTrust()))
+																.orElse(null))
+														.filter(Objects::nonNull)
+														.collect(Collectors.toSet()));
+
+											try {
+												context.setProvenance(getUniqueProvenance(provenance));
+											} catch (NoAvailableEntityTypeException e) {
+												e.printStackTrace();
+											}
+										}
+										return context;
+									}
+									return null;
+								})
+								.filter(Objects::nonNull)
+								.collect(Collectors.toList()))
+						.orElse(new ArrayList<>());
+
+					if (!contextList.isEmpty()) {
+						Set<Integer> hashValue = new HashSet<>();
+						List<Context> remainigContext = new ArrayList<>();
+						contextList.forEach(c -> {
+							if (!hashValue.contains(c.hashCode())) {
+								remainigContext.add(c);
+								hashValue.add(c.hashCode());
+							}
+						});
+						((CommunityResult) out).setContext(remainigContext);
+					}
+				}
+			} catch (ClassCastException cce) {
+				return out;
 			}
 		}
+
 		return out;
 
 	}
 
+	private static void addTypeSpecificInformation(Result out, eu.dnetlib.dhp.schema.oaf.Result input,
+		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> ort) throws NoAvailableEntityTypeException {
+		switch (ort.get().getClassid()) {
+			case "publication":
+				Optional<Journal> journal = Optional
+					.ofNullable(((Publication) input).getJournal());
+				if (journal.isPresent()) {
+					Journal j = journal.get();
+					Container c = new Container();
+					c.setConferencedate(j.getConferencedate());
+					c.setConferenceplace(j.getConferenceplace());
+					c.setEdition(j.getEdition());
+					c.setEp(j.getEp());
+					c.setIss(j.getIss());
+					c.setIssnLinking(j.getIssnLinking());
+					c.setIssnOnline(j.getIssnOnline());
+					c.setIssnPrinted(j.getIssnPrinted());
+					c.setName(j.getName());
+					c.setSp(j.getSp());
+					c.setVol(j.getVol());
+					out.setContainer(c);
+					out.setType(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE.getClassname());
+				}
+				break;
+			case "dataset":
+				Dataset id = (Dataset) input;
+				Optional.ofNullable(id.getSize()).ifPresent(v -> out.setSize(v.getValue()));
+				Optional.ofNullable(id.getVersion()).ifPresent(v -> out.setVersion(v.getValue()));
+
+				out
+					.setGeolocation(
+						Optional
+							.ofNullable(id.getGeolocation())
+							.map(
+								igl -> igl
+									.stream()
+									.filter(Objects::nonNull)
+									.map(gli -> {
+										GeoLocation gl = new GeoLocation();
+										gl.setBox(gli.getBox());
+										gl.setPlace(gli.getPlace());
+										gl.setPoint(gli.getPoint());
+										return gl;
+									})
+									.collect(Collectors.toList()))
+							.orElse(null));
+
+				out.setType(ModelConstants.DATASET_DEFAULT_RESULTTYPE.getClassname());
+				break;
+			case "software":
+
+				Software is = (Software) input;
+				Optional
+					.ofNullable(is.getCodeRepositoryUrl())
+					.ifPresent(value -> out.setCodeRepositoryUrl(value.getValue()));
+				Optional
+					.ofNullable(is.getDocumentationUrl())
+					.ifPresent(
+						value -> out
+							.setDocumentationUrl(
+								value
+									.stream()
+									.map(Field::getValue)
+									.collect(Collectors.toList())));
+
+				Optional
+					.ofNullable(is.getProgrammingLanguage())
+					.ifPresent(value -> out.setProgrammingLanguage(value.getClassid()));
+
+				out.setType(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE.getClassname());
+				break;
+			case "other":
+
+				OtherResearchProduct ir = (OtherResearchProduct) input;
+				out
+					.setContactgroup(
+						Optional
+							.ofNullable(ir.getContactgroup())
+							.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
+							.orElse(null));
+
+				out
+					.setContactperson(
+						Optional
+							.ofNullable(ir.getContactperson())
+							.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
+							.orElse(null));
+				out
+					.setTool(
+						Optional
+							.ofNullable(ir.getTool())
+							.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
+							.orElse(null));
+
+				out.setType(ModelConstants.ORP_DEFAULT_RESULTTYPE.getClassname());
+
+				break;
+			default:
+				throw new NoAvailableEntityTypeException();
+		}
+	}
+
 	private static Instance getGraphInstance(eu.dnetlib.dhp.schema.oaf.Instance i) {
 		Instance instance = new Instance();
 
@@ -397,21 +433,58 @@ public class ResultMapper implements Serializable {
 	}
 
 	private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {
-		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
+		Optional<eu.dnetlib.dhp.schema.oaf.AccessRight> opAr = Optional.ofNullable(i.getAccessright());
-			.ofNullable(i.getAccessright());
+
-		if (opAr.isPresent()) {
+		if (opAr.isPresent() && Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
-			if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
+			String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
-				String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
+
-				instance
+			instance
-					.setAccessright(
+				.setAccessright(
-						AccessRight
+					AccessRight
-							.newInstance(
+						.newInstance(
-								code,
+							code,
-								Constants.coarCodeLabelMap.get(code),
+							Constants.coarCodeLabelMap.get(code),
-								Constants.COAR_ACCESS_RIGHT_SCHEMA));
+							Constants.COAR_ACCESS_RIGHT_SCHEMA));
+			if (opAr.get().getOpenAccessRoute() != null) {
+				switch (opAr.get().getOpenAccessRoute()) {
+					case hybrid:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.hybrid);
+						break;
+					case gold:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.gold);
+						break;
+					case green:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.green);
+						break;
+					case bronze:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.bronze);
+						break;
+
+				}
 			}
+
 		}
 
+		Optional
+			.ofNullable(i.getPid())
+			.ifPresent(
+				pid -> instance
+					.setPid(
+						pid
+							.stream()
+							.map(p -> ControlledField.newInstance(p.getQualifier().getClassid(), p.getValue()))
+							.collect(Collectors.toList())));
+
+		Optional
+			.ofNullable(i.getAlternateIdentifier())
+			.ifPresent(
+				ai -> instance
+					.setAlternateIdentifier(
+						ai
+							.stream()
+							.map(p -> ControlledField.newInstance(p.getQualifier().getClassid(), p.getValue()))
+							.collect(Collectors.toList())));
+
 		Optional
 			.ofNullable(i.getLicense())
 			.ifPresent(value -> instance.setLicense(value.getValue()));
@@ -424,11 +497,26 @@ public class ResultMapper implements Serializable {
 		Optional
 			.ofNullable(i.getInstancetype())
 			.ifPresent(value -> instance.setType(value.getClassname()));
+		Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
+		Optional<Field<String>> oPca = Optional.ofNullable(i.getProcessingchargeamount());
+		Optional<Field<String>> oPcc = Optional.ofNullable(i.getProcessingchargecurrency());
+		if (oPca.isPresent() && oPcc.isPresent()) {
+			Field<String> pca = oPca.get();
+			Field<String> pcc = oPcc.get();
+			if (!pca.getValue().trim().equals("") && !pcc.getValue().trim().equals("")) {
+				APC apc = new APC();
+				apc.setCurrency(oPcc.get().getValue());
+				apc.setAmount(oPca.get().getValue());
+				instance.setArticleprocessingcharge(apc);
+			}
+
+		}
 		Optional.ofNullable(i.getUrl()).ifPresent(instance::setUrl);
 
 	}
 
-	private static List<Provenance> getUniqueProvenance(List<Provenance> provenance) {
+	private static List<Provenance> getUniqueProvenance(List<Provenance> provenance)
+		throws NoAvailableEntityTypeException {
 		Provenance iProv = new Provenance();
 
 		Provenance hProv = new Provenance();
@@ -450,6 +538,8 @@ public class ResultMapper implements Serializable {
 				case Constants.USER_CLAIM:
 					lProv = getHighestTrust(lProv, p);
 					break;
+				default:
+					throw new NoAvailableEntityTypeException();
 			}
 
 		}
@@ -503,34 +593,67 @@ public class ResultMapper implements Serializable {
 		return a;
 	}
 
-	private static Pid getOrcid(List<StructuredProperty> p) {
+	private static Pid getAuthorPid(StructuredProperty pid) {
-		for (StructuredProperty pid : p) {
+		Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
-			if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID)) {
+		if (di.isPresent()) {
-				Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
+			return Pid
-				if (di.isPresent()) {
+				.newInstance(
-					return Pid
+					ControlledField
 						.newInstance(
-							ControlledField
+							pid.getQualifier().getClassid(),
-								.newInstance(
+							pid.getValue()),
-									pid.getQualifier().getClassid(),
+					Provenance
-									pid.getValue()),
-							Provenance
-								.newInstance(
-									di.get().getProvenanceaction().getClassname(),
-									di.get().getTrust()));
-				} else {
-					return Pid
 						.newInstance(
-							ControlledField
+							di.get().getProvenanceaction().getClassname(),
-								.newInstance(
+							di.get().getTrust()));
-									pid.getQualifier().getClassid(),
+		} else {
-									pid.getValue())
+			return Pid
+				.newInstance(
+					ControlledField
+						.newInstance(
+							pid.getQualifier().getClassid(),
+							pid.getValue())
 
-						);
+				);
-				}
-
-			}
 		}
+	}
+
+	private static Pid getOrcid(List<StructuredProperty> p) {
+		List<StructuredProperty> pidList = p.stream().map(pid -> {
+			if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID) ||
+				(pid.getQualifier().getClassid().equals(ModelConstants.ORCID_PENDING))) {
+				return pid;
+			}
+			return null;
+		}).filter(Objects::nonNull).collect(Collectors.toList());
+
+		if (pidList.size() == 1) {
+			return getAuthorPid(pidList.get(0));
+		}
+
+		List<StructuredProperty> orcid = pidList
+			.stream()
+			.filter(
+				ap -> ap
+					.getQualifier()
+					.getClassid()
+					.equals(ModelConstants.ORCID))
+			.collect(Collectors.toList());
+		if (orcid.size() == 1) {
+			return getAuthorPid(orcid.get(0));
+		}
+		orcid = pidList
+			.stream()
+			.filter(
+				ap -> ap
+					.getQualifier()
+					.getClassid()
+					.equals(ModelConstants.ORCID_PENDING))
+			.collect(Collectors.toList());
+		if (orcid.size() == 1) {
+			return getAuthorPid(orcid.get(0));
+		}
+
 		return null;
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SaveCommunityMap.java

@@ -6,6 +6,7 @@ import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -18,6 +19,7 @@ import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 
 /**
@@ -30,9 +32,9 @@ import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 public class SaveCommunityMap implements Serializable {
 
 	private static final Logger log = LoggerFactory.getLogger(SaveCommunityMap.class);
-	private final QueryInformationSystem queryInformationSystem;
+	private final transient QueryInformationSystem queryInformationSystem;
 
-	private final BufferedWriter writer;
+	private final transient BufferedWriter writer;
 
 	public SaveCommunityMap(String hdfsPath, String hdfsNameNode, String isLookUpUrl) throws IOException {
 		final Configuration conf = new Configuration();
@@ -70,13 +72,28 @@ public class SaveCommunityMap implements Serializable {
 		final String isLookUpUrl = parser.get("isLookUpUrl");
 		log.info("isLookUpUrl: {}", isLookUpUrl);
 
+		final Boolean singleCommunity = Optional
+			.ofNullable(parser.get("singleDeposition"))
+			.map(Boolean::valueOf)
+			.orElse(false);
+
+		final String community_id = Optional.ofNullable(parser.get("communityId")).orElse(null);
+
 		final SaveCommunityMap scm = new SaveCommunityMap(outputPath, nameNode, isLookUpUrl);
 
-		scm.saveCommunityMap();
+		scm.saveCommunityMap(singleCommunity, community_id);
+
 	}
 
-	private void saveCommunityMap() throws ISLookUpException, IOException, DocumentException, SAXException {
+	private void saveCommunityMap(boolean singleCommunity, String communityId)
-		writer.write(Utils.OBJECT_MAPPER.writeValueAsString(queryInformationSystem.getCommunityMap()));
+		throws ISLookUpException, IOException, DocumentException, SAXException {
+		final String communityMapString = Utils.OBJECT_MAPPER
+			.writeValueAsString(queryInformationSystem.getCommunityMap(singleCommunity, communityId));
+		log.info("communityMap {} ", communityMapString);
+		writer
+			.write(
+				communityMapString);
 		writer.close();
 	}
+
 }

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java

@@ -5,15 +5,13 @@ import java.io.Serializable;
 import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.*;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
 import eu.dnetlib.dhp.common.api.ZenodoAPIClient;
-import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
+import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
 
 public class SendToZenodoHDFS implements Serializable {
 
@@ -21,8 +19,6 @@ public class SendToZenodoHDFS implements Serializable {
 	private static final String VERSION = "version"; // to be used to upload a new version of a published deposition
 	private static final String UPDATE = "update"; // to upload content to an open deposition not published
 
-	private static final Log log = LogFactory.getLog(SendToZenodoHDFS.class);
-
 	public static void main(final String[] args) throws Exception, MissingConceptDoiException {
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
@@ -48,15 +44,12 @@ public class SendToZenodoHDFS implements Serializable {
 			.orElse(false);
 
 		final String depositionId = Optional.ofNullable(parser.get("depositionId")).orElse(null);
-		final String communityMapPath = parser.get("communityMapPath");
 
 		Configuration conf = new Configuration();
 		conf.set("fs.defaultFS", hdfsNameNode);
 
 		FileSystem fileSystem = FileSystem.get(conf);
 
-		CommunityMap communityMap = Utils.readCommunityMap(fileSystem, communityMapPath);
-
 		RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
 			.listFiles(
 				new Path(hdfsPath), true);
@@ -77,19 +70,17 @@ public class SendToZenodoHDFS implements Serializable {
 				}
 				zenodoApiClient.uploadOpenDeposition(depositionId);
 				break;
+			default:
+				throw new NoAvailableEntityTypeException();
 		}
 
 		while (fileStatusListIterator.hasNext()) {
 			LocatedFileStatus fileStatus = fileStatusListIterator.next();
 
 			Path p = fileStatus.getPath();
-			String p_string = p.toString();
+			String pString = p.toString();
-			if (!p_string.endsWith("_SUCCESS")) {
+			if (!pString.endsWith("_SUCCESS")) {
-				String name = p_string.substring(p_string.lastIndexOf("/") + 1);
+				String name = pString.substring(pString.lastIndexOf("/") + 1);
-				log.info("Sending information for community: " + name);
-				if (communityMap.containsKey(name.substring(0, name.lastIndexOf(".")))) {
-					name = communityMap.get(name.substring(0, name.lastIndexOf("."))).replace(" ", "_") + ".tar";
-				}
 
 				FSDataInputStream inputStream = fileSystem.open(p);
 				zenodoApiClient.uploadIS(inputStream, name, fileStatus.getLen());
@@ -101,7 +92,7 @@ public class SendToZenodoHDFS implements Serializable {
 			zenodoApiClient.sendMretadata(metadata);
 		}
 
-		if (publish) {
+		if (Boolean.TRUE.equals(publish)) {
 			zenodoApiClient.publish();
 		}
 	}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/CommunitySplit.java

@@ -4,9 +4,7 @@ package eu.dnetlib.dhp.oa.graph.dump.community;
 import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 
 import java.io.Serializable;
-import java.util.NoSuchElementException;
 import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.spark.SparkConf;
@@ -17,6 +15,7 @@ import org.apache.spark.sql.SparkSession;
 
 import eu.dnetlib.dhp.oa.graph.dump.Utils;
 import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
+import eu.dnetlib.dhp.schema.dump.oaf.community.Context;
 
 /**
  * This class splits the dumped results according to the research community - research initiative/infrastructure they
@@ -35,12 +34,13 @@ public class CommunitySplit implements Serializable {
 			isSparkSessionManaged,
 			spark -> {
 				Utils.removeOutputDir(spark, outputPath);
-				execSplit(spark, inputPath, outputPath, Utils.getCommunityMap(spark, communityMapPath).keySet());
+				CommunityMap communityMap = Utils.getCommunityMap(spark, communityMapPath);
+				execSplit(spark, inputPath, outputPath, communityMap);
 			});
 	}
 
 	private static void execSplit(SparkSession spark, String inputPath, String outputPath,
-		Set<String> communities) {
+		CommunityMap communities) {
 
 		Dataset<CommunityResult> result = Utils
 			.readPath(spark, inputPath + "/publication", CommunityResult.class)
@@ -49,34 +49,32 @@ public class CommunitySplit implements Serializable {
 			.union(Utils.readPath(spark, inputPath + "/software", CommunityResult.class));
 
 		communities
-			.forEach(c -> printResult(c, result, outputPath));
+			.keySet()
+			.stream()
+			.forEach(c -> printResult(c, result, outputPath + "/" + communities.get(c).replace(" ", "_")));
 
 	}
 
-	private static void printResult(String community, Dataset<CommunityResult> result, String outputPath) {
+	private static void printResult(String c, Dataset<CommunityResult> result, String outputPath) {
 		Dataset<CommunityResult> communityProducts = result
-			.filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, community));
+			.filter((FilterFunction<CommunityResult>) r -> containsCommunity(r, c));
+
+		communityProducts
+			.write()
+			.option("compression", "gzip")
+			.mode(SaveMode.Overwrite)
+			.json(outputPath);
 
-		try {
-			communityProducts.first();
-			communityProducts
-				.write()
-				.option("compression", "gzip")
-				.mode(SaveMode.Overwrite)
-				.json(outputPath + "/" + community);
-		} catch (NoSuchElementException e) {
-			// ignoring it on purpose
-		}
 	}
 
-	private static boolean containsCommunity(CommunityResult r, String community) {
+	private static boolean containsCommunity(CommunityResult r, String c) {
 		if (Optional.ofNullable(r.getContext()).isPresent()) {
-			return !r
+			return r
 				.getContext()
 				.stream()
-				.filter(con -> con.getCode().equals(community))
+				.map(Context::getCode)
 				.collect(Collectors.toList())
-				.isEmpty();
+				.contains(c);
 		}
 		return false;
 	}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/community/SparkPrepareResultProject.java

@@ -26,9 +26,11 @@ import org.xml.sax.SAXException;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.oa.graph.dump.Utils;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.dump.oaf.Provenance;
 import eu.dnetlib.dhp.schema.dump.oaf.community.Funder;
 import eu.dnetlib.dhp.schema.dump.oaf.community.Project;
+import eu.dnetlib.dhp.schema.dump.oaf.community.Validated;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Field;
 import eu.dnetlib.dhp.schema.oaf.Relation;
@@ -78,7 +80,9 @@ public class SparkPrepareResultProject implements Serializable {
 	private static void prepareResultProjectList(SparkSession spark, String inputPath, String outputPath) {
 		Dataset<Relation> relation = Utils
 			.readPath(spark, inputPath + "/relation", Relation.class)
-			.filter("dataInfo.deletedbyinference = false and lower(relClass) = 'isproducedby'");
+			.filter(
+				"dataInfo.deletedbyinference = false and lower(relClass) = '"
+					+ ModelConstants.IS_PRODUCED_BY.toLowerCase() + "'");
 		Dataset<eu.dnetlib.dhp.schema.oaf.Project> projects = Utils
 			.readPath(spark, inputPath + "/project", eu.dnetlib.dhp.schema.oaf.Project.class);
 
@@ -98,7 +102,7 @@ public class SparkPrepareResultProject implements Serializable {
 					rp.setResultId(s);
 					eu.dnetlib.dhp.schema.oaf.Project p = first._1();
 					projectSet.add(p.getId());
-					Project ps = getProject(p);
+					Project ps = getProject(p, first._2);
 
 					List<Project> projList = new ArrayList<>();
 					projList.add(ps);
@@ -107,7 +111,7 @@ public class SparkPrepareResultProject implements Serializable {
 						eu.dnetlib.dhp.schema.oaf.Project op = c._1();
 						if (!projectSet.contains(op.getId())) {
 							projList
-								.add(getProject(op));
+								.add(getProject(op, c._2));
 
 							projectSet.add(op.getId());
 
@@ -122,7 +126,7 @@ public class SparkPrepareResultProject implements Serializable {
 			.json(outputPath);
 	}
 
-	private static Project getProject(eu.dnetlib.dhp.schema.oaf.Project op) {
+	private static Project getProject(eu.dnetlib.dhp.schema.oaf.Project op, Relation relation) {
 		Project p = Project
 			.newInstance(
 				op.getId(),
@@ -157,7 +161,9 @@ public class SparkPrepareResultProject implements Serializable {
 			provenance.setTrust(di.get().getTrust());
 			p.setProvenance(provenance);
 		}
-
+		if (Boolean.TRUE.equals(relation.getValidated())) {
+			p.setValidated(Validated.newInstance(relation.getValidated(), relation.getValidationDate()));
+		}
 		return p;
 
 	}
@@ -173,8 +179,8 @@ public class SparkPrepareResultProject implements Serializable {
 			f.setName(((Node) (doc.selectNodes("//funder/name").get(0))).getText());
 			f.setJurisdiction(((Node) (doc.selectNodes("//funder/jurisdiction").get(0))).getText());
 			for (Object o : doc.selectNodes("//funding_level_0")) {
-				List node = ((Node) o).selectNodes("./name");
+				List<Node> node = ((Node) o).selectNodes("./name");
-				f.setFundingStream(((Node) node.get(0)).getText());
+				f.setFundingStream((node.get(0)).getText());
 			}
 
 			return f;

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateContextRelation.java

@@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.oa.graph.dump.Utils;
+import eu.dnetlib.dhp.oa.graph.dump.exceptions.MyRuntimeException;
 import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.schema.dump.oaf.graph.*;
 import eu.dnetlib.dhp.schema.oaf.Datasource;
@@ -38,6 +39,7 @@ public class CreateContextRelation implements Serializable {
 	private final transient QueryInformationSystem queryInformationSystem;
 
 	private static final String CONTEX_RELATION_DATASOURCE = "contentproviders";
+	private static final String CONTEX_RELATION_PROJECT = "projects";
 
 	public static void main(String[] args) throws Exception {
 		String jsonConfiguration = IOUtils
@@ -72,6 +74,10 @@ public class CreateContextRelation implements Serializable {
 		cce.execute(Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class));
 
 		log.info("Creating relations for projects... ");
+		cce
+			.execute(
+				Process::getRelation, CONTEX_RELATION_PROJECT,
+				ModelSupport.getIdPrefix(eu.dnetlib.dhp.schema.oaf.Project.class));
 
 		cce.close();
 
@@ -115,7 +121,7 @@ public class CreateContextRelation implements Serializable {
 			writer.write(Utils.OBJECT_MAPPER.writeValueAsString(r));
 			writer.newLine();
 		} catch (final Exception e) {
-			throw new RuntimeException(e);
+			throw new MyRuntimeException(e);
 		}
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpGraphEntities.java

@@ -1,6 +1,7 @@
 
 package eu.dnetlib.dhp.oa.graph.dump.complete;
 
+import static com.jayway.jsonpath.Filter.filter;
 import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 
 import java.io.Serializable;
@@ -9,7 +10,10 @@ import java.util.*;
 import java.util.stream.Collectors;
 
 import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.FilterFunction;
+import org.apache.spark.api.java.function.ForeachFunction;
 import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SaveMode;
 import org.apache.spark.sql.SparkSession;
@@ -18,6 +22,8 @@ import org.dom4j.DocumentException;
 import org.dom4j.Node;
 import org.dom4j.io.SAXReader;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import eu.dnetlib.dhp.oa.graph.dump.DumpProducts;
 import eu.dnetlib.dhp.oa.graph.dump.Utils;
 import eu.dnetlib.dhp.schema.common.ModelSupport;
@@ -453,13 +459,17 @@ public class DumpGraphEntities implements Serializable {
 			.map(
 				(MapFunction<E, Organization>) o -> mapOrganization((eu.dnetlib.dhp.schema.oaf.Organization) o),
 				Encoders.bean(Organization.class))
+			.filter((FilterFunction<Organization>) o -> o != null)
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")
 			.json(outputPath);
 	}
 
-	private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) {
+	private static eu.dnetlib.dhp.schema.dump.oaf.graph.Organization mapOrganization(
+		eu.dnetlib.dhp.schema.oaf.Organization org) {
+		if (org.getDataInfo().getDeletedbyinference())
+			return null;
 		Organization organization = new Organization();
 
 		Optional

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java

@@ -62,22 +62,7 @@ public class Extractor implements Serializable {
 			.readPath(spark, inputPath, inputClazz)
 			.flatMap((FlatMapFunction<R, Relation>) value -> {
 				List<Relation> relationList = new ArrayList<>();
-				Optional
+				extractRelationsFromInstance(hashCodes, value, relationList);
-					.ofNullable(value.getInstance())
-					.ifPresent(inst -> inst.forEach(instance -> {
-						Optional
-							.ofNullable(instance.getCollectedfrom())
-							.ifPresent(
-								cf -> getRelatioPair(
-									value, relationList, cf,
-									ModelConstants.IS_PROVIDED_BY, ModelConstants.PROVIDES, hashCodes));
-						Optional
-							.ofNullable(instance.getHostedby())
-							.ifPresent(
-								hb -> getRelatioPair(
-									value, relationList, hb,
-									Constants.IS_HOSTED_BY, Constants.HOSTS, hashCodes));
-					}));
 				Set<String> communities = communityMap.keySet();
 				Optional
 					.ofNullable(value.getContext())
@@ -136,8 +121,28 @@ public class Extractor implements Serializable {
 
 	}
 
+	private <R extends Result> void extractRelationsFromInstance(Set<Integer> hashCodes, R value,
+		List<Relation> relationList) {
+		Optional
+			.ofNullable(value.getInstance())
+			.ifPresent(inst -> inst.forEach(instance -> {
+				Optional
+					.ofNullable(instance.getCollectedfrom())
+					.ifPresent(
+						cf -> getRelatioPair(
+							value, relationList, cf,
+							ModelConstants.IS_PROVIDED_BY, ModelConstants.PROVIDES, hashCodes));
+				Optional
+					.ofNullable(instance.getHostedby())
+					.ifPresent(
+						hb -> getRelatioPair(
+							value, relationList, hb,
+							Constants.IS_HOSTED_BY, Constants.HOSTS, hashCodes));
+			}));
+	}
+
 	private static <R extends Result> void getRelatioPair(R value, List<Relation> relationList, KeyValue cf,
-		String result_dtasource, String datasource_result,
+		String resultDatasource, String datasourceResult,
 		Set<Integer> hashCodes) {
 		Provenance provenance = Optional
 			.ofNullable(cf.getDataInfo())
@@ -147,7 +152,7 @@ public class Extractor implements Serializable {
 					.map(
 						paction -> Provenance
 							.newInstance(
-								paction.getClassid(),
+								paction.getClassname(),
 								dinfo.getTrust()))
 					.orElse(
 						Provenance
@@ -162,7 +167,7 @@ public class Extractor implements Serializable {
 		Relation r = getRelation(
 			value.getId(),
 			cf.getKey(), Constants.RESULT_ENTITY, Constants.DATASOURCE_ENTITY,
-			result_dtasource, ModelConstants.PROVISION,
+			resultDatasource, ModelConstants.PROVISION,
 			provenance);
 		if (!hashCodes.contains(r.hashCode())) {
 			relationList
@@ -173,7 +178,7 @@ public class Extractor implements Serializable {
 		r = getRelation(
 			cf.getKey(), value.getId(),
 			Constants.DATASOURCE_ENTITY, Constants.RESULT_ENTITY,
-			datasource_result, ModelConstants.PROVISION,
+			datasourceResult, ModelConstants.PROVISION,
 			provenance);
 
 		if (!hashCodes.contains(r.hashCode())) {

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Process.java

@@ -9,6 +9,7 @@ import org.apache.commons.lang3.StringUtils;
 
 import eu.dnetlib.dhp.oa.graph.dump.Constants;
 import eu.dnetlib.dhp.oa.graph.dump.Utils;
+import eu.dnetlib.dhp.oa.graph.dump.exceptions.MyRuntimeException;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.schema.dump.oaf.Provenance;
@@ -43,7 +44,7 @@ public class Process implements Serializable {
 			return (R) ri;
 
 		} catch (final Exception e) {
-			throw new RuntimeException(e);
+			throw new MyRuntimeException(e);
 		}
 	}
 
@@ -91,7 +92,7 @@ public class Process implements Serializable {
 			return relationList;
 
 		} catch (final Exception e) {
-			throw new RuntimeException(e);
+			throw new MyRuntimeException(e);
 		}
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/QueryInformationSystem.java

@@ -13,6 +13,8 @@ import org.dom4j.io.SAXReader;
 import org.jetbrains.annotations.NotNull;
 import org.xml.sax.SAXException;
 
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.utils.DHPUtils;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
 
@@ -91,9 +93,9 @@ public class QueryInformationSystem {
 				Element root = doc.getRootElement();
 				cinfo.setId(root.attributeValue("id"));
 
-				Iterator it = root.elementIterator();
+				Iterator<Element> it = root.elementIterator();
 				while (it.hasNext()) {
-					Element el = (Element) it.next();
+					Element el = it.next();
 					if (el.getName().equals("category")) {
 						String categoryId = el.attributeValue("id");
 						categoryId = categoryId.substring(categoryId.lastIndexOf("::") + 2);
@@ -115,14 +117,79 @@ public class QueryInformationSystem {
 	@NotNull
 	private List<String> getCategoryList(Element el, String prefix) {
 		List<String> datasourceList = new ArrayList<>();
-		for (Object node : el.selectNodes(".//param")) {
+		for (Object node : el.selectNodes(".//concept")) {
-			Node n = (Node) node;
+			String oid = getOpenaireId((Node) node, prefix);
-			if (n.valueOf("./@name").equals("openaireId")) {
+			if (oid != null)
-				datasourceList.add(prefix + "|" + n.getText());
+				datasourceList.add(oid);
-			}
 		}
 
 		return datasourceList;
 	}
 
+	private String getOpenaireId(Node el, String prefix) {
+		for (Object node : el.selectNodes(".//param")) {
+			Node n = (Node) node;
+			if (n.valueOf("./@name").equals("openaireId")) {
+				return prefix + "|" + n.getText();
+			}
+		}
+
+		return makeOpenaireId(el, prefix);
+
+	}
+
+	private String makeOpenaireId(Node el, String prefix) {
+		if (!prefix.equals(ModelSupport.entityIdPrefix.get("project"))) {
+			return null;
+		}
+		String funder = "";
+		String grantId = null;
+		String funding = null;
+		for (Object node : el.selectNodes(".//param")) {
+			Node n = (Node) node;
+			switch (n.valueOf("./@name")) {
+				case "funding":
+					funding = n.getText();
+					break;
+				case "funder":
+					funder = n.getText();
+					break;
+				case "CD_PROJECT_NUMBER":
+					grantId = n.getText();
+					break;
+				default:
+					break;
+			}
+		}
+		String nsp = null;
+
+		switch (funder.toLowerCase()) {
+			case "ec":
+				if (funding == null) {
+					return null;
+				}
+				if (funding.toLowerCase().contains("h2020")) {
+					nsp = "corda__h2020::";
+				} else {
+					nsp = "corda_______::";
+				}
+				break;
+			case "tubitak":
+				nsp = "tubitakf____::";
+				break;
+			case "dfg":
+				nsp = "dfgf________::";
+				break;
+			default:
+				StringBuilder bld = new StringBuilder();
+				bld.append(funder.toLowerCase());
+				for (int i = funder.length(); i < 12; i++)
+					bld.append("_");
+				bld.append("::");
+				nsp = bld.toString();
+		}
+
+		return prefix + "|" + nsp + DHPUtils.md5(grantId);
+	}
+
 }

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java

@@ -71,22 +71,22 @@ public class SparkDumpRelationJob implements Serializable {
 		Dataset<Relation> relations = Utils.readPath(spark, inputPath, Relation.class);
 		relations
 			.map((MapFunction<Relation, eu.dnetlib.dhp.schema.dump.oaf.graph.Relation>) relation -> {
-				eu.dnetlib.dhp.schema.dump.oaf.graph.Relation rel_new = new eu.dnetlib.dhp.schema.dump.oaf.graph.Relation();
+				eu.dnetlib.dhp.schema.dump.oaf.graph.Relation relNew = new eu.dnetlib.dhp.schema.dump.oaf.graph.Relation();
-				rel_new
+				relNew
 					.setSource(
 						Node
 							.newInstance(
 								relation.getSource(),
 								ModelSupport.idPrefixEntity.get(relation.getSource().substring(0, 2))));
 
-				rel_new
+				relNew
 					.setTarget(
 						Node
 							.newInstance(
 								relation.getTarget(),
 								ModelSupport.idPrefixEntity.get(relation.getTarget().substring(0, 2))));
 
-				rel_new
+				relNew
 					.setReltype(
 						RelType
 							.newInstance(
@@ -96,26 +96,22 @@ public class SparkDumpRelationJob implements Serializable {
 				Optional<DataInfo> odInfo = Optional.ofNullable(relation.getDataInfo());
 				if (odInfo.isPresent()) {
 					DataInfo dInfo = odInfo.get();
-					if (Optional.ofNullable(dInfo.getProvenanceaction()).isPresent()) {
+					if (Optional.ofNullable(dInfo.getProvenanceaction()).isPresent() &&
-						if (Optional.ofNullable(dInfo.getProvenanceaction().getClassname()).isPresent()) {
+						Optional.ofNullable(dInfo.getProvenanceaction().getClassname()).isPresent()) {
-							rel_new
+						relNew
-								.setProvenance(
+							.setProvenance(
-									Provenance
+								Provenance
-										.newInstance(
+									.newInstance(
-											dInfo.getProvenanceaction().getClassname(),
+										dInfo.getProvenanceaction().getClassname(),
-											dInfo.getTrust()));
+										dInfo.getTrust()));
-						}
 					}
 				}
-//						Optional
+				if (Boolean.TRUE.equals(relation.getValidated())) {
-//								.ofNullable(relation.getDataInfo())
+					relNew.setValidated(relation.getValidated());
-//								.ifPresent(
+					relNew.setValidationDate(relation.getValidationDate());
-//										datainfo -> rel_new
+				}
-//												.setProvenance(
-//														Provenance
-//																.newInstance(datainfo.getProvenanceaction().getClassname(), datainfo.getTrust())));
 
-				return rel_new;
+				return relNew;
 
 			}, Encoders.bean(eu.dnetlib.dhp.schema.dump.oaf.graph.Relation.class))
 			.write()

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java

@@ -58,7 +58,8 @@ public class SparkOrganizationRelation implements Serializable {
 
 		final OrganizationMap organizationMap = new Gson()
 			.fromJson(parser.get("organizationCommunityMap"), OrganizationMap.class);
-		log.info("organization map : {}", new Gson().toJson(organizationMap));
+		final String serializedOrganizationMap = new Gson().toJson(organizationMap);
+		log.info("organization map : {}", serializedOrganizationMap);
 
 		final String communityMapPath = parser.get("communityMapPath");
 		log.info("communityMapPath: {}", communityMapPath);

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkSelectValidRelationsJob.java

@@ -0,0 +1,136 @@
+
+package eu.dnetlib.dhp.oa.graph.dump.complete;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.io.Serializable;
+import java.util.Optional;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.oa.graph.dump.Utils;
+import eu.dnetlib.dhp.schema.oaf.*;
+
+/**
+ * It selects the valid relations among those present in the graph. One relation is valid if it is not deletedbyinference
+ * and if both the source and the target node are present in the graph and are not deleted by inference nor invisible.
+ * To check this I made a view of the ids of all the entities in the graph, and select the relations for which a join exists
+ * with this view for both the source and the target
+ */
+
+public class SparkSelectValidRelationsJob implements Serializable {
+
+	private static final Logger log = LoggerFactory.getLogger(SparkSelectValidRelationsJob.class);
+
+	public static void main(String[] args) throws Exception {
+		String jsonConfiguration = IOUtils
+			.toString(
+				SparkSelectValidRelationsJob.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/oa/graph/dump/complete/input_relationdump_parameters.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		final String inputPath = parser.get("sourcePath");
+		log.info("inputPath: {}", inputPath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath: {}", outputPath);
+
+		SparkConf conf = new SparkConf();
+
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				Utils.removeOutputDir(spark, outputPath);
+				selectValidRelation(spark, inputPath, outputPath);
+
+			});
+
+	}
+
+	private static void selectValidRelation(SparkSession spark, String inputPath, String outputPath) {
+		Dataset<Relation> relation = Utils.readPath(spark, inputPath + "/relation", Relation.class);
+		Dataset<Publication> publication = Utils.readPath(spark, inputPath + "/publication", Publication.class);
+		Dataset<eu.dnetlib.dhp.schema.oaf.Dataset> dataset = Utils
+			.readPath(spark, inputPath + "/dataset", eu.dnetlib.dhp.schema.oaf.Dataset.class);
+		Dataset<Software> software = Utils.readPath(spark, inputPath + "/software", Software.class);
+		Dataset<OtherResearchProduct> other = Utils
+			.readPath(spark, inputPath + "/otherresearchproduct", OtherResearchProduct.class);
+		Dataset<Organization> organization = Utils.readPath(spark, inputPath + "/organization", Organization.class);
+		Dataset<Project> project = Utils.readPath(spark, inputPath + "/project", Project.class);
+		Dataset<Datasource> datasource = Utils.readPath(spark, inputPath + "/datasource", Datasource.class);
+
+		relation.createOrReplaceTempView("relation");
+		publication.createOrReplaceTempView("publication");
+		dataset.createOrReplaceTempView("dataset");
+		other.createOrReplaceTempView("other");
+		software.createOrReplaceTempView("software");
+		organization.createOrReplaceTempView("organization");
+		project.createOrReplaceTempView("project");
+		datasource.createOrReplaceTempView("datasource");
+
+		spark
+			.sql(
+				"SELECT id " +
+					"FROM publication " +
+					"WHERE datainfo.deletedbyinference = false AND  datainfo.invisible = false " +
+					"UNION ALL " +
+					"SELECT id " +
+					"FROM dataset " +
+					"WHERE datainfo.deletedbyinference = false AND  datainfo.invisible = false " +
+					"UNION ALL " +
+					"SELECT id " +
+					"FROM other " +
+					"WHERE datainfo.deletedbyinference = false AND  datainfo.invisible = false " +
+					"UNION ALL " +
+					"SELECT id " +
+					"FROM software " +
+					"WHERE datainfo.deletedbyinference = false AND  datainfo.invisible = false " +
+					"UNION ALL " +
+					"SELECT id " +
+					"FROM organization " +
+					"WHERE datainfo.deletedbyinference = false AND  datainfo.invisible = false " +
+					"UNION ALL " +
+					"SELECT id " +
+					"FROM project " +
+					"WHERE datainfo.deletedbyinference = false AND  datainfo.invisible = false " +
+					"UNION ALL " +
+					"SELECT id " +
+					"FROM datasource " +
+					"WHERE datainfo.deletedbyinference = false AND  datainfo.invisible = false ")
+			.createOrReplaceTempView("identifiers");
+
+		spark
+			.sql(
+				"SELECT relation.* " +
+					"FROM relation " +
+					"JOIN identifiers i1 " +
+					"ON source = i1.id " +
+					"JOIN identifiers i2 " +
+					"ON target = i2.id " +
+					"WHERE datainfo.deletedbyinference = false")
+			.as(Encoders.bean(Relation.class))
+			.write()
+			.option("compression", "gzip")
+			.mode(SaveMode.Overwrite)
+			.json(outputPath);
+
+	}
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/exceptions/MyRuntimeException.java

@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.oa.graph.dump.exceptions;
+
+public class MyRuntimeException extends RuntimeException {
+
+	public MyRuntimeException() {
+		super();
+	}
+
+	public MyRuntimeException(
+		final String message,
+		final Throwable cause,
+		final boolean enableSuppression,
+		final boolean writableStackTrace) {
+		super(message, cause, enableSuppression, writableStackTrace);
+	}
+
+	public MyRuntimeException(final String message, final Throwable cause) {
+		super(message, cause);
+	}
+
+	public MyRuntimeException(final String message) {
+		super(message);
+	}
+
+	public MyRuntimeException(final Throwable cause) {
+		super(cause);
+	}
+
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/exceptions/NoAvailableEntityTypeException.java

@@ -0,0 +1,29 @@
+
+package eu.dnetlib.dhp.oa.graph.dump.exceptions;
+
+public class NoAvailableEntityTypeException extends Exception {
+	public NoAvailableEntityTypeException() {
+		super();
+	}
+
+	public NoAvailableEntityTypeException(
+		final String message,
+		final Throwable cause,
+		final boolean enableSuppression,
+		final boolean writableStackTrace) {
+		super(message, cause, enableSuppression, writableStackTrace);
+	}
+
+	public NoAvailableEntityTypeException(final String message, final Throwable cause) {
+		super(message, cause);
+	}
+
+	public NoAvailableEntityTypeException(final String message) {
+		super(message);
+	}
+
+	public NoAvailableEntityTypeException(final Throwable cause) {
+		super(cause);
+	}
+
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/funderresults/SparkDumpFunderResults.java

@@ -17,10 +17,8 @@ import org.slf4j.LoggerFactory;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.oa.graph.dump.Utils;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
 import eu.dnetlib.dhp.schema.dump.oaf.community.Project;
-import eu.dnetlib.dhp.schema.oaf.Relation;
 
 /**
  * Splits the dumped results by funder and stores them in a folder named as the funder nsp (for all the funders, but the EC
@@ -51,8 +49,8 @@ public class SparkDumpFunderResults implements Serializable {
 		final String outputPath = parser.get("outputPath");
 		log.info("outputPath: {}", outputPath);
 
-		final String relationPath = parser.get("relationPath");
+		final String graphPath = parser.get("graphPath");
-		log.info("relationPath: {}", relationPath);
+		log.info("relationPath: {}", graphPath);
 
 		SparkConf conf = new SparkConf();
 
@@ -61,18 +59,15 @@ public class SparkDumpFunderResults implements Serializable {
 			isSparkSessionManaged,
 			spark -> {
 				Utils.removeOutputDir(spark, outputPath);
-				writeResultProjectList(spark, inputPath, outputPath, relationPath);
+				writeResultProjectList(spark, inputPath, outputPath, graphPath);
 			});
 	}
 
 	private static void writeResultProjectList(SparkSession spark, String inputPath, String outputPath,
-		String relationPath) {
+		String graphPath) {
 
-		Dataset<Relation> relation = Utils
+		Dataset<eu.dnetlib.dhp.schema.oaf.Project> project = Utils
-			.readPath(spark, relationPath + "/relation", Relation.class)
+			.readPath(spark, graphPath + "/project", eu.dnetlib.dhp.schema.oaf.Project.class);
-			.filter(
-				"dataInfo.deletedbyinference = false and lower(relClass) = '"
-					+ ModelConstants.IS_PRODUCED_BY.toLowerCase() + "'");
 
 		Dataset<CommunityResult> result = Utils
 			.readPath(spark, inputPath + "/publication", CommunityResult.class)
@@ -80,8 +75,8 @@ public class SparkDumpFunderResults implements Serializable {
 			.union(Utils.readPath(spark, inputPath + "/orp", CommunityResult.class))
 			.union(Utils.readPath(spark, inputPath + "/software", CommunityResult.class));
 
-		List<String> funderList = relation
+		List<String> funderList = project
-			.select("target")
+			.select("id")
 			.map((MapFunction<Row, String>) value -> value.getString(0).substring(0, 15), Encoders.STRING())
 			.distinct()
 			.collectAsList();
@@ -99,19 +94,26 @@ public class SparkDumpFunderResults implements Serializable {
 			} else {
 				funderdump = fundernsp.substring(0, fundernsp.indexOf("_")).toUpperCase();
 			}
-			writeFunderResult(funder, result, outputPath + "/" + funderdump);
+			writeFunderResult(funder, result, outputPath, funderdump);
 		});
 
 	}
 
-	private static void writeFunderResult(String funder, Dataset<CommunityResult> results, String outputPath) {
+	private static void dumpResults(String nsp, Dataset<CommunityResult> results, String outputPath,
+		String funderName) {
 
 		results.map((MapFunction<CommunityResult, CommunityResult>) r -> {
 			if (!Optional.ofNullable(r.getProjects()).isPresent()) {
 				return null;
 			}
 			for (Project p : r.getProjects()) {
-				if (p.getId().startsWith(funder)) {
+				if (p.getId().startsWith(nsp)) {
+					if (nsp.startsWith("40|irb")) {
+						if (p.getFunder().getShortName().equals(funderName))
+							return r;
+						else
+							return null;
+					}
 					return r;
 				}
 			}
@@ -121,7 +123,18 @@ public class SparkDumpFunderResults implements Serializable {
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")
-			.json(outputPath);
+			.json(outputPath + "/" + funderName);
+	}
+
+	private static void writeFunderResult(String funder, Dataset<CommunityResult> results, String outputPath,
+		String funderDump) {
+
+		if (funder.startsWith("40|irb")) {
+			dumpResults(funder, results, outputPath, "HRZZ");
+			dumpResults(funder, results, outputPath, "MZOS");
+		} else
+			dumpResults(funder, results, outputPath, funderDump);
+
 	}
 
 }

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/funderresults/SparkResultLinkedToProject.java

@@ -20,13 +20,13 @@ import org.slf4j.LoggerFactory;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.oa.graph.dump.Utils;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.Project;
 import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.schema.oaf.Result;
-import scala.Tuple2;
 
 /**
  * Selects the results linked to projects. Only for these results the dump will be performed.
- * The code to perform the dump and to expend the dumped results with the informaiton related to projects
+ * The code to perform the dump and to expend the dumped results with the information related to projects
  * is the one used for the dump of the community products
  */
 public class SparkResultLinkedToProject implements Serializable {
@@ -58,8 +58,8 @@ public class SparkResultLinkedToProject implements Serializable {
 		final String resultClassName = parser.get("resultTableName");
 		log.info("resultTableName: {}", resultClassName);
 
-		final String relationPath = parser.get("relationPath");
+		final String graphPath = parser.get("graphPath");
-		log.info("relationPath: {}", relationPath);
+		log.info("graphPath: {}", graphPath);
 
 		@SuppressWarnings("unchecked")
 		Class<? extends Result> inputClazz = (Class<? extends Result>) Class.forName(resultClassName);
@@ -70,37 +70,47 @@ public class SparkResultLinkedToProject implements Serializable {
 			isSparkSessionManaged,
 			spark -> {
 				Utils.removeOutputDir(spark, outputPath);
-				writeResultsLinkedToProjects(spark, inputClazz, inputPath, outputPath, relationPath);
+				writeResultsLinkedToProjects(spark, inputClazz, inputPath, outputPath, graphPath);
 			});
 	}
 
 	private static <R extends Result> void writeResultsLinkedToProjects(SparkSession spark, Class<R> inputClazz,
-		String inputPath, String outputPath, String relationPath) {
+		String inputPath, String outputPath, String graphPath) {
 
 		Dataset<R> results = Utils
 			.readPath(spark, inputPath, inputClazz)
 			.filter("dataInfo.deletedbyinference = false and datainfo.invisible = false");
 		Dataset<Relation> relations = Utils
-			.readPath(spark, relationPath, Relation.class)
+			.readPath(spark, graphPath + "/relation", Relation.class)
 			.filter(
 				"dataInfo.deletedbyinference = false and lower(relClass) = '"
 					+ ModelConstants.IS_PRODUCED_BY.toLowerCase() + "'");
+		Dataset<Project> project = Utils.readPath(spark, graphPath + "/project", Project.class);
 
-		relations
+		results.createOrReplaceTempView("result");
-			.joinWith(
+		relations.createOrReplaceTempView("relation");
-				results, relations.col("source").equalTo(results.col("id")),
+		project.createOrReplaceTempView("project");
-				"inner")
+
+		Dataset<R> tmp = spark
+			.sql(
+				"Select res.* " +
+					"from relation rel " +
+					"join result res " +
+					"on rel.source = res.id " +
+					"join project p " +
+					"on rel.target = p.id " +
+					"")
+			.as(Encoders.bean(inputClazz));
+		tmp
 			.groupByKey(
-				(MapFunction<Tuple2<Relation, R>, String>) value -> value
+				(MapFunction<R, String>) value -> value
-					._2()
 					.getId(),
 				Encoders.STRING())
-			.mapGroups(
+			.mapGroups((MapGroupsFunction<String, R, R>) (k, it) -> it.next(), Encoders.bean(inputClazz))
-				(MapGroupsFunction<String, Tuple2<Relation, R>, R>) (k, it) -> it.next()._2(),
-				Encoders.bean(inputClazz))
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")
 			.json(outputPath);
+
 	}
 }

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/merge/MergeGraphTableSparkJob.java

@@ -127,13 +127,6 @@ public class MergeGraphTableSparkJob {
 				}
 			}, Encoders.bean(p_clazz))
 			.filter((FilterFunction<P>) Objects::nonNull)
-			.filter((FilterFunction<P>) o -> {
-				HashSet<String> collectedFromNames = Optional
-					.ofNullable(o.getCollectedfrom())
-					.map(c -> c.stream().map(KeyValue::getValue).collect(Collectors.toCollection(HashSet::new)))
-					.orElse(new HashSet<>());
-				return !collectedFromNames.contains("Datacite");
-			})
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/CopyHdfsOafSparkApplication.scala

@@ -0,0 +1,74 @@
+package eu.dnetlib.dhp.oa.graph.raw
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.common.HdfsSupport
+import eu.dnetlib.dhp.schema.common.ModelSupport
+import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo
+import eu.dnetlib.dhp.schema.oaf.Oaf
+import eu.dnetlib.dhp.utils.DHPUtils
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang3.StringUtils
+import org.apache.http.client.methods.HttpGet
+import org.apache.http.impl.client.HttpClients
+import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.{SparkConf, SparkContext}
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+import scala.io.Source
+
+object CopyHdfsOafSparkApplication {
+
+  def main(args: Array[String]): Unit = {
+    val log = LoggerFactory.getLogger(getClass)
+    val conf = new SparkConf()
+    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/copy_hdfs_oaf_parameters.json")).mkString)
+    parser.parseArgument(args)
+
+    val spark =
+      SparkSession
+        .builder()
+        .config(conf)
+        .appName(getClass.getSimpleName)
+        .master(parser.get("master")).getOrCreate()
+
+    val sc: SparkContext = spark.sparkContext
+
+    val mdstoreManagerUrl = parser.get("mdstoreManagerUrl")
+    log.info("mdstoreManagerUrl: {}", mdstoreManagerUrl)
+
+    val mdFormat = parser.get("mdFormat")
+    log.info("mdFormat: {}", mdFormat)
+
+    val mdLayout = parser.get("mdLayout")
+    log.info("mdLayout: {}", mdLayout)
+
+    val mdInterpretation = parser.get("mdInterpretation")
+    log.info("mdInterpretation: {}", mdInterpretation)
+
+    val hdfsPath = parser.get("hdfsPath")
+    log.info("hdfsPath: {}", hdfsPath)
+
+    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
+
+    val paths = DHPUtils.mdstorePaths(mdstoreManagerUrl, mdFormat, mdLayout, mdInterpretation, true).asScala
+
+    val validPaths: List[String] = paths.filter(p => HdfsSupport.exists(p, sc.hadoopConfiguration)).toList
+
+    if (validPaths.nonEmpty) {
+      val oaf = spark.read.load(validPaths: _*).as[Oaf]
+      val mapper = new ObjectMapper()
+      val l =ModelSupport.oafTypes.entrySet.asScala.map(e => e.getKey).toList
+      l.foreach(
+        e =>
+          oaf.filter(o => o.getClass.getSimpleName.equalsIgnoreCase(e))
+            .map(s => mapper.writeValueAsString(s))(Encoders.STRING)
+            .write
+            .option("compression", "gzip")
+            .mode(SaveMode.Append)
+            .text(s"$hdfsPath/${e}")
+      )
+    }
+  }
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java

@@ -186,6 +186,9 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
 
 					log.info("Processing Openorgs Merge Rels...");
 					smdbe.execute("queryOpenOrgsSimilarityForProvision.sql", smdbe::processOrgOrgMergeRels);
+
+					log.info("Processing Openorgs Parent/Child Rels...");
+					smdbe.execute("queryParentChildRelsOpenOrgs.sql", smdbe::processOrgOrgParentChildRels);
 					break;
 
 				case openaire_organizations:
@@ -689,6 +692,35 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
 		}
 	}
 
+	public List<Oaf> processOrgOrgParentChildRels(final ResultSet rs) {
+		try {
+			final DataInfo info = prepareDataInfo(rs); // TODO
+
+			final String orgId1 = createOpenaireId(20, rs.getString("source"), true);
+			final String orgId2 = createOpenaireId(20, rs.getString("target"), true);
+
+			final List<KeyValue> collectedFrom = listKeyValues(
+				createOpenaireId(10, rs.getString("collectedfromid"), true), rs.getString("collectedfromname"));
+
+			final Relation r = new Relation();
+			r.setRelType(ORG_ORG_RELTYPE);
+			r.setSubRelType(ModelConstants.RELATIONSHIP);
+			r
+				.setRelClass(
+					rs.getString("type").equalsIgnoreCase("parent") ? ModelConstants.IS_PARENT_OF
+						: ModelConstants.IS_CHILD_OF);
+			r.setSource(orgId1);
+			r.setTarget(orgId2);
+			r.setCollectedfrom(collectedFrom);
+			r.setDataInfo(info);
+			r.setLastupdatetimestamp(lastUpdateTimestamp);
+
+			return Arrays.asList(r);
+		} catch (final Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
 	public List<Oaf> processOrgOrgSimRels(final ResultSet rs) {
 		try {
 			final DataInfo info = prepareDataInfo(rs); // TODO

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateHdfsMdstoresApplication.java

@@ -135,30 +135,4 @@ public class MigrateHdfsMdstoresApplication extends AbstractMigrationApplication
 		}
 	}
 
-	private static Set<String> mdstorePaths(final String mdstoreManagerUrl,
-		final String format,
-		final String layout,
-		final String interpretation) throws IOException {
-		final String url = mdstoreManagerUrl + "/mdstores/";
-		final ObjectMapper objectMapper = new ObjectMapper();
-
-		final HttpGet req = new HttpGet(url);
-
-		try (final CloseableHttpClient client = HttpClients.createDefault()) {
-			try (final CloseableHttpResponse response = client.execute(req)) {
-				final String json = IOUtils.toString(response.getEntity().getContent());
-				final MDStoreWithInfo[] mdstores = objectMapper.readValue(json, MDStoreWithInfo[].class);
-				return Arrays
-					.stream(mdstores)
-					.filter(md -> md.getFormat().equalsIgnoreCase(format))
-					.filter(md -> md.getLayout().equalsIgnoreCase(layout))
-					.filter(md -> md.getInterpretation().equalsIgnoreCase(interpretation))
-					.filter(md -> StringUtils.isNotBlank(md.getHdfsPath()))
-					.filter(md -> StringUtils.isNotBlank(md.getCurrentVersion()))
-					.filter(md -> md.getSize() > 0)
-					.map(md -> md.getHdfsPath() + "/" + md.getCurrentVersion() + "/store")
-					.collect(Collectors.toSet());
-			}
-		}
-	}
 }

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/AbstractMigrationApplication.java

@@ -3,19 +3,30 @@ package eu.dnetlib.dhp.oa.graph.raw.common;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
 import eu.dnetlib.dhp.schema.oaf.Oaf;
+import eu.dnetlib.dhp.utils.DHPUtils;
 
 public class AbstractMigrationApplication implements Closeable {
 
@@ -47,6 +58,23 @@ public class AbstractMigrationApplication implements Closeable {
 				SequenceFile.Writer.valueClass(Text.class));
 	}
 
+	/**
+	 * Retrieves from the metadata store manager application the list of paths associated with mdstores characterized
+	 * by he given format, layout, interpretation
+	 * @param mdstoreManagerUrl the URL of the mdstore manager service
+	 * @param format the mdstore format
+	 * @param layout the mdstore layout
+	 * @param interpretation the mdstore interpretation
+	 * @return the set of hdfs paths
+	 * @throws IOException in case of HTTP communication issues
+	 */
+	protected static Set<String> mdstorePaths(final String mdstoreManagerUrl,
+		final String format,
+		final String layout,
+		final String interpretation) throws IOException {
+		return DHPUtils.mdstorePaths(mdstoreManagerUrl, format, layout, interpretation, false);
+	}
+
 	private Configuration getConf() {
 		return new Configuration();
 		/*

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/resolution/SparkResolveRelation.scala

@@ -0,0 +1,153 @@
+package eu.dnetlib.dhp.oa.graph.resolution
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.common.HdfsSupport
+import eu.dnetlib.dhp.schema.oaf.{Relation, Result}
+import eu.dnetlib.dhp.utils.DHPUtils
+import org.apache.commons.io.IOUtils
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.spark.SparkConf
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql._
+import org.json4s
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST.{JField, JObject, JString}
+import org.json4s.jackson.JsonMethods.parse
+import org.slf4j.{Logger, LoggerFactory}
+
+object SparkResolveRelation {
+  def main(args: Array[String]): Unit = {
+    val log: Logger = LoggerFactory.getLogger(getClass)
+    val conf: SparkConf = new SparkConf()
+    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/resolution/resolve_relations_params.json")))
+    parser.parseArgument(args)
+    val spark: SparkSession =
+      SparkSession
+        .builder()
+        .config(conf)
+        .appName(getClass.getSimpleName)
+        .master(parser.get("master")).getOrCreate()
+
+
+    val graphBasePath = parser.get("graphBasePath")
+    log.info(s"graphBasePath  -> $graphBasePath")
+    val workingPath = parser.get("workingPath")
+    log.info(s"workingPath  -> $workingPath")
+
+    implicit val relEncoder: Encoder[Relation] = Encoders.kryo(classOf[Relation])
+    import spark.implicits._
+
+
+    //CLEANING TEMPORARY FOLDER
+    HdfsSupport.remove(workingPath, spark.sparkContext.hadoopConfiguration)
+    val fs = FileSystem.get(spark.sparkContext.hadoopConfiguration)
+    fs.mkdirs(new Path(workingPath))
+
+    extractPidResolvedTableFromJsonRDD(spark, graphBasePath, workingPath)
+
+    val mapper: ObjectMapper = new ObjectMapper()
+
+    val rPid: Dataset[(String, String)] = spark.read.load(s"$workingPath/relationResolvedPid").as[(String, String)]
+
+    val relationDs: Dataset[(String, Relation)] = spark.read.text(s"$graphBasePath/relation").as[String]
+      .map(s => mapper.readValue(s, classOf[Relation])).as[Relation]
+      .map(r => (r.getSource.toLowerCase, r))(Encoders.tuple(Encoders.STRING, relEncoder))
+
+    relationDs.joinWith(rPid, relationDs("_1").equalTo(rPid("_2")), "left").map {
+      m =>
+        val sourceResolved = m._2
+        val currentRelation = m._1._2
+        if (sourceResolved != null && sourceResolved._1 != null && sourceResolved._1.nonEmpty)
+          currentRelation.setSource(sourceResolved._1)
+        currentRelation
+    }.write
+      .mode(SaveMode.Overwrite)
+      .save(s"$workingPath/relationResolvedSource")
+
+
+    val relationSourceResolved: Dataset[(String, Relation)] = spark.read.load(s"$workingPath/relationResolvedSource").as[Relation]
+      .map(r => (r.getTarget.toLowerCase, r))(Encoders.tuple(Encoders.STRING, relEncoder))
+    relationSourceResolved.joinWith(rPid, relationSourceResolved("_1").equalTo(rPid("_2")), "left").map {
+      m =>
+        val targetResolved = m._2
+        val currentRelation = m._1._2
+        if (targetResolved != null && targetResolved._1.nonEmpty)
+          currentRelation.setTarget(targetResolved._1)
+        currentRelation
+    }
+      .write
+      .mode(SaveMode.Overwrite)
+      .save(s"$workingPath/relation_resolved")
+
+
+    // TO BE conservative we keep the original relation in the working dir
+    // and save the relation resolved on the graphBasePath
+    //In future this two line of code should be removed
+
+    fs.rename(new Path(s"$graphBasePath/relation"), new Path(s"$workingPath/relation"))
+
+    spark.read.load(s"$workingPath/relation_resolved").as[Relation]
+      .filter(r => !r.getSource.startsWith("unresolved") && !r.getTarget.startsWith("unresolved"))
+      .map(r => mapper.writeValueAsString(r))
+      .write
+      .option("compression", "gzip")
+      .mode(SaveMode.Overwrite)
+      .text(s"$graphBasePath/relation")
+  }
+
+
+  def extractPidsFromRecord(input: String): (String, List[(String, String)]) = {
+    implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
+    lazy val json: json4s.JValue = parse(input)
+    val id: String = (json \ "id").extract[String]
+    val result: List[(String, String)] = for {
+      JObject(pids) <- json \\ "instance" \ "pid"
+      JField("value", JString(pidValue)) <- pids
+      JField("qualifier", JObject(qualifier)) <- pids
+      JField("classid", JString(pidType)) <- qualifier
+    } yield (pidValue, pidType)
+
+    val alternateIds: List[(String, String)] = for {
+      JObject(pids) <- json \\ "alternateIdentifier"
+      JField("value", JString(pidValue)) <- pids
+      JField("qualifier", JObject(qualifier)) <- pids
+      JField("classid", JString(pidType)) <- qualifier
+    } yield (pidValue, pidType)
+
+    (id, result ::: alternateIds)
+  }
+
+
+  private def isRelation(input: String): Boolean = {
+
+    implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
+    lazy val json: json4s.JValue = parse(input)
+    val source = (json \ "source").extractOrElse[String](null)
+
+    source != null
+  }
+
+  private def extractPidResolvedTableFromJsonRDD(spark: SparkSession, graphPath: String, workingPath: String) = {
+    import spark.implicits._
+
+    val d: RDD[(String, String)] = spark.sparkContext.textFile(s"$graphPath/*")
+      .filter(i => !isRelation(i))
+      .map(i => extractPidsFromRecord(i))
+      .filter(s => s != null && s._1 != null && s._2 != null && s._2.nonEmpty)
+      .flatMap { p =>
+        p._2.map(pid =>
+          (p._1, DHPUtils.generateUnresolvedIdentifier(pid._1, pid._2))
+        )
+      }.filter(r => r._1 != null || r._2 != null)
+
+    spark.createDataset(d)
+      .groupByKey(_._2)
+      .reduceGroups((x, y) => if (x._1.startsWith("50|doi") || x._1.startsWith("50|pmid")) x else y)
+      .map(s => s._2)
+      .write
+      .mode(SaveMode.Overwrite)
+      .save(s"$workingPath/relationResolvedPid")
+  }
+
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/sx/graph/SparkResolveRelation.scala

@@ -1,154 +0,0 @@
-package eu.dnetlib.dhp.sx.graph
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Relation, Result}
-import org.apache.commons.io.IOUtils
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.spark.SparkConf
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql._
-import org.json4s
-import org.json4s.DefaultFormats
-import org.json4s.JsonAST.{JField, JObject, JString}
-import org.json4s.jackson.JsonMethods.parse
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.collection.JavaConverters._
-object SparkResolveRelation {
-  def main(args: Array[String]): Unit = {
-    val log: Logger = LoggerFactory.getLogger(getClass)
-    val conf: SparkConf = new SparkConf()
-    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/resolve_relations_params.json")))
-    parser.parseArgument(args)
-    val spark: SparkSession =
-      SparkSession
-        .builder()
-        .config(conf)
-        .appName(getClass.getSimpleName)
-        .master(parser.get("master")).getOrCreate()
-
-
-    val relationPath = parser.get("relationPath")
-    log.info(s"sourcePath  -> $relationPath")
-    val entityPath = parser.get("entityPath")
-    log.info(s"entityPath  -> $entityPath")
-    val workingPath = parser.get("workingPath")
-    log.info(s"workingPath  -> $workingPath")
-
-    implicit  val relEncoder: Encoder[Relation] = Encoders.kryo(classOf[Relation])
-    import spark.implicits._
-
-
-    extractPidResolvedTableFromJsonRDD(spark, entityPath, workingPath)
-
-    val mappper = new ObjectMapper()
-
-    val rPid:Dataset[(String,String)] = spark.read.load(s"$workingPath/relationResolvedPid").as[(String,String)]
-
-    val relationDs:Dataset[(String,Relation)] = spark.read.load(relationPath).as[Relation].map(r => (r.getSource.toLowerCase, r))(Encoders.tuple(Encoders.STRING, relEncoder))
-
-    relationDs.joinWith(rPid, relationDs("_1").equalTo(rPid("_2")), "left").map{
-      m =>
-        val sourceResolved = m._2
-        val currentRelation = m._1._2
-        if (sourceResolved!=null && sourceResolved._1!=null && sourceResolved._1.nonEmpty)
-          currentRelation.setSource(sourceResolved._1)
-        currentRelation
-    }.write
-      .mode(SaveMode.Overwrite)
-      .save(s"$workingPath/relationResolvedSource")
-
-
-    val relationSourceResolved:Dataset[(String,Relation)] = spark.read.load(s"$workingPath/relationResolvedSource").as[Relation].map(r => (r.getTarget.toLowerCase, r))(Encoders.tuple(Encoders.STRING, relEncoder))
-    relationSourceResolved.joinWith(rPid, relationSourceResolved("_1").equalTo(rPid("_2")), "left").map{
-      m =>
-        val targetResolved = m._2
-        val currentRelation = m._1._2
-        if (targetResolved!=null && targetResolved._1.nonEmpty)
-          currentRelation.setTarget(targetResolved._1)
-        currentRelation
-    }.filter(r => r.getSource.startsWith("50")&& r.getTarget.startsWith("50"))
-      .write
-      .mode(SaveMode.Overwrite)
-      .save(s"$workingPath/relation_resolved")
-
-    spark.read.load(s"$workingPath/relation_resolved").as[Relation]
-                    .map(r => mappper.writeValueAsString(r))
-                    .rdd.saveAsTextFile(s"$workingPath/relation", classOf[GzipCodec])
-
-  }
-
-
-  def extractPidsFromRecord(input:String):(String,List[(String,String)]) = {
-    implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
-    lazy val json: json4s.JValue = parse(input)
-    val id:String = (json \ "id").extract[String]
-    val result: List[(String,String)] = for {
-      JObject(pids) <- json \ "pid"
-      JField("value", JString(pidValue)) <- pids
-      JField("qualifier", JObject(qualifier)) <- pids
-      JField("classname", JString(pidType)) <- qualifier
-    } yield (pidValue, pidType)
-
-    val alternateIds: List[(String,String)] = for {
-      JObject(pids) <- json \\ "alternateIdentifier"
-      JField("value", JString(pidValue)) <- pids
-      JField("qualifier", JObject(qualifier)) <- pids
-      JField("classname", JString(pidType)) <- qualifier
-    } yield (pidValue, pidType)
-
-    (id,result:::alternateIds)
-  }
-
-  private def extractPidResolvedTableFromJsonRDD(spark: SparkSession, entityPath: String, workingPath: String) = {
-    import spark.implicits._
-
-    val d: RDD[(String,String)] = spark.sparkContext.textFile(s"$entityPath/*")
-      .map(i => extractPidsFromRecord(i))
-      .filter(s => s != null && s._1!= null && s._2!=null && s._2.nonEmpty)
-      .flatMap{ p =>
-                  p._2.map(pid =>
-                    (p._1, convertPidToDNETIdentifier(pid._1, pid._2))
-                  )
-      }.filter(r =>r._1 != null || r._2 != null)
-
-    spark.createDataset(d)
-    .groupByKey(_._2)
-      .reduceGroups((x, y) => if (x._1.startsWith("50|doi") || x._1.startsWith("50|pmid")) x else y)
-      .map(s => s._2)
-      .write
-      .mode(SaveMode.Overwrite)
-      .save(s"$workingPath/relationResolvedPid")
-  }
-
-
-  /*
-    This method should be used once we finally convert everythings in Kryo dataset
-    instead of using rdd of json
-   */
-  private def extractPidResolvedTableFromKryo(spark: SparkSession, entityPath: String, workingPath: String) = {
-    import spark.implicits._
-    implicit val oafEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
-    val entities: Dataset[Result] = spark.read.load(s"$entityPath/*").as[Result]
-    entities.flatMap(e => e.getPid.asScala
-      .map(p =>
-        convertPidToDNETIdentifier(p.getValue, p.getQualifier.getClassid))
-      .filter(s => s != null)
-      .map(s => (s, e.getId))
-    ).groupByKey(_._1)
-      .reduceGroups((x, y) => if (x._2.startsWith("50|doi") || x._2.startsWith("50|pmid")) x else y)
-      .map(s => s._2)
-      .write
-      .mode(SaveMode.Overwrite)
-      .save(s"$workingPath/relationResolvedPid")
-  }
-
-  def convertPidToDNETIdentifier(pid:String, pidType: String):String = {
-    if (pid==null || pid.isEmpty || pidType== null || pidType.isEmpty)
-      null
-    else
-      s"unresolved::${pid.toLowerCase}::${pidType.toLowerCase}"
-  }
-
-}

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/copy_hdfs_oaf_parameters.json

@@ -0,0 +1,38 @@
+[
+	{
+		"paramName": "p",
+		"paramLongName": "hdfsPath",
+		"paramDescription": "the path where storing the sequential file",
+		"paramRequired": true
+	},
+	{
+		"paramName": "u",
+		"paramLongName": "mdstoreManagerUrl",
+		"paramDescription": "the MdstoreManager url",
+		"paramRequired": true
+	},
+	{
+		"paramName": "f",
+		"paramLongName": "mdFormat",
+		"paramDescription": "metadata format",
+		"paramRequired": true
+	},
+	{
+		"paramName": "l",
+		"paramLongName": "mdLayout",
+		"paramDescription": "metadata layout",
+		"paramRequired": true
+	},
+	{
+		"paramName": "m",
+		"paramLongName": "master",
+		"paramDescription": "should be yarn or local",
+		"paramRequired": true
+	},
+	{
+		"paramName": "i",
+		"paramLongName": "mdInterpretation",
+		"paramDescription": "metadata interpretation",
+		"paramRequired": true
+	}
+]

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/complete/schema/result_schema.json

@@ -1,6 +1,23 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "definitions": {
+    "AccessRight":{
+      "type":"object",
+      "properties":{
+        "code": {
+          "type": "string",
+          "description": "COAR access mode code: http://vocabularies.coar-repositories.org/documentation/access_rights/"
+        },
+        "label": {
+          "type": "string",
+          "description": "Label for the access mode"
+        },
+        "scheme": {
+          "type": "string",
+          "description": "Scheme of reference for access right code. Always set to COAR access rights vocabulary: http://vocabularies.coar-repositories.org/documentation/access_rights/"
+        }
+      }
+    },
     "ControlledField": {
       "type": "object",
       "properties": {
@@ -266,6 +283,57 @@
         ]
       }
     },
+    "instance":{
+      "type":"array",
+      "items":{
+        "type":"object",
+        "properties":{
+          "accessright":{
+            "allOf":[
+              {
+                "$ref":"#/definitions/AccessRight"
+              },
+              {
+                "description":"The accessright of this materialization of the result"
+              }
+            ]
+          },
+          "articleprocessingcharge":{
+            "type":"object",
+            "properties":{
+              "amount":{
+                "type":"string"
+              },
+              "currency":{
+                "type":"string"
+              }
+            }
+          },
+          "license":{
+            "type":"string"
+          },
+          "publicationdate":{
+            "type":"string"
+          },
+          "refereed":{
+            "type":"string"
+          },
+          "type":{
+            "type":"string",
+            "description":"The specific sub-type of this materialization of the result (see https://api.openaire.eu/vocabularies/dnet:result_typologies following the links)"
+          },
+          "url":{
+            "description":"Description of url",
+            "type":"array",
+            "items":{
+              "type":"string",
+              "description":"urls where it is possible to access the materialization of the result"
+            }
+          }
+        },
+        "description":"One of the materialization for this result"
+      }
+    },
     "programmingLanguage": {
       "type": "string",
       "description": "Only for results with type 'software': the programming language"
@@ -302,7 +370,7 @@
           "subject": {
             "allOf": [
               {"$ref": "#/definitions/ControlledField"},
-              {"description": "OpenAIRE subject classification scheme (https://api.openaire.eu/vocabularies/dnet:subject_classification_typologies) and value. When the scheme is 'keyword', it means that the subject is free-text (i.e. not a term from a controlled vocabulary)."},
+              {"description": "OpenAIRE subject classification scheme (https://api.openaire.eu/vocabularies/dnet:subject_classification_typologies) and value. When the scheme is 'keyword', it means that the subject is free-text (i.e. not a term from a controlled vocabulary)."}
             ]
           }
         }

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/funder_result_parameters.json

@@ -18,8 +18,8 @@
 		"paramRequired": false
 	},
 {
-	"paramName": "rp",
+	"paramName": "gp",
-	"paramLongName": "relationPath",
+	"paramLongName": "graphPath",
 	"paramDescription": "the relationPath",
 	"paramRequired": true
 }

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_cm_parameters.json

@@ -18,6 +18,18 @@
 		"paramLongName": "outputPath",
 		"paramDescription": "the path used to store temporary output files",
 		"paramRequired": true
+	},
+	{
+		"paramName": "sd",
+		"paramLongName": "singleDeposition",
+		"paramDescription": "true if the dump should be created for a single community",
+		"paramRequired": true
+	},
+	{
+		"paramName": "ci",
+		"paramLongName": "communityId",
+		"paramDescription": "the id of the community for which to create the dump",
+		"paramRequired": true
 	}
 ]
 

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters.json

@@ -35,7 +35,12 @@
 		"paramLongName":"dumpType",
 		"paramDescription": "the type of the dump (complete for the whole graph, community for the products related to communities, funder for the results with at least a link to project",
 		"paramRequired": false
-	}
+	},	{
+	"paramName":"cid",
+	"paramLongName":"communityId",
+	"paramDescription": "the id of the community to be dumped",
+	"paramRequired": false
+}
 ]
 
 

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_parameters_link_prj.json

@@ -24,8 +24,8 @@
 		"paramRequired": true
 	},
 	{
-		"paramName":"rp",
+		"paramName":"gp",
-		"paramLongName":"relationPath",
+		"paramLongName":"graphPath",
 		"paramDescription": "the path to the relations",
 		"paramRequired": true
 	}

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/split_parameters.json

@@ -25,7 +25,12 @@
 		"paramLongName": "isSparkSessionManaged",
 		"paramDescription": "true if the spark session is managed, false otherwise",
 		"paramRequired": false
-	}
+	},	{
+	"paramName":"cid",
+	"paramLongName":"communityId",
+	"paramDescription": "the id of the community to be dumped",
+	"paramRequired": false
+}
 ]
 
 

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json

@@ -12,12 +12,7 @@
     "paramDescription": "The id of the concept record for a new version",
     "paramRequired": false
   },
-  {
+
-    "paramName":"cmp",
-    "paramLongName":"communityMapPath",
-    "paramDescription": "the path to the serialization of the community map",
-    "paramRequired": false
-  },
   {
     "paramName":"di",
     "paramLongName":"depositionId",

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/main/oozie_app/config-default.xml

@@ -0,0 +1,30 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>hiveMetastoreUris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hiveJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
+    </property>
+    <property>
+        <name>hiveDbName</name>
+        <value>openaire</value>
+    </property>
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/main/oozie_app/import.txt

@@ -0,0 +1,4 @@
+## This is a classpath-based import file (this header is required)
+dump_complete classpath eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/complete/oozie_app
+dump_funder classpath eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app
+dump_community classpath eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/community/oozie_app

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/main/oozie_app/workflow.xml

@@ -0,0 +1,306 @@
+<workflow-app name="dump_graph" xmlns="uri:oozie:workflow:0.5">
+
+    <parameters>
+        <property>
+            <name>singleDeposition</name>
+            <description>Indicates if it is a single community deposition</description>
+        </property>
+        <property>
+            <name>communityId</name>
+            <description>the id of the community to be dumped if a dump for a single community should be done</description>
+        </property>
+        <property>
+            <name>dumpType</name>
+            <description>the type of the dump one of {complete, community, funder}</description>
+        </property>
+        <property>
+            <name>onlyUpload</name>
+            <description>true if the dump is already done and should only be upload in zenodo</description>
+        </property>
+        <property>
+            <name>upload</name>
+            <description>true if the dump should be upload in zenodo</description>
+        </property>
+        <property>
+            <name>sourcePath</name>
+            <description>the source path</description>
+        </property>
+        <property>
+            <name>isLookUpUrl</name>
+            <description>the isLookup service endpoint</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>the output path</description>
+        </property>
+        <property>
+            <name>resultAggregation</name>
+            <description>true if all the result type have to be dumped under result. false otherwise</description>
+        </property>
+        <property>
+            <name>accessToken</name>
+            <description>the access token used for the deposition in Zenodo</description>
+        </property>
+        <property>
+            <name>connectionUrl</name>
+            <description>the connection url for Zenodo</description>
+        </property>
+        <property>
+            <name>metadata</name>
+            <description> the metadata associated to the deposition</description>
+        </property>
+        <property>
+            <name>depositionType</name>
+            <description>the type of deposition we want to perform. "new" for brand new deposition, "version" for a new version of a published deposition (in this case the concept record id must be provided), "upload" to upload content to an open deposition for which we already have the deposition id (in this case the deposition id should be provided)</description>
+        </property>
+        <property>
+            <name>conceptRecordId</name>
+            <description>for new version, the id of the record for the old deposition</description>
+        </property>
+        <property>
+            <name>depositionId</name>
+            <description>the depositionId of a deposition open that has to be added content</description>
+        </property>
+        <property>
+            <name>organizationCommunityMap</name>
+            <description>the organization community map</description>
+        </property>
+
+        <property>
+            <name>hiveDbName</name>
+            <description>the target hive database name</description>
+        </property>
+        <property>
+            <name>hiveJdbcUrl</name>
+            <description>hive server jdbc url</description>
+        </property>
+        <property>
+            <name>hiveMetastoreUris</name>
+            <description>hive server metastore URIs</description>
+        </property>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>oozieActionShareLibForSpark2</name>
+            <description>oozie action sharelib for spark 2.*</description>
+        </property>
+        <property>
+            <name>spark2ExtraListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+            <description>spark 2.* extra listeners classname</description>
+        </property>
+        <property>
+            <name>spark2SqlQueryExecutionListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+            <description>spark 2.* sql query execution listeners classname</description>
+        </property>
+        <property>
+            <name>spark2YarnHistoryServerAddress</name>
+            <description>spark 2.* yarn history server address</description>
+        </property>
+        <property>
+            <name>spark2EventLogDir</name>
+            <description>spark 2.* event log dir location</description>
+        </property>
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+            <property>
+                <name>oozie.action.sharelib.for.spark</name>
+                <value>${oozieActionShareLibForSpark2}</value>
+            </property>
+
+        </configuration>
+    </global>
+
+    <start to="only_upload"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <decision name="only_upload">
+        <switch>
+            <case to="send_zenodo">${wf:conf('onlyUpload') eq true}</case>
+            <default to="reset_outputpath"/>
+        </switch>
+    </decision>
+
+    <action name="reset_outputpath">
+        <fs>
+            <delete path="${outputPath}"/>
+            <mkdir path="${outputPath}"/>
+        </fs>
+        <ok to="save_community_map"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="save_community_map">
+        <java>
+            <main-class>eu.dnetlib.dhp.oa.graph.dump.SaveCommunityMap</main-class>
+            <arg>--outputPath</arg><arg>${workingDir}/communityMap</arg>
+            <arg>--nameNode</arg><arg>${nameNode}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+            <arg>--singleDeposition</arg><arg>${singleDeposition}</arg>
+            <arg>--communityId</arg><arg>${communityId}</arg>
+        </java>
+        <ok to="choose_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <decision name="choose_dump">
+        <switch>
+            <case to="dump_funder">${wf:conf('dumpType') eq "funder"}</case>
+            <case to="dump_community">${wf:conf('dumpType') eq "community"}</case>
+            <default to="dump_complete"/>
+        </switch>
+    </decision>
+
+    <!-- Sub-workflow which runs the dump for the complete graph -->
+    <action name="dump_complete">
+        <sub-workflow>
+            <app-path>${wf:appPath()}/dump_complete
+            </app-path>
+            <propagate-configuration/>
+            <configuration>
+                <property>
+                    <name>communityMapPath</name>
+                    <value>${workingDir}/communityMap</value>
+                </property>
+                <property>
+                    <name>outputPath</name>
+                    <value>${workingDir}/tar</value>
+                </property>
+                <property>
+                    <name>sourcePath</name>
+                    <value>${sourcePath}</value>
+                </property>
+                <property>
+                    <name>organizationCommunityMap</name>
+                    <value>${organizationCommunityMap}</value>
+                </property>
+                <property>
+                    <name>isLookUpUrl</name>
+                    <value>${isLookUpUrl}</value>
+                </property>
+                <property>
+                    <name>resultAggregation</name>
+                    <value>${resultAggregation}</value>
+                </property>
+            </configuration>
+        </sub-workflow>
+        <ok to="make_archive" />
+        <error to="Kill" />
+    </action>
+
+    <!-- Sub-workflow which runs the dump for the complete graph -->
+    <action name="dump_community">
+        <sub-workflow>
+            <app-path>${wf:appPath()}/dump_community
+            </app-path>
+            <propagate-configuration/>
+            <configuration>
+                <property>
+                    <name>sourcePath</name>
+                    <value>${sourcePath}</value>
+                </property>
+                <property>
+                    <name>communityMapPath</name>
+                    <value>${workingDir}/communityMap</value>
+                </property>
+                <property>
+                    <name>outputPath</name>
+                    <value>${workingDir}/tar</value>
+                </property>
+            </configuration>
+        </sub-workflow>
+        <ok to="make_archive" />
+        <error to="Kill" />
+    </action>
+
+    <action name="dump_funder">
+        <sub-workflow>
+            <app-path>${wf:appPath()}/dump_funder
+            </app-path>
+            <propagate-configuration/>
+            <configuration>
+                <property>
+                    <name>communityMapPath</name>
+                    <value>${workingDir}/communityMap</value>
+                </property>
+                <property>
+                    <name>outputPath</name>
+                    <value>${workingDir}/tar</value>
+                </property>
+                <property>
+                    <name>sourcePath</name>
+                    <value>${sourcePath}</value>
+                </property>
+                <property>
+                    <name>dumpType</name>
+                    <value>${dumpType}</value>
+                </property>
+            </configuration>
+        </sub-workflow>
+        <ok to="make_archive" />
+        <error to="Kill" />
+    </action>
+
+    <action name="make_archive">
+        <java>
+            <main-class>eu.dnetlib.dhp.oa.graph.dump.MakeTar</main-class>
+            <arg>--hdfsPath</arg><arg>${outputPath}</arg>
+            <arg>--nameNode</arg><arg>${nameNode}</arg>
+            <arg>--sourcePath</arg><arg>${workingDir}/tar</arg>
+        </java>
+        <ok to="should_upload"/>
+        <error to="Kill"/>
+    </action>
+
+    <decision name="should_upload">
+        <switch>
+            <case to="send_zenodo">${wf:conf('upload') eq true}</case>
+            <default to="End"/>
+        </switch>
+    </decision>
+
+    <action name="send_zenodo">
+        <java>
+            <main-class>eu.dnetlib.dhp.oa.graph.dump.SendToZenodoHDFS</main-class>
+            <arg>--hdfsPath</arg><arg>${outputPath}</arg>
+            <arg>--nameNode</arg><arg>${nameNode}</arg>
+            <arg>--accessToken</arg><arg>${accessToken}</arg>
+            <arg>--connectionUrl</arg><arg>${connectionUrl}</arg>
+            <arg>--metadata</arg><arg>${metadata}</arg>
+            <arg>--conceptRecordId</arg><arg>${conceptRecordId}</arg>
+            <arg>--depositionType</arg><arg>${depositionType}</arg>
+            <arg>--depositionId</arg><arg>${depositionId}</arg>
+        </java>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <end name="End"/>
+
+</workflow-app>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/commoncommunityfunder/oozie_app/config-default.xml

@@ -12,10 +12,17 @@
         <value>true</value>
     </property>
     <property>
-        <name>oozie.action.sharelib.for.spark</name>
+        <name>hiveMetastoreUris</name>
-        <value>spark2</value>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hiveJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
+    </property>
+    <property>
+        <name>hiveDbName</name>
+        <value>openaire</value>
     </property>
-
     <property>
         <name>oozie.launcher.mapreduce.user.classpath.first</name>
         <value>true</value>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/commoncommunityfunder/oozie_app/workflow.xml

@@ -0,0 +1,347 @@
+<workflow-app name="sub_dump_community_funder_results" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+
+        <property>
+            <name>sourcePath</name>
+            <description>the source path</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>the output path</description>
+        </property>
+        <property>
+            <name>communityMapPath</name>
+            <description>the path to the community map</description>
+        </property>
+        <property>
+            <name>selectedResults</name>
+            <description>the path the the possible subset ot results to be dumped</description>
+        </property>
+        <property>
+            <name>hiveDbName</name>
+            <description>the target hive database name</description>
+        </property>
+        <property>
+            <name>hiveJdbcUrl</name>
+            <description>hive server jdbc url</description>
+        </property>
+        <property>
+            <name>hiveMetastoreUris</name>
+            <description>hive server metastore URIs</description>
+        </property>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>oozieActionShareLibForSpark2</name>
+            <description>oozie action sharelib for spark 2.*</description>
+        </property>
+        <property>
+            <name>spark2ExtraListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+            <description>spark 2.* extra listeners classname</description>
+        </property>
+        <property>
+            <name>spark2SqlQueryExecutionListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+            <description>spark 2.* sql query execution listeners classname</description>
+        </property>
+        <property>
+            <name>spark2YarnHistoryServerAddress</name>
+            <description>spark 2.* yarn history server address</description>
+        </property>
+        <property>
+            <name>spark2EventLogDir</name>
+            <description>spark 2.* event log dir location</description>
+        </property>
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+            <property>
+                <name>oozie.action.sharelib.for.spark</name>
+                <value>${oozieActionShareLibForSpark2}</value>
+            </property>
+
+        </configuration>
+    </global>
+
+    <start to="fork_dump"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+
+
+
+    <fork name="fork_dump">
+        <path start="dump_publication"/>
+        <path start="dump_dataset"/>
+        <path start="dump_orp"/>
+        <path start="dump_software"/>
+    </fork>
+
+    <action name="dump_publication">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table publication for community/funder related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkDumpCommunityProducts</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${selectedResults}/publication</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/dump/publication</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+            <arg>--dumpType</arg><arg>${dumpType}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_dataset">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table dataset for community/funder related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkDumpCommunityProducts</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${selectedResults}/dataset</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/dump/dataset</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_orp">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table ORP for community related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkDumpCommunityProducts</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${selectedResults}/otherresearchproduct</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/dump/otherresearchproduct</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_software">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table software for community related products</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkDumpCommunityProducts</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${selectedResults}/software</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/dump/software</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <join name="join_dump" to="prepareResultProject"/>
+
+    <action name="prepareResultProject">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Prepare association result subset of project info</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkPrepareResultProject</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/preparedInfo</arg>
+        </spark>
+        <ok to="fork_extendWithProject"/>
+        <error to="Kill"/>
+    </action>
+
+    <fork name="fork_extendWithProject">
+        <path start="extend_publication"/>
+        <path start="extend_dataset"/>
+        <path start="extend_orp"/>
+        <path start="extend_software"/>
+    </fork>
+
+    <action name="extend_publication">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Extend dumped publications with information about project</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkUpdateProjectInfo</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/dump/publication</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/ext/publication</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
+        </spark>
+        <ok to="join_extend"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="extend_dataset">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Extend dumped dataset with information about project</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkUpdateProjectInfo</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/dump/dataset</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/ext/dataset</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
+        </spark>
+        <ok to="join_extend"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="extend_orp">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Extend dumped ORP with information about project</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkUpdateProjectInfo</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/dump/otherresearchproduct</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/ext/orp</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
+        </spark>
+        <ok to="join_extend"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="extend_software">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Extend dumped software with information about project</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkUpdateProjectInfo</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/dump/software</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/ext/software</arg>
+            <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
+        </spark>
+        <ok to="join_extend"/>
+        <error to="Kill"/>
+    </action>
+    <join name="join_extend" to="End"/>
+
+    <end name="End"/>
+
+</workflow-app>
+
+

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/community/oozie_app/config-default.xml

@@ -0,0 +1,30 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>hiveMetastoreUris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hiveJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
+    </property>
+    <property>
+        <name>hiveDbName</name>
+        <value>openaire</value>
+    </property>
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/community/oozie_app/import.txt

@@ -0,0 +1,2 @@
+## This is a classpath-based import file (this header is required)
+dump_common classpath eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/commoncommunityfunder/oozie_app

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/community/oozie_app/workflow.xml

@@ -0,0 +1,145 @@
+<workflow-app name="sub_dump_community_products" xmlns="uri:oozie:workflow:0.5">
+
+    <parameters>
+        <property>
+            <name>sourcePath</name>
+            <description>the source path</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>the output path</description>
+        </property>
+        <property>
+            <name>hiveDbName</name>
+            <description>the target hive database name</description>
+        </property>
+        <property>
+            <name>hiveJdbcUrl</name>
+            <description>hive server jdbc url</description>
+        </property>
+        <property>
+            <name>hiveMetastoreUris</name>
+            <description>hive server metastore URIs</description>
+        </property>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>oozieActionShareLibForSpark2</name>
+            <description>oozie action sharelib for spark 2.*</description>
+        </property>
+        <property>
+            <name>spark2ExtraListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+            <description>spark 2.* extra listeners classname</description>
+        </property>
+        <property>
+            <name>spark2SqlQueryExecutionListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+            <description>spark 2.* sql query execution listeners classname</description>
+        </property>
+        <property>
+            <name>spark2YarnHistoryServerAddress</name>
+            <description>spark 2.* yarn history server address</description>
+        </property>
+        <property>
+            <name>spark2EventLogDir</name>
+            <description>spark 2.* event log dir location</description>
+        </property>
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+            <property>
+                <name>oozie.action.sharelib.for.spark</name>
+                <value>${oozieActionShareLibForSpark2}</value>
+            </property>
+
+        </configuration>
+    </global>
+
+    <start to="common_action_community_funder"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <action name="common_action_community_funder">
+        <sub-workflow>
+            <app-path>${wf:appPath()}/dump_common
+            </app-path>
+            <propagate-configuration/>
+            <configuration>
+                <property>
+                    <name>sourcePath</name>
+                    <value>${sourcePath}</value>
+                </property>
+                <property>
+                    <name>selectedResults</name>
+                    <value>${sourcePath}</value>
+                </property>
+                <property>
+                    <name>communityMapPath</name>
+                    <value>${workingDir}/communityMap</value>
+                </property>
+                <property>
+                    <name>outputPath</name>
+                    <value>${workingDir}</value>
+                </property>
+            </configuration>
+        </sub-workflow>
+        <ok to="splitForCommunities" />
+        <error to="Kill" />
+    </action>
+
+
+
+    <action name="splitForCommunities">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Split dumped result for community</name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.community.SparkSplitForCommunity</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/ext</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+
+
+    <end name="End"/>
+
+</workflow-app>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/complete/oozie_app/config-default.xml

@@ -0,0 +1,30 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>hiveMetastoreUris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hiveJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
+    </property>
+    <property>
+        <name>hiveDbName</name>
+        <value>openaire</value>
+    </property>
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/complete/oozie_app/workflow.xml

@@ -0,0 +1,537 @@
+<workflow-app name="sub-dump_complete" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>sourcePath</name>
+            <description>the source path</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>the output path</description>
+        </property>
+        <property>
+            <name>resultAggregation</name>
+            <description>true if all the result type have to be dumped under result. false otherwise</description>
+        </property>
+        <property>
+            <name>organizationCommunityMap</name>
+            <description>the organization community map</description>
+        </property>
+
+        <property>
+            <name>hiveDbName</name>
+            <description>the target hive database name</description>
+        </property>
+        <property>
+            <name>hiveJdbcUrl</name>
+            <description>hive server jdbc url</description>
+        </property>
+        <property>
+            <name>hiveMetastoreUris</name>
+            <description>hive server metastore URIs</description>
+        </property>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>oozieActionShareLibForSpark2</name>
+            <description>oozie action sharelib for spark 2.*</description>
+        </property>
+        <property>
+            <name>spark2ExtraListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+            <description>spark 2.* extra listeners classname</description>
+        </property>
+        <property>
+            <name>spark2SqlQueryExecutionListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+            <description>spark 2.* sql query execution listeners classname</description>
+        </property>
+        <property>
+            <name>spark2YarnHistoryServerAddress</name>
+            <description>spark 2.* yarn history server address</description>
+        </property>
+        <property>
+            <name>spark2EventLogDir</name>
+            <description>spark 2.* event log dir location</description>
+        </property>
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+            <property>
+                <name>oozie.action.sharelib.for.spark</name>
+                <value>${oozieActionShareLibForSpark2}</value>
+            </property>
+
+        </configuration>
+    </global>
+
+    <start to="fork_dump" />
+
+
+
+    <fork name="fork_dump">
+        <path start="dump_publication"/>
+        <path start="dump_dataset"/>
+        <path start="dump_orp"/>
+        <path start="dump_software"/>
+        <path start="dump_organization"/>
+        <path start="dump_project"/>
+        <path start="dump_datasource"/>
+        <path start="select_relation"/>
+    </fork>
+
+    <action name="dump_publication">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table publication </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/publication</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_dataset">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table dataset </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/dataset</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_orp">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table ORP </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/otherresearchproduct</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_software">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table software </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/software</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_organization">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table organization </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/organization</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Organization</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/organization</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_project">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table project </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/project</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/project</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_datasource">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table datasource </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpEntitiesJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/datasource</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Datasource</arg>
+            <arg>--outputPath</arg><arg>${outputPath}/datasource</arg>
+            <arg>--communityMapPath</arg><arg>${workingDir}/communityMap</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="select_relation">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Select valid table relation </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkSelectValidRelationsJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/validrelation</arg>
+        </spark>
+        <ok to="dump_relation"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="dump_relation">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table relation </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkDumpRelationJob</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/validrelation</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation/relation</arg>
+        </spark>
+        <ok to="join_dump"/>
+        <error to="Kill"/>
+    </action>
+
+    <join name="join_dump" to="fork_context"/>
+
+    <fork name="fork_context">
+        <path start="create_entities_fromcontext"/>
+        <path start="create_relation_fromcontext"/>
+        <path start="create_relation_fromorgs"/>
+    </fork>
+
+    <action name="create_entities_fromcontext">
+        <java>
+            <main-class>eu.dnetlib.dhp.oa.graph.dump.complete.CreateContextEntities</main-class>
+            <arg>--hdfsPath</arg><arg>${outputPath}/communities_infrastructures/communities_infrastructure.json.gz</arg>
+            <arg>--nameNode</arg><arg>${nameNode}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+        </java>
+        <ok to="join_context"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="create_relation_fromcontext">
+        <java>
+            <main-class>eu.dnetlib.dhp.oa.graph.dump.complete.CreateContextRelation</main-class>
+            <arg>--hdfsPath</arg><arg>${workingDir}/relation/context</arg>
+            <arg>--nameNode</arg><arg>${nameNode}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+        </java>
+        <ok to="join_context"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="create_relation_fromorgs">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table relation </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkOrganizationRelation</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/relation</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation/contextOrg</arg>
+            <arg>--organizationCommunityMap</arg><arg>${organizationCommunityMap}</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_context"/>
+        <error to="Kill"/>
+    </action>
+
+    <join name="join_context" to="fork_extract_relations"/>
+
+    <fork name="fork_extract_relations">
+        <path start="rels_from_pubs"/>
+        <path start="rels_from_dats"/>
+        <path start="rels_from_orp"/>
+        <path start="rels_from_sw"/>
+    </fork>
+
+    <action name="rels_from_pubs">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Extract Relations from publication </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkExtractRelationFromEntities</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation/publication</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_extract_relations"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="rels_from_dats">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table dataset </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkExtractRelationFromEntities</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation/dataset</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_extract_relations"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="rels_from_orp">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table ORP </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkExtractRelationFromEntities</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation/orp</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_extract_relations"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="rels_from_sw">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump table software </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkExtractRelationFromEntities</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/relation/software</arg>
+            <arg>--communityMapPath</arg><arg>${communityMapPath}</arg>
+        </spark>
+        <ok to="join_extract_relations"/>
+        <error to="Kill"/>
+    </action>
+
+    <join name="join_extract_relations" to="collect_and_save"/>
+
+    <action name="collect_and_save">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Collect Results and Relations and put them in the right path </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.complete.SparkCollectAndSave</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--resultAggregation</arg><arg>${resultAggregation}</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+
+
+    <kill name="Kill">
+        <message>Sub-workflow dump complete failed with error message ${wf:errorMessage()}
+        </message>
+    </kill>
+
+    <end name="End" />
+</workflow-app>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/config-default.xml

@@ -0,0 +1,30 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>hiveMetastoreUris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hiveJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
+    </property>
+    <property>
+        <name>hiveDbName</name>
+        <value>openaire</value>
+    </property>
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/import.txt

@@ -0,0 +1,2 @@
+## This is a classpath-based import file (this header is required)
+dump_common classpath eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/commoncommunityfunder/oozie_app

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/wf/subworkflows/funder/oozie_app/workflow.xml

@@ -0,0 +1,256 @@
+<workflow-app name="sub_dump_funder_results" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+
+        <property>
+            <name>sourcePath</name>
+            <description>the source path</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>the output path</description>
+        </property>
+        <property>
+            <name>hiveDbName</name>
+            <description>the target hive database name</description>
+        </property>
+        <property>
+            <name>hiveJdbcUrl</name>
+            <description>hive server jdbc url</description>
+        </property>
+        <property>
+            <name>hiveMetastoreUris</name>
+            <description>hive server metastore URIs</description>
+        </property>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>oozieActionShareLibForSpark2</name>
+            <description>oozie action sharelib for spark 2.*</description>
+        </property>
+        <property>
+            <name>spark2ExtraListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+            <description>spark 2.* extra listeners classname</description>
+        </property>
+        <property>
+            <name>spark2SqlQueryExecutionListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+            <description>spark 2.* sql query execution listeners classname</description>
+        </property>
+        <property>
+            <name>spark2YarnHistoryServerAddress</name>
+            <description>spark 2.* yarn history server address</description>
+        </property>
+        <property>
+            <name>spark2EventLogDir</name>
+            <description>spark 2.* event log dir location</description>
+        </property>
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+            <property>
+                <name>oozie.action.sharelib.for.spark</name>
+                <value>${oozieActionShareLibForSpark2}</value>
+            </property>
+
+        </configuration>
+    </global>
+
+    <start to="fork_result_linked_to_projects"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+
+    <fork name="fork_result_linked_to_projects">
+        <path start="select_publication_linked_to_projects"/>
+        <path start="select_dataset_linked_to_projects"/>
+        <path start="select_orp_linked_to_project"/>
+        <path start="select_software_linked_to_projects"/>
+    </fork>
+
+    <action name="select_publication_linked_to_projects">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump funder results </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.funderresults.SparkResultLinkedToProject</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/publication</arg>
+            <arg>--graphPath</arg><arg>${sourcePath}</arg>
+        </spark>
+        <ok to="join_link"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="select_dataset_linked_to_projects">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump funder results </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.funderresults.SparkResultLinkedToProject</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/dataset</arg>
+            <arg>--graphPath</arg><arg>${sourcePath}</arg>
+        </spark>
+        <ok to="join_link"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="select_orp_linked_to_project">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump funder results </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.funderresults.SparkResultLinkedToProject</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/otherresearchproduct</arg>
+            <arg>--graphPath</arg><arg>${sourcePath}</arg>
+        </spark>
+        <ok to="join_link"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="select_software_linked_to_projects">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump funder results </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.funderresults.SparkResultLinkedToProject</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
+            <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/result/software</arg>
+            <arg>--graphPath</arg><arg>${sourcePath}</arg>
+        </spark>
+        <ok to="join_link"/>
+        <error to="Kill"/>
+    </action>
+
+    <join name="join_link" to="common_action_community_funder"/>
+
+    <action name="common_action_community_funder">
+        <sub-workflow>
+            <app-path>${wf:appPath()}/dump_common
+            </app-path>
+            <propagate-configuration/>
+            <configuration>
+                <property>
+                    <name>sourcePath</name>
+                    <value>${sourcePath}</value>
+                </property>
+                <property>
+                    <name>selectedResults</name>
+                    <value>${workingDir}/result</value>
+                </property>
+                <property>
+                    <name>communityMapPath</name>
+                    <value>${workingDir}/communityMap</value>
+                </property>
+                <property>
+                    <name>outputPath</name>
+                    <value>${workingDir}</value>
+                </property>
+            </configuration>
+        </sub-workflow>
+        <ok to="dump_funder_results" />
+        <error to="Kill" />
+    </action>
+
+    <action name="dump_funder_results">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Dump funder results </name>
+            <class>eu.dnetlib.dhp.oa.graph.dump.funderresults.SparkDumpFunderResults</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/ext</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--graphPath</arg><arg>${sourcePath}</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+
+    <end name="End"/>
+
+</workflow-app>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/raw_all/oozie_app/workflow.xml

@@ -258,7 +258,7 @@
         <switch>
             <case to="ImportDB">${wf:conf('reuseDB') eq false}</case>
             <case to="reuse_odf">${wf:conf('reuseDB') eq true}</case>
-            <default to="ImportDB_claims"/>
+            <default to="ImportDB"/>
         </switch>
     </decision>
 
@@ -544,6 +544,33 @@
             <arg>--sourcePath</arg><arg>${workingDir}/entities</arg>
             <arg>--graphRawPath</arg><arg>${workingDir}/graph_raw</arg>
         </spark>
+        <ok to="Import_oaf_store_graph"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="Import_oaf_store_graph">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>ImportOAF_hdfs_graph</name>
+            <class>eu.dnetlib.dhp.oa.graph.raw.CopyHdfsOafSparkApplication</class>
+            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory ${sparkExecutorMemory}
+                --executor-cores ${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--hdfsPath</arg><arg>${workingDir}/graph_raw</arg>
+            <arg>--mdstoreManagerUrl</arg><arg>${mdstoreManagerUrl}</arg>
+            <arg>--mdFormat</arg><arg>OAF</arg>
+            <arg>--mdLayout</arg><arg>store</arg>
+            <arg>--master</arg><arg>yarn</arg>
+            <arg>--mdInterpretation</arg><arg>graph</arg>
+        </spark>
         <ok to="wait_graphs"/>
         <error to="Kill"/>
     </action>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/workflow.xml

@@ -1,59 +1,37 @@
 <workflow-app name="Resolve Relation" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
-            <name>entityPath</name>
+            <name>graphBasePath</name>
-            <description>the path of deduplicate Entities</description>
+            <description>the path of the graph</description>
         </property>
-        <property>
-            <name>relationPath</name>
-            <description>the path of relation unresolved</description>
-        </property>
-        <property>
-            <name>targetPath</name>
-            <description>the path of relation unresolved</description>
-        </property>
-
     </parameters>
 
-    <start to="DropRelFolder"/>
+    <start to="ResolveRelations"/>
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
-
-    <action name="DropRelFolder">
-        <fs>
-            <delete path='${targetPath}/relation'/>
-            <delete path='${targetPath}/relation_resolved'/>
-            <delete path='${targetPath}/resolvedSource'/>
-            <delete path='${targetPath}/resolvedPid'/>
-
-        </fs>
-        <ok to="ResolveRelations"/>
-        <error to="Kill"/>
-    </action>
     <action name="ResolveRelations">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
             <name>Resolve Relations in raw graph</name>
-            <class>eu.dnetlib.dhp.sx.graph.SparkResolveRelation</class>
+            <class>eu.dnetlib.dhp.oa.graph.resolution.SparkResolveRelation</class>
             <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.shuffle.partitions=3000
+                --conf spark.sql.shuffle.partitions=15000
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
             <arg>--master</arg><arg>yarn</arg>
-            <arg>--relationPath</arg><arg>${relationPath}</arg>
+            <arg>--graphBasePath</arg><arg>${graphBasePath}</arg>
-            <arg>--workingPath</arg><arg>${targetPath}</arg>
+            <arg>--workingPath</arg><arg>${workingDir}</arg>
-            <arg>--entityPath</arg><arg>${entityPath}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/resolve_relations_params.json

@@ -1,6 +1,5 @@
 [
   {"paramName":"mt",  "paramLongName":"master",       "paramDescription": "should be local or yarn",    "paramRequired": true},
-  {"paramName":"r",   "paramLongName":"relationPath", "paramDescription": "the source Path",            "paramRequired": true},
   {"paramName":"w",   "paramLongName":"workingPath",  "paramDescription": "the source Path",            "paramRequired": true},
-  {"paramName":"e",   "paramLongName":"entityPath",   "paramDescription": "the path of the raw graph",  "paramRequired": true}
+  {"paramName":"g",   "paramLongName":"graphBasePath",   "paramDescription": "the path of the raw graph",  "paramRequired": true}
 ]

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryParentChildRelsOpenOrgs.sql

@@ -0,0 +1,13 @@
+SELECT
+	id1                                                            AS source,
+	id2                                                            AS target,
+	reltype                                                        AS type,
+	false                                                          AS inferred,
+	false                                                          AS deletedbyinference,
+	0.95                                                           AS trust,
+	''                                                             AS inferenceprovenance,
+	'openaire____::openorgs'                                       AS collectedfromid,
+	'OpenOrgs Database'                                            AS collectedfromname,
+	'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction
+FROM relationships
+WHERE reltype = 'Child' OR reltype = 'Parent'

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/graph/bio/oozie_app/workflow.xml

@@ -1,177 +0,0 @@
-    <workflow-app name="Transform_BioEntity_Workflow" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>PDBPath</name>
-            <description>the PDB Database Working Path</description>
-        </property>
-
-        <property>
-            <name>UNIPROTDBPath</name>
-            <description>the UNIPROT Database Working Path</description>
-        </property>
-
-        <property>
-            <name>EBIDataset</name>
-            <description>the EBI Links Dataset  Path</description>
-        </property>
-
-        <property>
-            <name>ScholixResolvedDBPath</name>
-            <description>the Scholix Resolved Dataset  Path</description>
-        </property>
-
-        <property>
-            <name>CrossrefLinksPath</name>
-            <description>the CrossrefLinks Path</description>
-        </property>
-        <property>
-            <name>targetPath</name>
-            <description>the Target Working dir path</description>
-        </property>
-    </parameters>
-
-    <start to="ConvertPDB"/>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-    <action name="ConvertPDB">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Convert PDB to OAF Dataset</name>
-            <class>eu.dnetlib.dhp.sx.graph.bio.SparkTransformBioDatabaseToOAF</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.shuffle.partitions=2000
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--master</arg><arg>yarn</arg>
-            <arg>--dbPath</arg><arg>${PDBPath}</arg>
-            <arg>--database</arg><arg>PDB</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/pdb_OAF</arg>
-        </spark>
-        <ok to="ConvertUNIPROT"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="ConvertUNIPROT">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Convert UNIPROT to OAF Dataset</name>
-            <class>eu.dnetlib.dhp.sx.graph.bio.SparkTransformBioDatabaseToOAF</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.shuffle.partitions=2000
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--master</arg><arg>yarn</arg>
-            <arg>--dbPath</arg><arg>${UNIPROTDBPath}</arg>
-            <arg>--database</arg><arg>UNIPROT</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/uniprot_OAF</arg>
-        </spark>
-        <ok to="ConvertEBILinks"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="ConvertEBILinks">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Convert EBI Links to OAF Dataset</name>
-            <class>eu.dnetlib.dhp.sx.graph.ebi.SparkEBILinksToOaf</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.shuffle.partitions=2000
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--master</arg><arg>yarn</arg>
-            <arg>--sourcePath</arg><arg>${EBIDataset}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/ebi_OAF</arg>
-        </spark>
-        <ok to="ConvertScholixResolved"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="ConvertScholixResolved">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Convert Scholix to OAF Dataset</name>
-            <class>eu.dnetlib.dhp.sx.graph.bio.SparkTransformBioDatabaseToOAF</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.shuffle.partitions=2000
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--master</arg><arg>yarn</arg>
-            <arg>--dbPath</arg><arg>${ScholixResolvedDBPath}</arg>
-            <arg>--database</arg><arg>SCHOLIX</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/scholix_resolved_OAF</arg>
-        </spark>
-        <ok to="ConvertCrossrefLinks"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="ConvertCrossrefLinks">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Convert Crossref Links to OAF Dataset</name>
-            <class>eu.dnetlib.dhp.sx.graph.bio.SparkTransformBioDatabaseToOAF</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.shuffle.partitions=2000
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--master</arg><arg>yarn</arg>
-            <arg>--dbPath</arg><arg>${CrossrefLinksPath}</arg>
-            <arg>--database</arg><arg>CROSSREF_LINKS</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/crossref_unresolved_relation_OAF</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-
-
-
-    <end name="End"/>
-
-</workflow-app>

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/DumpJobTest.java

@@ -11,17 +11,21 @@ import org.apache.commons.io.FileUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.ForeachFunction;
 import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SparkSession;
 import org.junit.jupiter.api.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.gson.Gson;
 
 import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.dump.oaf.Instance;
+import eu.dnetlib.dhp.schema.dump.oaf.OpenAccessRoute;
 import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
 import eu.dnetlib.dhp.schema.dump.oaf.graph.GraphResult;
 import eu.dnetlib.dhp.schema.oaf.Dataset;
@@ -137,6 +141,512 @@ public class DumpJobTest {
 		System.out.println(new Gson().toJson(map));
 	}
 
+	@Test
+	public void testPublicationDump() {
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/publication_extendedinstance")
+			.getPath();
+
+		final String communityMapPath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
+			.getPath();
+
+		DumpProducts dump = new DumpProducts();
+		dump
+			.run(
+				// false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
+				false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
+				GraphResult.class, Constants.DUMPTYPE.COMPLETE.getType());
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<GraphResult> tmp = sc
+			.textFile(workingDir.toString() + "/result")
+			.map(item -> OBJECT_MAPPER.readValue(item, GraphResult.class));
+
+		org.apache.spark.sql.Dataset<GraphResult> verificationDataset = spark
+			.createDataset(tmp.rdd(), Encoders.bean(GraphResult.class));
+
+		Assertions.assertEquals(1, verificationDataset.count());
+
+		GraphResult gr = verificationDataset.first();
+
+		Assertions.assertEquals(2, gr.getMeasures().size());
+		Assertions
+			.assertTrue(
+				gr
+					.getMeasures()
+					.stream()
+					.anyMatch(
+						m -> m.getKey().equals("influence")
+							&& m.getValue().equals("1.62759106106e-08")));
+		Assertions
+			.assertTrue(
+				gr
+					.getMeasures()
+					.stream()
+					.anyMatch(
+						m -> m.getKey().equals("popularity")
+							&& m.getValue().equals("0.22519296")));
+
+		Assertions.assertEquals(6, gr.getAuthor().size());
+		Assertions
+			.assertTrue(
+				gr
+					.getAuthor()
+					.stream()
+					.anyMatch(
+						a -> a.getFullname().equals("Nikolaidou,Charitini") &&
+							a.getName().equals("Charitini") && a.getSurname().equals("Nikolaidou")
+							&& a.getRank() == 1 && a.getPid() == null));
+
+		Assertions
+			.assertTrue(
+				gr
+					.getAuthor()
+					.stream()
+					.anyMatch(
+						a -> a.getFullname().equals("Votsi,Nefta") &&
+							a.getName().equals("Nefta") && a.getSurname().equals("Votsi")
+							&& a.getRank() == 2 && a.getPid().getId().getScheme().equals(ModelConstants.ORCID)
+							&& a.getPid().getId().getValue().equals("0000-0001-6651-1178")
+							&& a.getPid().getProvenance() != null));
+
+		Assertions
+			.assertTrue(
+				gr
+					.getAuthor()
+					.stream()
+					.anyMatch(
+						a -> a.getFullname().equals("Sgardelis,Steanos") &&
+							a.getName().equals("Steanos") && a.getSurname().equals("Sgardelis")
+							&& a.getRank() == 3 && a.getPid().getId().getScheme().equals(ModelConstants.ORCID_PENDING)
+							&& a.getPid().getId().getValue().equals("0000-0001-6651-1178")
+							&& a.getPid().getProvenance() != null));
+
+		Assertions
+			.assertTrue(
+				gr
+					.getAuthor()
+					.stream()
+					.anyMatch(
+						a -> a.getFullname().equals("Halley,John") &&
+							a.getName().equals("John") && a.getSurname().equals("Halley")
+							&& a.getRank() == 4 && a.getPid() == null));
+
+		Assertions
+			.assertTrue(
+				gr
+					.getAuthor()
+					.stream()
+					.anyMatch(
+						a -> a.getFullname().equals("Pantis,John") &&
+							a.getName().equals("John") && a.getSurname().equals("Pantis")
+							&& a.getRank() == 5 && a.getPid().getId().getScheme().equals(ModelConstants.ORCID)
+							&& a.getPid().getId().getValue().equals("0000-0001-6651-1178")
+							&& a.getPid().getProvenance() != null));
+
+		Assertions
+			.assertTrue(
+				gr
+					.getAuthor()
+					.stream()
+					.anyMatch(
+						a -> a.getFullname().equals("Tsiafouli,Maria") &&
+							a.getName().equals("Maria") && a.getSurname().equals("Tsiafouli")
+							&& a.getRank() == 6 && a.getPid().getId().getScheme().equals(ModelConstants.ORCID_PENDING)
+							&& a.getPid().getId().getValue().equals("0000-0001-6651-1178")
+							&& a.getPid().getProvenance() != null));
+
+		Assertions.assertEquals("publication", gr.getType());
+
+		Assertions.assertEquals("eng", gr.getLanguage().getCode());
+		Assertions.assertEquals("English", gr.getLanguage().getLabel());
+
+		Assertions.assertEquals(1, gr.getCountry().size());
+		Assertions.assertEquals("IT", gr.getCountry().get(0).getCode());
+		Assertions.assertEquals("Italy", gr.getCountry().get(0).getLabel());
+		Assertions.assertTrue(gr.getCountry().get(0).getProvenance() == null);
+
+		Assertions.assertEquals(12, gr.getSubjects().size());
+		Assertions
+			.assertTrue(
+				gr
+					.getSubjects()
+					.stream()
+					.anyMatch(
+						s -> s.getSubject().getValue().equals("Ecosystem Services hotspots")
+							&& s.getSubject().getScheme().equals("ACM") && s.getProvenance() != null &&
+							s.getProvenance().getProvenance().equals("sysimport:crosswalk:repository")));
+		Assertions
+			.assertTrue(
+				gr
+					.getSubjects()
+					.stream()
+					.anyMatch(
+						s -> s.getSubject().getValue().equals("Natura 2000")
+							&& s.getSubject().getScheme().equals("") && s.getProvenance() != null &&
+							s.getProvenance().getProvenance().equals("sysimport:crosswalk:repository")));
+
+		Assertions
+			.assertEquals(
+				"Ecosystem Service capacity is higher in areas of multiple designation types",
+				gr.getMaintitle());
+
+		Assertions.assertEquals(null, gr.getSubtitle());
+
+		Assertions.assertEquals(1, gr.getDescription().size());
+
+		Assertions
+			.assertTrue(
+				gr
+					.getDescription()
+					.get(0)
+					.startsWith("The implementation of the Ecosystem Service (ES) concept into practice"));
+		Assertions
+			.assertTrue(
+				gr
+					.getDescription()
+					.get(0)
+					.endsWith(
+						"start complying with new standards and demands for nature conservation and environmental management."));
+
+		Assertions.assertEquals("2017-01-01", gr.getPublicationdate());
+
+		Assertions.assertEquals("Pensoft Publishers", gr.getPublisher());
+
+		Assertions.assertEquals(null, gr.getEmbargoenddate());
+
+		Assertions.assertEquals(1, gr.getSource().size());
+		Assertions.assertEquals("One Ecosystem 2: e13718", gr.getSource().get(0));
+
+		Assertions.assertEquals(1, gr.getFormat().size());
+		Assertions.assertEquals("text/html", gr.getFormat().get(0));
+
+		Assertions.assertEquals(0, gr.getContributor().size());
+
+		Assertions.assertEquals(0, gr.getCoverage().size());
+
+		Assertions.assertEquals(ModelConstants.ACCESS_RIGHT_OPEN, gr.getBestaccessright().getLabel());
+		Assertions
+			.assertEquals(
+				Constants.accessRightsCoarMap.get(ModelConstants.ACCESS_RIGHT_OPEN), gr.getBestaccessright().getCode());
+		Assertions.assertEquals(null, gr.getBestaccessright().getOpenAccessRoute());
+
+		Assertions.assertEquals("One Ecosystem", gr.getContainer().getName());
+		Assertions.assertEquals("2367-8194", gr.getContainer().getIssnOnline());
+		Assertions.assertEquals("", gr.getContainer().getIssnPrinted());
+		Assertions.assertEquals("", gr.getContainer().getIssnLinking());
+
+		Assertions.assertTrue(null == gr.getDocumentationUrl() || gr.getDocumentationUrl().size() == 0);
+
+		Assertions.assertTrue(null == gr.getCodeRepositoryUrl());
+
+		Assertions.assertEquals(null, gr.getProgrammingLanguage());
+
+		Assertions.assertTrue(null == gr.getContactperson() || gr.getContactperson().size() == 0);
+
+		Assertions.assertTrue(null == gr.getContactgroup() || gr.getContactgroup().size() == 0);
+
+		Assertions.assertTrue(null == gr.getTool() || gr.getTool().size() == 0);
+
+		Assertions.assertEquals(null, gr.getSize());
+
+		Assertions.assertEquals(null, gr.getVersion());
+
+		Assertions.assertTrue(null == gr.getGeolocation() || gr.getGeolocation().size() == 0);
+
+		Assertions.assertEquals("50|pensoft_____::00ea4a1cd53806a97d62ea6bf268f2a2", gr.getId());
+
+		Assertions.assertEquals(2, gr.getOriginalId().size());
+		Assertions
+			.assertTrue(
+				gr.getOriginalId().contains("50|pensoft_____::00ea4a1cd53806a97d62ea6bf268f2a2")
+					&& gr.getOriginalId().contains("10.3897/oneeco.2.e13718"));
+
+		Assertions.assertEquals(1, gr.getPid().size());
+		Assertions
+			.assertTrue(
+				gr.getPid().get(0).getScheme().equals("doi")
+					&& gr.getPid().get(0).getValue().equals("10.1016/j.triboint.2014.05.004"));
+
+		Assertions.assertEquals("2020-03-23T00:20:51.392Z", gr.getDateofcollection());
+
+		Assertions.assertEquals(1, gr.getInstance().size());
+
+		Instance instance = gr.getInstance().get(0);
+		Assertions.assertEquals(0, instance.getPid().size());
+		Assertions.assertEquals(1, instance.getAlternateIdentifier().size());
+		Assertions
+			.assertTrue(
+				instance.getAlternateIdentifier().get(0).getScheme().equals("doi")
+					&& instance.getAlternateIdentifier().get(0).getValue().equals("10.3897/oneeco.2.e13718"));
+		Assertions.assertEquals(null, instance.getLicense());
+		Assertions
+			.assertTrue(
+				instance
+					.getAccessright()
+					.getCode()
+					.equals(
+						Constants.accessRightsCoarMap
+							.get(ModelConstants.ACCESS_RIGHT_OPEN)));
+		Assertions.assertTrue(instance.getAccessright().getLabel().equals(ModelConstants.ACCESS_RIGHT_OPEN));
+		Assertions.assertTrue(instance.getAccessright().getOpenAccessRoute().equals(OpenAccessRoute.green));
+		Assertions.assertTrue(instance.getType().equals("Article"));
+		Assertions.assertEquals(2, instance.getUrl().size());
+		Assertions
+			.assertTrue(
+				instance.getUrl().contains("https://doi.org/10.3897/oneeco.2.e13718")
+					&& instance.getUrl().contains("https://oneecosystem.pensoft.net/article/13718/"));
+		Assertions.assertEquals("2017-01-01", instance.getPublicationdate());
+		Assertions.assertEquals(null, instance.getArticleprocessingcharge());
+		Assertions.assertEquals("peerReviewed", instance.getRefereed());
+	}
+
+	@Test
+	public void testDatasetDump() {
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/dataset_extendedinstance")
+			.getPath();
+
+		final String communityMapPath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
+			.getPath();
+
+		DumpProducts dump = new DumpProducts();
+		dump
+			.run(
+				false, sourcePath, workingDir.toString() + "/result",
+				communityMapPath, Dataset.class,
+				GraphResult.class, Constants.DUMPTYPE.COMPLETE.getType());
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<GraphResult> tmp = sc
+			.textFile(workingDir.toString() + "/result")
+			.map(item -> OBJECT_MAPPER.readValue(item, GraphResult.class));
+
+		org.apache.spark.sql.Dataset<GraphResult> verificationDataset = spark
+			.createDataset(tmp.rdd(), Encoders.bean(GraphResult.class));
+
+		Assertions.assertEquals(1, verificationDataset.count());
+
+		Assertions.assertEquals(1, verificationDataset.filter("type = 'dataset'").count());
+
+		// the common fields in the result have been already checked. Now checking only
+		// community specific fields
+
+		GraphResult gr = verificationDataset.first();
+
+		Assertions.assertEquals(2, gr.getGeolocation().size());
+		Assertions.assertEquals(2, gr.getGeolocation().stream().filter(gl -> gl.getBox().equals("")).count());
+		Assertions.assertEquals(1, gr.getGeolocation().stream().filter(gl -> gl.getPlace().equals("")).count());
+		Assertions.assertEquals(1, gr.getGeolocation().stream().filter(gl -> gl.getPoint().equals("")).count());
+		Assertions
+			.assertEquals(
+				1,
+				gr
+					.getGeolocation()
+					.stream()
+					.filter(gl -> gl.getPlace().equals("18 York St, Ottawa, ON K1N 5S6; Ottawa; Ontario; Canada"))
+					.count());
+		Assertions
+			.assertEquals(
+				1, gr.getGeolocation().stream().filter(gl -> gl.getPoint().equals("45.427242 -75.693904")).count());
+		Assertions
+			.assertEquals(
+				1,
+				gr
+					.getGeolocation()
+					.stream()
+					.filter(gl -> gl.getPoint().equals("") && !gl.getPlace().equals(""))
+					.count());
+		Assertions
+			.assertEquals(
+				1,
+				gr
+					.getGeolocation()
+					.stream()
+					.filter(gl -> !gl.getPoint().equals("") && gl.getPlace().equals(""))
+					.count());
+
+		Assertions.assertEquals("1024Gb", gr.getSize());
+
+		Assertions.assertEquals("1.01", gr.getVersion());
+
+		Assertions.assertEquals(null, gr.getContainer());
+		Assertions.assertEquals(null, gr.getCodeRepositoryUrl());
+		Assertions.assertEquals(null, gr.getProgrammingLanguage());
+		Assertions.assertEquals(null, gr.getDocumentationUrl());
+		Assertions.assertEquals(null, gr.getContactperson());
+		Assertions.assertEquals(null, gr.getContactgroup());
+		Assertions.assertEquals(null, gr.getTool());
+
+	}
+
+	@Test
+	public void testSoftwareDump() {
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/software_extendedinstance")
+			.getPath();
+
+		final String communityMapPath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
+			.getPath();
+
+		DumpProducts dump = new DumpProducts();
+		dump
+			.run(
+				false, sourcePath, workingDir.toString() + "/result",
+				communityMapPath, Software.class,
+				GraphResult.class, Constants.DUMPTYPE.COMPLETE.getType());
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<GraphResult> tmp = sc
+			.textFile(workingDir.toString() + "/result")
+			.map(item -> OBJECT_MAPPER.readValue(item, GraphResult.class));
+
+		org.apache.spark.sql.Dataset<GraphResult> verificationDataset = spark
+			.createDataset(tmp.rdd(), Encoders.bean(GraphResult.class));
+
+		Assertions.assertEquals(1, verificationDataset.count());
+
+		Assertions.assertEquals(1, verificationDataset.filter("type = 'software'").count());
+
+		GraphResult gr = verificationDataset.first();
+
+		Assertions.assertEquals(2, gr.getDocumentationUrl().size());
+		Assertions.assertTrue(gr.getDocumentationUrl().contains("doc_url_1"));
+		Assertions.assertTrue(gr.getDocumentationUrl().contains("doc_url_2"));
+
+		Assertions.assertEquals("code_repo", gr.getCodeRepositoryUrl());
+
+		Assertions.assertEquals("perl", gr.getProgrammingLanguage());
+
+		Assertions.assertEquals(null, gr.getContainer());
+		Assertions.assertEquals(null, gr.getContactperson());
+		Assertions.assertEquals(null, gr.getContactgroup());
+		Assertions.assertEquals(null, gr.getTool());
+		Assertions.assertEquals(null, gr.getGeolocation());
+		Assertions.assertEquals(null, gr.getSize());
+		Assertions.assertEquals(null, gr.getVersion());
+
+	}
+
+	@Test
+	public void testOrpDump() {
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/orp_extendedinstance")
+			.getPath();
+
+		final String communityMapPath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
+			.getPath();
+
+		DumpProducts dump = new DumpProducts();
+		dump
+			.run(
+				false, sourcePath, workingDir.toString() + "/result",
+				communityMapPath, OtherResearchProduct.class,
+				GraphResult.class, Constants.DUMPTYPE.COMPLETE.getType());
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<GraphResult> tmp = sc
+			.textFile(workingDir.toString() + "/result")
+			.map(item -> OBJECT_MAPPER.readValue(item, GraphResult.class));
+
+		org.apache.spark.sql.Dataset<GraphResult> verificationDataset = spark
+			.createDataset(tmp.rdd(), Encoders.bean(GraphResult.class));
+
+		Assertions.assertEquals(1, verificationDataset.count());
+
+		Assertions.assertEquals(1, verificationDataset.filter("type = 'other'").count());
+
+		GraphResult gr = verificationDataset.first();
+
+		Assertions.assertEquals(2, gr.getContactperson().size());
+		Assertions.assertTrue(gr.getContactperson().contains(("contact_person1")));
+		Assertions.assertTrue(gr.getContactperson().contains(("contact_person2")));
+
+		Assertions.assertEquals(1, gr.getContactgroup().size());
+		Assertions.assertTrue(gr.getContactgroup().contains(("contact_group")));
+
+		Assertions.assertEquals(2, gr.getTool().size());
+		Assertions.assertTrue(gr.getTool().contains("tool1"));
+		Assertions.assertTrue(gr.getTool().contains("tool2"));
+
+		Assertions.assertEquals(null, gr.getContainer());
+		Assertions.assertEquals(null, gr.getDocumentationUrl());
+		Assertions.assertEquals(null, gr.getCodeRepositoryUrl());
+		Assertions.assertEquals(null, gr.getProgrammingLanguage());
+		Assertions.assertEquals(null, gr.getGeolocation());
+		Assertions.assertEquals(null, gr.getSize());
+		Assertions.assertEquals(null, gr.getVersion());
+
+	}
+
+	@Test
+	public void testPublicationDumpCommunity() throws JsonProcessingException {
+
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/publication_extendedinstance")
+			.getPath();
+
+		final String communityMapPath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
+			.getPath();
+
+		DumpProducts dump = new DumpProducts();
+		dump
+			.run(
+				false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
+				CommunityResult.class, Constants.DUMPTYPE.COMMUNITY.getType());
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<CommunityResult> tmp = sc
+			.textFile(workingDir.toString() + "/result")
+			.map(item -> OBJECT_MAPPER.readValue(item, CommunityResult.class));
+
+		org.apache.spark.sql.Dataset<CommunityResult> verificationDataset = spark
+			.createDataset(tmp.rdd(), Encoders.bean(CommunityResult.class));
+
+		Assertions.assertEquals(1, verificationDataset.count());
+
+		Assertions.assertEquals(1, verificationDataset.filter("type = 'publication'").count());
+
+		// the common fields in the result have been already checked. Now checking only
+		// community specific fields
+
+		CommunityResult cr = verificationDataset.first();
+
+		Assertions.assertEquals(1, cr.getContext().size());
+		Assertions.assertEquals("dh-ch", cr.getContext().get(0).getCode());
+		Assertions.assertEquals("Digital Humanities and Cultural Heritage", cr.getContext().get(0).getLabel());
+		Assertions.assertEquals(1, cr.getContext().get(0).getProvenance().size());
+		Assertions.assertEquals("Inferred by OpenAIRE", cr.getContext().get(0).getProvenance().get(0).getProvenance());
+		Assertions.assertEquals("0.9", cr.getContext().get(0).getProvenance().get(0).getTrust());
+
+		Assertions.assertEquals(1, cr.getCollectedfrom().size());
+		Assertions
+			.assertEquals("10|openaire____::fdc7e0400d8c1634cdaf8051dbae23db", cr.getCollectedfrom().get(0).getKey());
+		Assertions.assertEquals("Pensoft", cr.getCollectedfrom().get(0).getValue());
+
+		Assertions.assertEquals(1, cr.getInstance().size());
+		Assertions
+			.assertEquals(
+				"10|openaire____::fdc7e0400d8c1634cdaf8051dbae23db",
+				cr.getInstance().get(0).getCollectedfrom().getKey());
+		Assertions.assertEquals("Pensoft", cr.getInstance().get(0).getCollectedfrom().getValue());
+		Assertions
+			.assertEquals(
+				"10|openaire____::e707e544b9a5bd23fc27fbfa65eb60dd", cr.getInstance().get(0).getHostedby().getKey());
+		Assertions.assertEquals("One Ecosystem", cr.getInstance().get(0).getHostedby().getValue());
+
+	}
+
 	@Test
 	public void testDataset() {
 
@@ -151,7 +661,6 @@ public class DumpJobTest {
 		DumpProducts dump = new DumpProducts();
 		dump
 			.run(
-				// false, sourcePath, workingDir.toString() + "/result", communityMapPath, Dataset.class,
 				false, sourcePath, workingDir.toString() + "/result", communityMapPath, Dataset.class,
 				CommunityResult.class, Constants.DUMPTYPE.COMMUNITY.getType());
 
@@ -166,10 +675,6 @@ public class DumpJobTest {
 
 		Assertions.assertEquals(90, verificationDataset.count());
 
-//		verificationDataset
-//			.filter("id = '50|DansKnawCris::1a960e20087cb46b93588e4e184e8a58'")
-//			.foreach((ForeachFunction<CommunityResult>) rec -> System.out.println(OBJECT_MAPPER.writeValueAsString(rec)));
-
 		Assertions
 			.assertTrue(
 				verificationDataset.filter("bestAccessright.code = 'c_abf2'").count() == verificationDataset
@@ -198,8 +703,6 @@ public class DumpJobTest {
 
 		Assertions.assertTrue(verificationDataset.filter("type = 'dataset'").count() == 90);
 
-//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
-
 	}
 
 	@Test
@@ -231,8 +734,6 @@ public class DumpJobTest {
 
 		Assertions.assertEquals(5, verificationDataset.count());
 
-		verificationDataset
-			.foreach((ForeachFunction<GraphResult>) res -> System.out.println(OBJECT_MAPPER.writeValueAsString(res)));
 	}
 
 	@Test
@@ -249,7 +750,6 @@ public class DumpJobTest {
 		DumpProducts dump = new DumpProducts();
 		dump
 			.run(
-				// false, sourcePath, workingDir.toString() + "/result", communityMapPath, Dataset.class,
 				false, sourcePath, workingDir.toString() + "/result", communityMapPath, Dataset.class,
 				CommunityResult.class, Constants.DUMPTYPE.COMMUNITY.getType());
 
@@ -264,7 +764,6 @@ public class DumpJobTest {
 
 		Assertions.assertEquals(0, verificationDataset.count());
 
-		verificationDataset.show(false);
 	}
 
 	@Test
@@ -299,8 +798,6 @@ public class DumpJobTest {
 
 		Assertions.assertEquals(74, verificationDataset.filter("type = 'publication'").count());
 
-//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
-
 	}
 
 	@Test
@@ -333,9 +830,6 @@ public class DumpJobTest {
 		Assertions.assertEquals(6, verificationDataset.count());
 
 		Assertions.assertEquals(6, verificationDataset.filter("type = 'software'").count());
-		verificationDataset.show(false);
-
-//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
 
 	}
 
@@ -369,9 +863,6 @@ public class DumpJobTest {
 		Assertions.assertEquals(3, verificationDataset.count());
 
 		Assertions.assertEquals(3, verificationDataset.filter("type = 'other'").count());
-		verificationDataset.show(false);
-
-//TODO verify value and name of the fields for vocab related value (i.e. accessright, bestaccessright)
 
 	}
 
@@ -388,7 +879,6 @@ public class DumpJobTest {
 		DumpProducts dump = new DumpProducts();
 		dump
 			.run(
-				// false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
 				false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
 				CommunityResult.class, Constants.DUMPTYPE.COMMUNITY.getType());
 
@@ -408,4 +898,51 @@ public class DumpJobTest {
 
 	}
 
+	@Test
+	public void testArticlePCA() {
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/resultDump/publication_pca")
+			.getPath();
+
+		final String communityMapPath = getClass()
+			.getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
+			.getPath();
+
+		DumpProducts dump = new DumpProducts();
+		dump
+			.run(
+				// false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
+				false, sourcePath, workingDir.toString() + "/result", communityMapPath, Publication.class,
+				GraphResult.class, Constants.DUMPTYPE.COMPLETE.getType());
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<GraphResult> tmp = sc
+			.textFile(workingDir.toString() + "/result")
+			.map(item -> OBJECT_MAPPER.readValue(item, GraphResult.class));
+
+		org.apache.spark.sql.Dataset<GraphResult> verificationDataset = spark
+			.createDataset(tmp.rdd(), Encoders.bean(GraphResult.class));
+
+		Assertions.assertEquals(23, verificationDataset.count());
+
+		Assertions.assertEquals(23, verificationDataset.filter("type = 'publication'").count());
+
+		verificationDataset.createOrReplaceTempView("check");
+
+		org.apache.spark.sql.Dataset<Row> temp = spark
+			.sql(
+				"select id " +
+					"from check " +
+					"lateral view explode (instance) i as inst " +
+					"where inst.articleprocessingcharge is not null");
+
+		Assertions.assertTrue(temp.count() == 2);
+
+		Assertions.assertTrue(temp.filter("id = '50|datacite____::05c611fdfc93d7a2a703d1324e28104a'").count() == 1);
+
+		Assertions.assertTrue(temp.filter("id = '50|dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'").count() == 1);
+
+	}
+
 }