merge with beta - resolved conflict in pom

dhp-common/src/main/java/eu/dnetlib/dhp/common/HdfsSupport.java

@@ -28,7 +28,7 @@ public class HdfsSupport {
 	 * @param configuration Configuration of hadoop env
 	 */
 	public static boolean exists(String path, Configuration configuration) {
-		logger.info("Removing path: {}", path);
+		logger.info("Checking existence for path: {}", path);
 		return rethrowAsRuntimeException(
 			() -> {
 				Path f = new Path(path);

dhp-common/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java

@@ -85,6 +85,13 @@ public class MakeTarArchive implements Serializable {
 		String p_string = p.toString();
 		if (!p_string.endsWith("_SUCCESS")) {
 			String name = p_string.substring(p_string.lastIndexOf("/") + 1);
+			if (name.startsWith("part-") & name.length() > 10) {
+				String tmp = name.substring(0, 10);
+				if (name.contains(".")) {
+					tmp += name.substring(name.indexOf("."));
+				}
+				name = tmp;
+			}
 			TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
 			entry.setSize(fileStatus.getLen());
 			current_size += fileStatus.getLen();

dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java

@@ -4,19 +4,19 @@ package eu.dnetlib.dhp.utils;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Base64OutputStream;
+import java.util.*;
+import java.util.stream.Collectors;
+
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.SaveMode;
 import org.slf4j.Logger;
@@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Maps;
 import com.jayway.jsonpath.JsonPath;
 
+import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
 import net.minidev.json.JSONArray;
 import scala.collection.JavaConverters;
 import scala.collection.Seq;
@@ -52,10 +54,56 @@ public class DHPUtils {
 		}
 	}
 
+	/**
+	 * Retrieves from the metadata store manager application the list of paths associated with mdstores characterized
+	 * by he given format, layout, interpretation
+	 * @param mdstoreManagerUrl the URL of the mdstore manager service
+	 * @param format the mdstore format
+	 * @param layout the mdstore layout
+	 * @param interpretation the mdstore interpretation
+	 * @param includeEmpty include Empty mdstores
+	 * @return the set of hdfs paths
+	 * @throws IOException in case of HTTP communication issues
+	 */
+	public static Set<String> mdstorePaths(final String mdstoreManagerUrl,
+		final String format,
+		final String layout,
+		final String interpretation,
+		boolean includeEmpty) throws IOException {
+		final String url = mdstoreManagerUrl + "/mdstores/";
+		final ObjectMapper objectMapper = new ObjectMapper();
+
+		final HttpGet req = new HttpGet(url);
+
+		try (final CloseableHttpClient client = HttpClients.createDefault()) {
+			try (final CloseableHttpResponse response = client.execute(req)) {
+				final String json = IOUtils.toString(response.getEntity().getContent());
+				final MDStoreWithInfo[] mdstores = objectMapper.readValue(json, MDStoreWithInfo[].class);
+				return Arrays
+					.stream(mdstores)
+					.filter(md -> md.getFormat().equalsIgnoreCase(format))
+					.filter(md -> md.getLayout().equalsIgnoreCase(layout))
+					.filter(md -> md.getInterpretation().equalsIgnoreCase(interpretation))
+					.filter(md -> StringUtils.isNotBlank(md.getHdfsPath()))
+					.filter(md -> StringUtils.isNotBlank(md.getCurrentVersion()))
+					.filter(md -> includeEmpty || md.getSize() > 0)
+					.map(md -> md.getHdfsPath() + "/" + md.getCurrentVersion() + "/store")
+					.collect(Collectors.toSet());
+			}
+		}
+	}
+
 	public static String generateIdentifier(final String originalId, final String nsPrefix) {
 		return String.format("%s::%s", nsPrefix, DHPUtils.md5(originalId));
 	}
 
+	public static String generateUnresolvedIdentifier(final String pid, final String pidType) {
+
+		final String cleanedPid = CleaningFunctions.normalizePidValue(pidType, pid);
+
+		return String.format("unresolved::%s::%s", cleanedPid, pidType.toLowerCase().trim());
+	}
+
 	public static String getJPathString(final String jsonPath, final String json) {
 		try {
 			Object o = JsonPath.read(json, jsonPath);

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/Constants.java

@@ -0,0 +1,49 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
+
+import java.util.Optional;
+
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+
+public class Constants {
+
+	public static final String DOI = "doi";
+
+	public static final String UPDATE_DATA_INFO_TYPE = "update";
+	public static final String UPDATE_SUBJECT_FOS_CLASS_ID = "subject:fos";
+	public static final String UPDATE_CLASS_NAME = "Inferred by OpenAIRE";
+	public static final String UPDATE_MEASURE_BIP_CLASS_ID = "measure:bip";
+
+	public static final String FOS_CLASS_ID = "FOS";
+	public static final String FOS_CLASS_NAME = "Fields of Science and Technology classification";
+
+	public static final String NULL = "NULL";
+
+	public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	private Constants() {
+	}
+
+	public static Boolean isSparkSessionManaged(ArgumentApplicationParser parser) {
+		return Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+	}
+
+	public static <R> Dataset<R> readPath(
+		SparkSession spark, String inputPath, Class<R> clazz) {
+		return spark
+			.read()
+			.textFile(inputPath)
+			.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSData.java

@@ -0,0 +1,77 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.collection.GetCSV;
+
+public class GetFOSData implements Serializable {
+
+	private static final Logger log = LoggerFactory.getLogger(GetFOSData.class);
+
+	public static final char DEFAULT_DELIMITER = '\t';
+
+	public static void main(final String[] args) throws Exception {
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					Objects
+						.requireNonNull(
+							GetFOSData.class
+								.getResourceAsStream(
+									"/eu/dnetlib/dhp/actionmanager/createunresolvedentities/get_fos_parameters.json"))));
+
+		parser.parseArgument(args);
+
+		// the path where the original fos csv file is stored
+		final String sourcePath = parser.get("sourcePath");
+		log.info("sourcePath {}", sourcePath);
+
+		// the path where to put the file as json
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath {}", outputPath);
+
+		final String hdfsNameNode = parser.get("hdfsNameNode");
+		log.info("hdfsNameNode {}", hdfsNameNode);
+
+		final String classForName = parser.get("classForName");
+		log.info("classForName {}", classForName);
+
+		final char delimiter = Optional
+			.ofNullable(parser.get("delimiter"))
+			.map(s -> s.charAt(0))
+			.orElse(DEFAULT_DELIMITER);
+		log.info("delimiter {}", delimiter);
+
+		Configuration conf = new Configuration();
+		conf.set("fs.defaultFS", hdfsNameNode);
+
+		FileSystem fileSystem = FileSystem.get(conf);
+
+		new GetFOSData().doRewrite(sourcePath, outputPath, classForName, delimiter, fileSystem);
+
+	}
+
+	public void doRewrite(String inputPath, String outputFile, String classForName, char delimiter, FileSystem fs)
+		throws IOException, ClassNotFoundException {
+
+		// reads the csv and writes it as its json equivalent
+		try (InputStreamReader reader = new InputStreamReader(fs.open(new Path(inputPath)))) {
+			GetCSV.getCsv(fs, reader, outputFile, classForName, delimiter);
+		}
+
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java

@@ -0,0 +1,145 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
+
+import static eu.dnetlib.dhp.actionmanager.createunresolvedentities.Constants.*;
+import static eu.dnetlib.dhp.actionmanager.createunresolvedentities.Constants.UPDATE_CLASS_NAME;
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hdfs.client.HdfsUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.BipDeserialize;
+import eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.BipScore;
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Measure;
+import eu.dnetlib.dhp.schema.oaf.Result;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
+import eu.dnetlib.dhp.utils.DHPUtils;
+
+public class PrepareBipFinder implements Serializable {
+
+	private static final Logger log = LoggerFactory.getLogger(PrepareBipFinder.class);
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	public static <I extends Result> void main(String[] args) throws Exception {
+
+		String jsonConfiguration = IOUtils
+			.toString(
+				PrepareBipFinder.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/actionmanager/createunresolvedentities/prepare_parameters.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		final String sourcePath = parser.get("sourcePath");
+		log.info("sourcePath {}: ", sourcePath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath {}: ", outputPath);
+
+		SparkConf conf = new SparkConf();
+
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration());
+				prepareResults(spark, sourcePath, outputPath);
+			});
+	}
+
+	private static <I extends Result> void prepareResults(SparkSession spark, String inputPath, String outputPath) {
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc
+			.textFile(inputPath)
+			.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class));
+
+		spark
+			.createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> {
+				BipScore bs = new BipScore();
+				bs.setId(key);
+				bs.setScoreList(entry.get(key));
+				return bs;
+			}).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class))
+			.map((MapFunction<BipScore, Result>) v -> {
+				Result r = new Result();
+
+				r.setId(DHPUtils.generateUnresolvedIdentifier(v.getId(), DOI));
+				r.setMeasures(getMeasure(v));
+				return r;
+			}, Encoders.bean(Result.class))
+			.write()
+			.mode(SaveMode.Overwrite)
+			.option("compression", "gzip")
+			.json(outputPath + "/bip");
+	}
+
+	private static List<Measure> getMeasure(BipScore value) {
+		return value
+			.getScoreList()
+			.stream()
+			.map(score -> {
+				Measure m = new Measure();
+				m.setId(score.getId());
+				m
+					.setUnit(
+						score
+							.getUnit()
+							.stream()
+							.map(unit -> {
+								KeyValue kv = new KeyValue();
+								kv.setValue(unit.getValue());
+								kv.setKey(unit.getKey());
+								kv
+									.setDataInfo(
+										OafMapperUtils
+											.dataInfo(
+												false,
+												UPDATE_DATA_INFO_TYPE,
+												true,
+												false,
+												OafMapperUtils
+													.qualifier(
+														UPDATE_MEASURE_BIP_CLASS_ID,
+														UPDATE_CLASS_NAME,
+														ModelConstants.DNET_PROVENANCE_ACTIONS,
+														ModelConstants.DNET_PROVENANCE_ACTIONS),
+												""));
+								return kv;
+							})
+							.collect(Collectors.toList()));
+				return m;
+			})
+			.collect(Collectors.toList());
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java

@@ -0,0 +1,133 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
+
+import static eu.dnetlib.dhp.actionmanager.createunresolvedentities.Constants.*;
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.FOSDataModel;
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.Result;
+import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
+import eu.dnetlib.dhp.utils.DHPUtils;
+
+public class PrepareFOSSparkJob implements Serializable {
+	private static final Logger log = LoggerFactory.getLogger(PrepareFOSSparkJob.class);
+
+	public static void main(String[] args) throws Exception {
+
+		String jsonConfiguration = IOUtils
+			.toString(
+				PrepareFOSSparkJob.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/actionmanager/createunresolvedentities/prepare_parameters.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = isSparkSessionManaged(parser);
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		String sourcePath = parser.get("sourcePath");
+		log.info("sourcePath: {}", sourcePath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath: {}", outputPath);
+
+		SparkConf conf = new SparkConf();
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				distributeFOSdois(
+					spark,
+					sourcePath,
+
+					outputPath);
+			});
+	}
+
+	private static void distributeFOSdois(SparkSession spark, String sourcePath, String outputPath) {
+		Dataset<FOSDataModel> fosDataset = readPath(spark, sourcePath, FOSDataModel.class);
+
+		fosDataset.flatMap((FlatMapFunction<FOSDataModel, FOSDataModel>) v -> {
+			List<FOSDataModel> fosList = new ArrayList<>();
+			final String level1 = v.getLevel1();
+			final String level2 = v.getLevel2();
+			final String level3 = v.getLevel3();
+			Arrays
+				.stream(v.getDoi().split("\u0002"))
+				.forEach(d -> fosList.add(FOSDataModel.newInstance(d, level1, level2, level3)));
+			return fosList.iterator();
+		}, Encoders.bean(FOSDataModel.class))
+			.map((MapFunction<FOSDataModel, Result>) value -> {
+				Result r = new Result();
+				r.setId(DHPUtils.generateUnresolvedIdentifier(value.getDoi(), DOI));
+				r.setSubject(getSubjects(value));
+				return r;
+			}, Encoders.bean(Result.class))
+			.write()
+			.mode(SaveMode.Overwrite)
+			.option("compression", "gzip")
+			.json(outputPath + "/fos");
+	}
+
+	private static List<StructuredProperty> getSubjects(FOSDataModel fos) {
+		return Arrays
+			.asList(getSubject(fos.getLevel1()), getSubject(fos.getLevel2()), getSubject(fos.getLevel3()))
+			.stream()
+			.filter(Objects::nonNull)
+			.collect(Collectors.toList());
+	}
+
+	private static StructuredProperty getSubject(String sbj) {
+		if (sbj.equals(NULL))
+			return null;
+		StructuredProperty sp = new StructuredProperty();
+		sp.setValue(sbj);
+		sp
+			.setQualifier(
+				OafMapperUtils
+					.qualifier(
+						FOS_CLASS_ID,
+						FOS_CLASS_NAME,
+						ModelConstants.DNET_SUBJECT_TYPOLOGIES,
+						ModelConstants.DNET_SUBJECT_TYPOLOGIES));
+		sp
+			.setDataInfo(
+				OafMapperUtils
+					.dataInfo(
+						false,
+						UPDATE_DATA_INFO_TYPE,
+						true,
+						false,
+						OafMapperUtils
+							.qualifier(
+								UPDATE_SUBJECT_FOS_CLASS_ID,
+								UPDATE_CLASS_NAME,
+								ModelConstants.DNET_PROVENANCE_ACTIONS,
+								ModelConstants.DNET_PROVENANCE_ACTIONS),
+						""));
+
+		return sp;
+
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java

@@ -0,0 +1,79 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
+
+import static eu.dnetlib.dhp.actionmanager.createunresolvedentities.Constants.*;
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.io.Serializable;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.api.java.function.MapGroupsFunction;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.schema.oaf.Result;
+
+public class SparkSaveUnresolved implements Serializable {
+	private static final Logger log = LoggerFactory.getLogger(PrepareFOSSparkJob.class);
+
+	public static void main(String[] args) throws Exception {
+
+		String jsonConfiguration = IOUtils
+			.toString(
+				PrepareFOSSparkJob.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/actionmanager/createunresolvedentities/produce_unresolved_parameters.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = isSparkSessionManaged(parser);
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		String sourcePath = parser.get("sourcePath");
+		log.info("sourcePath: {}", sourcePath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath: {}", outputPath);
+
+		SparkConf conf = new SparkConf();
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				saveUnresolved(
+					spark,
+					sourcePath,
+
+					outputPath);
+			});
+	}
+
+	private static void saveUnresolved(SparkSession spark, String sourcePath, String outputPath) {
+
+		spark
+			.read()
+			.textFile(sourcePath + "/*")
+			.map(
+				(MapFunction<String, Result>) l -> OBJECT_MAPPER.readValue(l, Result.class),
+				Encoders.bean(Result.class))
+			.groupByKey((MapFunction<Result, String>) r -> r.getId(), Encoders.STRING())
+			.mapGroups((MapGroupsFunction<String, Result, Result>) (k, it) -> {
+				Result ret = it.next();
+				it.forEachRemaining(r -> ret.mergeFrom(r));
+				return ret;
+			}, Encoders.bean(Result.class))
+			.write()
+			.mode(SaveMode.Overwrite)
+			.option("compression", "gzip")
+			.json(outputPath);
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/BipDeserialize.java

@@ -0,0 +1,28 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Class that maps the model of the bipFinder! input data.
+ * Only needed for deserialization purposes
+ */
+
+public class BipDeserialize extends HashMap<String, List<Score>> implements Serializable {
+
+	public BipDeserialize() {
+		super();
+	}
+
+	public List<Score> get(String key) {
+
+		if (super.get(key) == null) {
+			return new ArrayList<>();
+		}
+		return super.get(key);
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/BipScore.java

@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities.model;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Rewriting of the bipFinder input data by extracting the identifier of the result (doi)
+ */
+
+public class BipScore implements Serializable {
+	private String id; // doi
+	private List<Score> scoreList; // unit as given in the inputfile
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public List<Score> getScoreList() {
+		return scoreList;
+	}
+
+	public void setScoreList(List<Score> scoreList) {
+		this.scoreList = scoreList;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java

@@ -0,0 +1,71 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities.model;
+
+import java.io.Serializable;
+
+import com.opencsv.bean.CsvBindByPosition;
+
+public class FOSDataModel implements Serializable {
+	@CsvBindByPosition(position = 1)
+//    @CsvBindByName(column = "doi")
+	private String doi;
+
+	@CsvBindByPosition(position = 2)
+//    @CsvBindByName(column = "level1")
+	private String level1;
+
+	@CsvBindByPosition(position = 3)
+//    @CsvBindByName(column = "level2")
+	private String level2;
+
+	@CsvBindByPosition(position = 4)
+//    @CsvBindByName(column = "level3")
+	private String level3;
+
+	public FOSDataModel() {
+
+	}
+
+	public FOSDataModel(String doi, String level1, String level2, String level3) {
+		this.doi = doi;
+		this.level1 = level1;
+		this.level2 = level2;
+		this.level3 = level3;
+	}
+
+	public static FOSDataModel newInstance(String d, String level1, String level2, String level3) {
+		return new FOSDataModel(d, level1, level2, level3);
+	}
+
+	public String getDoi() {
+		return doi;
+	}
+
+	public void setDoi(String doi) {
+		this.doi = doi;
+	}
+
+	public String getLevel1() {
+		return level1;
+	}
+
+	public void setLevel1(String level1) {
+		this.level1 = level1;
+	}
+
+	public String getLevel2() {
+		return level2;
+	}
+
+	public void setLevel2(String level2) {
+		this.level2 = level2;
+	}
+
+	public String getLevel3() {
+		return level3;
+	}
+
+	public void setLevel3(String level3) {
+		this.level3 = level3;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/KeyValue.java

@@ -0,0 +1,26 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities.model;
+
+import java.io.Serializable;
+
+public class KeyValue implements Serializable {
+
+	private String key;
+	private String value;
+
+	public String getKey() {
+		return key;
+	}
+
+	public void setKey(String key) {
+		this.key = key;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(String value) {
+		this.value = value;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/Score.java

@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities.model;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * represents the score in the input file
+ */
+public class Score implements Serializable {
+
+	private String id;
+	private List<KeyValue> unit;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public List<KeyValue> getUnit() {
+		return unit;
+	}
+
+	public void setUnit(List<KeyValue> unit) {
+		this.unit = unit;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/ExportActionSetJobNode.scala

@@ -1,41 +0,0 @@
-package eu.dnetlib.dhp.actionmanager.datacite
-
-import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.Oaf
-import org.apache.hadoop.io.Text
-import org.apache.hadoop.io.compress.GzipCodec
-import org.apache.hadoop.mapred.SequenceFileOutputFormat
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.io.Source
-
-object ExportActionSetJobNode {
-
-  val log: Logger = LoggerFactory.getLogger(ExportActionSetJobNode.getClass)
-
-  def main(args: Array[String]): Unit = {
-    val conf = new SparkConf
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/exportDataset_parameters.json")).mkString)
-    parser.parseArgument(args)
-    val master = parser.get("master")
-    val sourcePath = parser.get("sourcePath")
-    val targetPath = parser.get("targetPath")
-
-    val spark: SparkSession = SparkSession.builder().config(conf)
-      .appName(ExportActionSetJobNode.getClass.getSimpleName)
-      .master(master)
-      .getOrCreate()
-    implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
-    implicit val tEncoder:Encoder[(String,String)] = Encoders.tuple(Encoders.STRING,Encoders.STRING)
-
-    spark.read.load(sourcePath).as[Oaf]
-      .map(o =>DataciteToOAFTransformation.toActionSet(o))
-      .filter(o => o!= null)
-      .rdd.map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$targetPath", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text,Text]], classOf[GzipCodec])
-
-
-  }
-
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/FilterCrossrefEntitiesSpark.scala

@@ -1,46 +0,0 @@
-package eu.dnetlib.dhp.actionmanager.datacite
-
-import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
-import eu.dnetlib.dhp.schema.mdstore.MetadataRecord
-import eu.dnetlib.dhp.schema.oaf.{Oaf, Result}
-import eu.dnetlib.dhp.utils.ISLookupClientFactory
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.io.Source
-
-object FilterCrossrefEntitiesSpark {
-
-  val log: Logger = LoggerFactory.getLogger(getClass.getClass)
-
-  def main(args: Array[String]): Unit = {
-    val conf = new SparkConf
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/filter_crossref_param.json")).mkString)
-    parser.parseArgument(args)
-    val master = parser.get("master")
-    val sourcePath = parser.get("sourcePath")
-    log.info("sourcePath: {}", sourcePath)
-    val targetPath = parser.get("targetPath")
-    log.info("targetPath: {}", targetPath)
-
-
-
-    val spark: SparkSession = SparkSession.builder().config(conf)
-      .appName(getClass.getSimpleName)
-      .master(master)
-      .getOrCreate()
-
-
-
-    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
-    implicit val resEncoder: Encoder[Result] = Encoders.kryo[Result]
-
-    val d:Dataset[Oaf]= spark.read.load(sourcePath).as[Oaf]
-
-    d.filter(r => r.isInstanceOf[Result]).map(r => r.asInstanceOf[Result]).write.mode(SaveMode.Overwrite).save(targetPath)
-
-  }
-
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/scholix/SparkCreateActionset.scala

@@ -60,14 +60,10 @@ object SparkCreateActionset {
 
     val entities: Dataset[(String, Result)] = spark.read.load(s"$sourcePath/entities/*").as[Result].map(p => (p.getId, p))(Encoders.tuple(Encoders.STRING, resultEncoders))
 
-
-    entities.filter(r => r.isInstanceOf[Result]).map(r => r.asInstanceOf[Result])
     entities
       .joinWith(idRelation, entities("_1").equalTo(idRelation("value")))
       .map(p => p._1._2)
       .write.mode(SaveMode.Append).save(s"$workingDirFolder/actionSetOaf")
-
-
   }
 
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectionUtils.scala

@@ -0,0 +1,49 @@
+package eu.dnetlib.dhp.collection
+
+import eu.dnetlib.dhp.schema.common.ModelSupport
+import eu.dnetlib.dhp.schema.oaf.{Oaf, OafEntity, Relation}
+
+object CollectionUtils {
+
+  /**
+   * This method in pipeline to the transformation phase,
+   * generates relations in both verse, typically it should be a phase of flatMap
+   *
+   * @param i input OAF
+   * @return
+   * If the input OAF is an entity -> List(i)
+   * If the input OAF is a relation -> List(relation, inverseRelation)
+   *
+   */
+
+  def fixRelations(i: Oaf): List[Oaf] = {
+    if (i.isInstanceOf[OafEntity])
+      return List(i)
+    else {
+      val r: Relation = i.asInstanceOf[Relation]
+      val currentRel = ModelSupport.findRelation(r.getRelClass)
+      if (currentRel != null) {
+
+        // Cleaning relation
+        r.setRelType(currentRel.getRelType)
+        r.setSubRelType(currentRel.getSubReltype)
+        r.setRelClass(currentRel.getRelClass)
+        val inverse = new Relation
+        inverse.setSource(r.getTarget)
+        inverse.setTarget(r.getSource)
+        inverse.setRelType(currentRel.getRelType)
+        inverse.setSubRelType(currentRel.getSubReltype)
+        inverse.setRelClass(currentRel.getInverseRelClass)
+        inverse.setCollectedfrom(r.getCollectedfrom)
+        inverse.setDataInfo(r.getDataInfo)
+        inverse.setProperties(r.getProperties)
+        inverse.setLastupdatetimestamp(r.getLastupdatetimestamp)
+        inverse.setValidated(r.getValidated)
+        inverse.setValidationDate(r.getValidationDate)
+        return List(r, inverse)
+      }
+    }
+    List()
+  }
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/AbstractRestClient.scala

@@ -1,12 +1,10 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
 import org.apache.commons.io.IOUtils
 import org.apache.http.client.config.RequestConfig
-import org.apache.http.client.methods.{HttpGet, HttpPost, HttpRequestBase, HttpUriRequest}
+import org.apache.http.client.methods.{HttpGet, HttpPost, HttpUriRequest}
 import org.apache.http.entity.StringEntity
-import org.apache.http.impl.client.{HttpClientBuilder, HttpClients}
-
-import java.io.IOException
+import org.apache.http.impl.client.HttpClientBuilder
 
 
 abstract class AbstractRestClient extends Iterator[String] {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/DataciteAPIImporter.scala

@@ -1,7 +1,7 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
-import org.json4s.{DefaultFormats, JValue}
 import org.json4s.jackson.JsonMethods.{compact, parse, render}
+import org.json4s.{DefaultFormats, JValue}
 
 class DataciteAPIImporter(timestamp: Long = 0, blocks: Long = 10, until:Long = -1) extends AbstractRestClient {
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala

@@ -1,4 +1,4 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
@@ -325,8 +325,9 @@ object DataciteToOAFTransformation {
       val grantId = m.matcher(awardUri).replaceAll("$2")
       val targetId = s"$p${DHPUtils.md5(grantId)}"
       List(
-        generateRelation(sourceId, targetId, "isProducedBy", DATACITE_COLLECTED_FROM, dataInfo),
-        generateRelation(targetId, sourceId, "produces", DATACITE_COLLECTED_FROM, dataInfo)
+        generateRelation(sourceId, targetId, "isProducedBy", DATACITE_COLLECTED_FROM, dataInfo)
+// REMOVED INVERSE RELATION since there is a specific method that should generate later
+//        generateRelation(targetId, sourceId, "produces", DATACITE_COLLECTED_FROM, dataInfo)
       )
     }
     else
@@ -580,11 +581,11 @@ object DataciteToOAFTransformation {
         rel.setProperties(List(dateProps).asJava)
 
         rel.setSource(id)
-        rel.setTarget(s"unresolved::${r.relatedIdentifier}::${r.relatedIdentifierType}")
+        rel.setTarget(DHPUtils.generateUnresolvedIdentifier(r.relatedIdentifier,r.relatedIdentifierType))
         rel.setCollectedfrom(List(DATACITE_COLLECTED_FROM).asJava)
-        rel.getCollectedfrom.asScala.map(c => c.getValue)(collection.breakOut)
+        rel.getCollectedfrom.asScala.map(c => c.getValue).toList
         rel
-      })(collection breakOut)
+      }).toList
   }
 
   def generateDataInfo(trust: String): DataInfo = {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/GenerateDataciteDatasetSpark.scala

@@ -1,9 +1,14 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.collection.CollectionUtils.fixRelations
+import eu.dnetlib.dhp.common.Constants.MDSTORE_DATA_PATH
+import eu.dnetlib.dhp.common.Constants.MDSTORE_SIZE_PATH
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
-import eu.dnetlib.dhp.schema.mdstore.MetadataRecord
+import eu.dnetlib.dhp.schema.mdstore.{MDStoreVersion, MetadataRecord}
 import eu.dnetlib.dhp.schema.oaf.Oaf
+import eu.dnetlib.dhp.utils.DHPUtils.writeHdfsFile
 import eu.dnetlib.dhp.utils.ISLookupClientFactory
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
@@ -17,11 +22,10 @@ object GenerateDataciteDatasetSpark {
 
   def main(args: Array[String]): Unit = {
     val conf = new SparkConf
-    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/generate_dataset_params.json")).mkString)
+    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/datacite/generate_dataset_params.json")).mkString)
     parser.parseArgument(args)
     val master = parser.get("master")
     val sourcePath = parser.get("sourcePath")
-    val targetPath = parser.get("targetPath")
     val exportLinks = "true".equalsIgnoreCase(parser.get("exportLinks"))
     val isLookupUrl: String = parser.get("isLookupUrl")
     log.info("isLookupUrl: {}", isLookupUrl)
@@ -33,16 +37,28 @@ object GenerateDataciteDatasetSpark {
       .master(master)
       .getOrCreate()
 
+    import spark.implicits._
+
     implicit val mrEncoder: Encoder[MetadataRecord] = Encoders.kryo[MetadataRecord]
 
     implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
 
-    import spark.implicits._
+    val mdstoreOutputVersion = parser.get("mdstoreOutputVersion")
+    val mapper = new ObjectMapper()
+    val cleanedMdStoreVersion = mapper.readValue(mdstoreOutputVersion, classOf[MDStoreVersion])
+    val outputBasePath = cleanedMdStoreVersion.getHdfsPath
+
+    log.info("outputBasePath: {}", outputBasePath)
+    val targetPath = s"$outputBasePath/$MDSTORE_DATA_PATH"
 
     spark.read.load(sourcePath).as[DataciteType]
       .filter(d => d.isActive)
       .flatMap(d => DataciteToOAFTransformation.generateOAF(d.json, d.timestamp, d.timestamp, vocabularies, exportLinks))
       .filter(d => d != null)
+      .flatMap(i => fixRelations(i)).filter(i => i != null)
       .write.mode(SaveMode.Overwrite).save(targetPath)
+
+    val total_items = spark.read.load(targetPath).as[Oaf].count()
+    writeHdfsFile(spark.sparkContext.hadoopConfiguration, s"$total_items", outputBasePath + MDSTORE_SIZE_PATH)
   }
-}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/ImportDatacite.scala

@@ -1,6 +1,5 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
-import eu.dnetlib.dhp.actionmanager.datacite.DataciteToOAFTransformation.df_it
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, LocalFileSystem, Path}
@@ -9,14 +8,14 @@ import org.apache.hadoop.io.{IntWritable, SequenceFile, Text}
 import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.expressions.Aggregator
+import org.apache.spark.sql.functions.max
 import org.apache.spark.sql.{Dataset, Encoder, SaveMode, SparkSession}
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
-import org.apache.spark.sql.functions.max
 import org.slf4j.{Logger, LoggerFactory}
 
-import java.time.format.DateTimeFormatter._
-import java.time.{LocalDate, LocalDateTime, ZoneOffset}
+import java.time.format.DateTimeFormatter.ISO_DATE_TIME
+import java.time.{LocalDateTime, ZoneOffset}
 import scala.io.Source
 
 object ImportDatacite {
@@ -138,11 +137,11 @@ object ImportDatacite {
     }
   }
 
-  private def writeSequenceFile(hdfsTargetPath: Path, timestamp: Long, conf: Configuration, bs:Int): Long = {
-    var from:Long = timestamp * 1000
-    val delta:Long = 100000000L
+  private def writeSequenceFile(hdfsTargetPath: Path, timestamp: Long, conf: Configuration, bs: Int): Long = {
+    var from: Long = timestamp * 1000
+    val delta: Long = 100000000L
     var client: DataciteAPIImporter = null
-    val now :Long =System.currentTimeMillis()
+    val now: Long = System.currentTimeMillis()
     var i = 0
     try {
       val writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(hdfsTargetPath), SequenceFile.Writer.keyClass(classOf[IntWritable]), SequenceFile.Writer.valueClass(classOf[Text]))
@@ -168,7 +167,7 @@ object ImportDatacite {
               start = System.currentTimeMillis
             }
           }
-          println(s"updating from value: $from  -> ${from+delta}")
+          println(s"updating from value: $from  -> ${from + delta}")
           from = from + delta
         }
       } catch {
@@ -183,4 +182,4 @@ object ImportDatacite {
     i
   }
 
-}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/datacite/SparkDownloadUpdateDatacite.scala

@@ -0,0 +1,49 @@
+package eu.dnetlib.dhp.datacite
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.schema.oaf.{Oaf, Result}
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.functions.max
+import org.apache.spark.sql.{Encoder, Encoders, SparkSession}
+import org.slf4j.{Logger, LoggerFactory}
+
+import java.text.SimpleDateFormat
+import java.util.Locale
+import scala.io.Source
+
+object SparkDownloadUpdateDatacite {
+  val log: Logger = LoggerFactory.getLogger(getClass)
+
+  def main(args: Array[String]): Unit = {
+
+    val conf = new SparkConf
+    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/datacite/generate_dataset_params.json")).mkString)
+    parser.parseArgument(args)
+    val master = parser.get("master")
+    val sourcePath = parser.get("sourcePath")
+    val workingPath = parser.get("workingPath")
+
+    val hdfsuri = parser.get("namenode")
+    log.info(s"namenode is $hdfsuri")
+
+
+    val spark: SparkSession = SparkSession.builder().config(conf)
+      .appName(getClass.getSimpleName)
+      .master(master)
+      .getOrCreate()
+
+    implicit val oafEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
+    implicit val resEncoder: Encoder[Result] = Encoders.kryo[Result]
+
+    import spark.implicits._
+
+
+    val maxDate: String = spark.read.load(workingPath).as[Oaf].filter(s => s.isInstanceOf[Result]).map(r => r.asInstanceOf[Result].getDateofcollection).select(max("value")).first().getString(0)
+    val ISO8601FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US)
+    val string_to_date = ISO8601FORMAT.parse(maxDate)
+    val ts = string_to_date.getTime
+
+
+  }
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala

@@ -1,14 +1,12 @@
-package eu.dnetlib.dhp.sx.graph.bio
+package eu.dnetlib.dhp.sx.bio
 
 import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.utils.{GraphCleaningFunctions, OafMapperUtils}
-import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Dataset, Instance, KeyValue, Oaf, Relation, StructuredProperty}
+import eu.dnetlib.dhp.schema.oaf._
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.{compact, parse, render}
-
-import scala.collection.JavaConverters._
-
+import collection.JavaConverters._
 object BioDBToOAF {
 
   case class EBILinkItem(id: Long, links: String) {}
@@ -17,23 +15,23 @@ object BioDBToOAF {
 
   case class UniprotDate(date: String, date_info: String) {}
 
-  case class ScholixResolved(pid:String, pidType:String, typology:String, tilte:List[String], datasource:List[String], date:List[String], authors:List[String]){}
+  case class ScholixResolved(pid: String, pidType: String, typology: String, tilte: List[String], datasource: List[String], date: List[String], authors: List[String]) {}
 
   val DATA_INFO: DataInfo = OafMapperUtils.dataInfo(false, null, false, false, ModelConstants.PROVENANCE_ACTION_SET_QUALIFIER, "0.9")
   val SUBJ_CLASS = "Keywords"
 
   val DATE_RELATION_KEY = "RelationDate"
 
-  val resolvedURL:Map[String,String] = Map(
-    "genbank"->             "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "ncbi-n" ->             "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "ncbi-wgs" ->           "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "ncbi-p" ->             "https://www.ncbi.nlm.nih.gov/protein/",
-    "ena" ->                "https://www.ebi.ac.uk/ena/browser/view/",
-    "clinicaltrials.gov"->  "https://clinicaltrials.gov/ct2/show/",
-    "onim"->                "https://omim.org/entry/",
-    "refseq"->              "https://www.ncbi.nlm.nih.gov/nuccore/",
-    "geo"->                 "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc="
+  val resolvedURL: Map[String, String] = Map(
+    "genbank" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "ncbi-n" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "ncbi-wgs" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "ncbi-p" -> "https://www.ncbi.nlm.nih.gov/protein/",
+    "ena" -> "https://www.ebi.ac.uk/ena/browser/view/",
+    "clinicaltrials.gov" -> "https://clinicaltrials.gov/ct2/show/",
+    "onim" -> "https://omim.org/entry/",
+    "refseq" -> "https://www.ncbi.nlm.nih.gov/nuccore/",
+    "geo" -> "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc="
   )
 
 
@@ -45,7 +43,7 @@ object BioDBToOAF {
     val ElsevierCollectedFrom: KeyValue = OafMapperUtils.keyValue("10|openaire____::8f87e10869299a5fe80b315695296b88", "Elsevier")
     val springerNatureCollectedFrom: KeyValue = OafMapperUtils.keyValue("10|openaire____::6e380d9cf51138baec8480f5a0ce3a2e", "Springer Nature")
     val EBICollectedFrom: KeyValue = OafMapperUtils.keyValue("10|opendoar____::83e60e09c222f206c725385f53d7e567c", "EMBL-EBIs Protein Data Bank in Europe (PDBe)")
-    val pubmedCollectedFrom:KeyValue = OafMapperUtils.keyValue(ModelConstants.EUROPE_PUBMED_CENTRAL_ID, "Europe PubMed Central")
+    val pubmedCollectedFrom: KeyValue = OafMapperUtils.keyValue(ModelConstants.EUROPE_PUBMED_CENTRAL_ID, "Europe PubMed Central")
 
     UNIPROTCollectedFrom.setDataInfo(DATA_INFO)
     PDBCollectedFrom.setDataInfo(DATA_INFO)
@@ -58,9 +56,9 @@ object BioDBToOAF {
 
     Map(
       "uniprot" -> UNIPROTCollectedFrom,
-      "pdb"-> PDBCollectedFrom,
-      "elsevier" ->ElsevierCollectedFrom,
-      "ebi" ->EBICollectedFrom,
+      "pdb" -> PDBCollectedFrom,
+      "elsevier" -> ElsevierCollectedFrom,
+      "ebi" -> EBICollectedFrom,
       "Springer Nature" -> springerNatureCollectedFrom,
       "NCBI Nucleotide" -> ncbiCollectedFrom,
       "European Nucleotide Archive" -> enaCollectedFrom,
@@ -68,7 +66,7 @@ object BioDBToOAF {
     )
   }
 
-  def crossrefLinksToOaf(input:String):Oaf = {
+  def crossrefLinksToOaf(input: String): Oaf = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
     lazy val json = parse(input)
     val source_pid = (json \ "Source" \ "Identifier" \ "ID").extract[String].toLowerCase
@@ -77,16 +75,16 @@ object BioDBToOAF {
     val target_pid = (json \ "Target" \ "Identifier" \ "ID").extract[String].toLowerCase
     val target_pid_type = (json \ "Target" \ "Identifier" \ "IDScheme").extract[String].toLowerCase
 
-    val relation_semantic= (json \ "RelationshipType" \ "Name").extract[String]
+    val relation_semantic = (json \ "RelationshipType" \ "Name").extract[String]
 
     val date = GraphCleaningFunctions.cleanDate((json \ "LinkedPublicationDate").extract[String])
 
-    createRelation(target_pid, target_pid_type, generate_unresolved_id(source_pid, source_pid_type),collectedFromMap("elsevier"),"relationship", relation_semantic, date)
+    createRelation(target_pid, target_pid_type, generate_unresolved_id(source_pid, source_pid_type), collectedFromMap("elsevier"), "relationship", relation_semantic, date)
 
   }
 
 
-  def scholixResolvedToOAF(input:ScholixResolved):Oaf = {
+  def scholixResolvedToOAF(input: ScholixResolved): Oaf = {
 
     val d = new Dataset
 
@@ -127,18 +125,18 @@ object BioDBToOAF {
     d.setInstance(List(i).asJava)
 
     if (input.authors != null && input.authors.nonEmpty) {
-      val authors = input.authors.map(a =>{
+      val authors = input.authors.map(a => {
         val authorOAF = new Author
         authorOAF.setFullname(a)
         authorOAF
       })
       d.setAuthor(authors.asJava)
     }
-    if (input.date!= null && input.date.nonEmpty) {
-        val dt = input.date.head
-        i.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
-        d.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
-      }
+    if (input.date != null && input.date.nonEmpty) {
+      val dt = input.date.head
+      i.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
+      d.setDateofacceptance(OafMapperUtils.field(GraphCleaningFunctions.cleanDate(dt), DATA_INFO))
+    }
     d
   }
 
@@ -190,7 +188,7 @@ object BioDBToOAF {
           OafMapperUtils.structuredProperty(s, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, null)
         ).asJava)
     }
-    var i_date:Option[UniprotDate] = None
+    var i_date: Option[UniprotDate] = None
 
     if (dates.nonEmpty) {
       i_date = dates.find(d => d.date_info.contains("entry version"))
@@ -218,12 +216,12 @@ object BioDBToOAF {
 
 
     if (references_pmid != null && references_pmid.nonEmpty) {
-      val rel = createRelation(references_pmid.head, "pmid", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if  (i_date.isDefined) i_date.get.date else null)
+      val rel = createRelation(references_pmid.head, "pmid", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if (i_date.isDefined) i_date.get.date else null)
       rel.getCollectedfrom
       List(d, rel)
     }
     else if (references_doi != null && references_doi.nonEmpty) {
-      val rel = createRelation(references_doi.head, "doi", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if  (i_date.isDefined) i_date.get.date else null)
+      val rel = createRelation(references_doi.head, "doi", d.getId, collectedFromMap("uniprot"), ModelConstants.RELATIONSHIP, ModelConstants.IS_RELATED_TO, if (i_date.isDefined) i_date.get.date else null)
       List(d, rel)
     }
     else
@@ -231,13 +229,12 @@ object BioDBToOAF {
   }
 
 
-
-  def generate_unresolved_id(pid:String, pidType:String) :String = {
+  def generate_unresolved_id(pid: String, pidType: String): String = {
     s"unresolved::$pid::$pidType"
   }
 
 
-  def createRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, subRelType:String, relClass:String, date:String):Relation = {
+  def createRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, subRelType: String, relClass: String, date: String): Relation = {
 
     val rel = new Relation
     rel.setCollectedfrom(List(collectedFromMap("pdb")).asJava)
@@ -251,7 +248,7 @@ object BioDBToOAF {
     rel.setTarget(s"unresolved::$pid::$pidType")
 
 
-    val dateProps:KeyValue = OafMapperUtils.keyValue(DATE_RELATION_KEY, date)
+    val dateProps: KeyValue = OafMapperUtils.keyValue(DATE_RELATION_KEY, date)
 
     rel.setProperties(List(dateProps).asJava)
 
@@ -262,8 +259,8 @@ object BioDBToOAF {
   }
 
 
-  def createSupplementaryRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, date:String): Relation = {
-    createRelation(pid,pidType,sourceId,collectedFrom, ModelConstants.SUPPLEMENT, ModelConstants.IS_SUPPLEMENT_TO, date)
+  def createSupplementaryRelation(pid: String, pidType: String, sourceId: String, collectedFrom: KeyValue, date: String): Relation = {
+    createRelation(pid, pidType, sourceId, collectedFrom, ModelConstants.SUPPLEMENT, ModelConstants.IS_SUPPLEMENT_TO, date)
   }
 
 
@@ -338,7 +335,7 @@ object BioDBToOAF {
 
   def EBITargetLinksFilter(input: EBILinks): Boolean = {
 
-    input.targetPidType.equalsIgnoreCase("ena") ||   input.targetPidType.equalsIgnoreCase("pdb") ||  input.targetPidType.equalsIgnoreCase("uniprot")
+    input.targetPidType.equalsIgnoreCase("ena") || input.targetPidType.equalsIgnoreCase("pdb") || input.targetPidType.equalsIgnoreCase("uniprot")
 
   }
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/SparkTransformBioDatabaseToOAF.scala

@@ -1,8 +1,9 @@
-package eu.dnetlib.dhp.sx.graph.bio
+package eu.dnetlib.dhp.sx.bio
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
-import eu.dnetlib.dhp.schema.oaf.{Oaf, Result}
+import eu.dnetlib.dhp.schema.oaf.Oaf
 import BioDBToOAF.ScholixResolved
+import eu.dnetlib.dhp.collection.CollectionUtils
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
@@ -13,7 +14,7 @@ object SparkTransformBioDatabaseToOAF {
   def main(args: Array[String]): Unit = {
     val conf: SparkConf = new SparkConf()
     val log: Logger = LoggerFactory.getLogger(getClass)
-    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/bio_to_oaf_params.json")))
+    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/bio/ebi/bio_to_oaf_params.json")))
     parser.parseArgument(args)
     val database: String = parser.get("database")
     log.info("database: {}", database)
@@ -31,18 +32,17 @@ object SparkTransformBioDatabaseToOAF {
         .master(parser.get("master")).getOrCreate()
     val sc = spark.sparkContext
 
-    implicit  val resultEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
-    import  spark.implicits._
-
+    implicit val resultEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
+    import spark.implicits._
     database.toUpperCase() match {
       case "UNIPROT" =>
-        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.uniprotToOAF(i))).write.mode(SaveMode.Overwrite).save(targetPath)
-      case "PDB"=>
-        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.pdbTOOaf(i))).write.mode(SaveMode.Overwrite).save(targetPath)
+        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.uniprotToOAF(i))).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
+      case "PDB" =>
+        spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.pdbTOOaf(i))).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
       case "SCHOLIX" =>
-        spark.read.load(dbPath).as[ScholixResolved].map(i => BioDBToOAF.scholixResolvedToOAF(i)).write.mode(SaveMode.Overwrite).save(targetPath)
-      case "CROSSREF_LINKS"=>
-        spark.createDataset(sc.textFile(dbPath).map(i => BioDBToOAF.crossrefLinksToOaf(i))).write.mode(SaveMode.Overwrite).save(targetPath)
+        spark.read.load(dbPath).as[ScholixResolved].map(i => BioDBToOAF.scholixResolvedToOAF(i)).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
+      case "CROSSREF_LINKS" =>
+        spark.createDataset(sc.textFile(dbPath).map(i => BioDBToOAF.crossrefLinksToOaf(i))).flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null).write.mode(SaveMode.Overwrite).save(targetPath)
     }
   }
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala

@@ -0,0 +1,200 @@
+package eu.dnetlib.dhp.sx.bio.ebi
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
+import eu.dnetlib.dhp.schema.oaf.Result
+import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal, PMParser, PubMedToOaf}
+import eu.dnetlib.dhp.utils.ISLookupClientFactory
+import org.apache.commons.io.IOUtils
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}
+import org.apache.http.client.config.RequestConfig
+import org.apache.http.client.methods.HttpGet
+import org.apache.http.impl.client.HttpClientBuilder
+import org.apache.spark.SparkConf
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.expressions.Aggregator
+import org.apache.spark.sql._
+import org.slf4j.{Logger, LoggerFactory}
+
+import java.io.InputStream
+import scala.io.Source
+import scala.xml.pull.XMLEventReader
+
+object SparkCreateBaselineDataFrame {
+
+
+  def requestBaseLineUpdatePage(maxFile: String): List[(String, String)] = {
+    val data = requestPage("https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/")
+
+    val result = data.lines.filter(l => l.startsWith("<a href=")).map { l =>
+      val end = l.lastIndexOf("\">")
+      val start = l.indexOf("<a href=\"")
+
+      if (start >= 0 && end > start)
+        l.substring(start + 9, end - start)
+      else
+        ""
+    }.filter(s => s.endsWith(".gz")).filter(s => s > maxFile).map(s => (s, s"https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/$s")).toList
+
+    result
+  }
+
+
+  def downloadBaselinePart(url: String): InputStream = {
+    val r = new HttpGet(url)
+    val timeout = 60; // seconds
+    val config = RequestConfig.custom()
+      .setConnectTimeout(timeout * 1000)
+      .setConnectionRequestTimeout(timeout * 1000)
+      .setSocketTimeout(timeout * 1000).build()
+    val client = HttpClientBuilder.create().setDefaultRequestConfig(config).build()
+    val response = client.execute(r)
+    println(s"get response with status${response.getStatusLine.getStatusCode}")
+    response.getEntity.getContent
+
+  }
+
+  def requestPage(url: String): String = {
+    val r = new HttpGet(url)
+    val timeout = 60; // seconds
+    val config = RequestConfig.custom()
+      .setConnectTimeout(timeout * 1000)
+      .setConnectionRequestTimeout(timeout * 1000)
+      .setSocketTimeout(timeout * 1000).build()
+    val client = HttpClientBuilder.create().setDefaultRequestConfig(config).build()
+    try {
+      var tries = 4
+      while (tries > 0) {
+        println(s"requesting ${r.getURI}")
+        try {
+          val response = client.execute(r)
+          println(s"get response with status${response.getStatusLine.getStatusCode}")
+          if (response.getStatusLine.getStatusCode > 400) {
+            tries -= 1
+          }
+          else
+            return IOUtils.toString(response.getEntity.getContent)
+        } catch {
+          case e: Throwable =>
+            println(s"Error on requesting ${r.getURI}")
+            e.printStackTrace()
+            tries -= 1
+        }
+      }
+      ""
+    } finally {
+      if (client != null)
+        client.close()
+    }
+  }
+
+
+  def downloadBaseLineUpdate(baselinePath: String, hdfsServerUri: String): Unit = {
+
+
+    val conf = new Configuration
+    conf.set("fs.defaultFS", hdfsServerUri)
+    val fs = FileSystem.get(conf)
+    val p = new Path(baselinePath)
+    val files = fs.listFiles(p, false)
+    var max_file = ""
+    while (files.hasNext) {
+      val c = files.next()
+      val data = c.getPath.toString
+      val fileName = data.substring(data.lastIndexOf("/") + 1)
+
+      if (fileName > max_file)
+        max_file = fileName
+    }
+
+    val files_to_download = requestBaseLineUpdatePage(max_file)
+
+    files_to_download.foreach { u =>
+      val hdfsWritePath: Path = new Path(s"$baselinePath/${u._1}")
+      val fsDataOutputStream: FSDataOutputStream = fs.create(hdfsWritePath, true)
+      val i = downloadBaselinePart(u._2)
+      IOUtils.copy(i, fsDataOutputStream)
+      println(s"Downloaded ${u._2} into $baselinePath/${u._1}")
+      fsDataOutputStream.close()
+    }
+
+  }
+
+
+  val pmArticleAggregator: Aggregator[(String, PMArticle), PMArticle, PMArticle] = new Aggregator[(String, PMArticle), PMArticle, PMArticle] with Serializable {
+    override def zero: PMArticle = new PMArticle
+
+    override def reduce(b: PMArticle, a: (String, PMArticle)): PMArticle = {
+      if (b != null && b.getPmid != null) b else a._2
+    }
+
+    override def merge(b1: PMArticle, b2: PMArticle): PMArticle = {
+      if (b1 != null && b1.getPmid != null) b1 else b2
+
+    }
+
+    override def finish(reduction: PMArticle): PMArticle = reduction
+
+    override def bufferEncoder: Encoder[PMArticle] = Encoders.kryo[PMArticle]
+
+    override def outputEncoder: Encoder[PMArticle] = Encoders.kryo[PMArticle]
+  }
+
+
+  def main(args: Array[String]): Unit = {
+    val conf: SparkConf = new SparkConf()
+    val log: Logger = LoggerFactory.getLogger(getClass)
+    val parser = new ArgumentApplicationParser(IOUtils.toString(SparkEBILinksToOaf.getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json")))
+    parser.parseArgument(args)
+    val isLookupUrl: String = parser.get("isLookupUrl")
+    log.info("isLookupUrl: {}", isLookupUrl)
+    val workingPath = parser.get("workingPath")
+    log.info("workingPath: {}", workingPath)
+
+    val targetPath = parser.get("targetPath")
+    log.info("targetPath: {}", targetPath)
+
+    val hdfsServerUri = parser.get("hdfsServerUri")
+    log.info("hdfsServerUri: {}", targetPath)
+
+    val skipUpdate = parser.get("skipUpdate")
+    log.info("skipUpdate: {}", skipUpdate)
+
+
+    val isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl)
+    val vocabularies = VocabularyGroup.loadVocsFromIS(isLookupService)
+    val spark: SparkSession =
+      SparkSession
+        .builder()
+        .config(conf)
+        .appName(SparkEBILinksToOaf.getClass.getSimpleName)
+        .master(parser.get("master")).getOrCreate()
+
+    val sc = spark.sparkContext
+    import spark.implicits._
+
+    implicit val PMEncoder: Encoder[PMArticle] = Encoders.kryo(classOf[PMArticle])
+    implicit val PMJEncoder: Encoder[PMJournal] = Encoders.kryo(classOf[PMJournal])
+    implicit val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
+    implicit val resultEncoder: Encoder[Result] = Encoders.kryo(classOf[Result])
+
+    if (!"true".equalsIgnoreCase(skipUpdate)) {
+      downloadBaseLineUpdate(s"$workingPath/baseline", hdfsServerUri)
+      val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline", 2000)
+      val ds: Dataset[PMArticle] = spark.createDataset(k.filter(i => i._1.endsWith(".gz")).flatMap(i => {
+        val xml = new XMLEventReader(Source.fromBytes(i._2.getBytes()))
+        new PMParser(xml)
+      }))
+      ds.map(p => (p.getPmid, p))(Encoders.tuple(Encoders.STRING, PMEncoder)).groupByKey(_._1)
+        .agg(pmArticleAggregator.toColumn)
+        .map(p => p._2).write.mode(SaveMode.Overwrite).save(s"$workingPath/baseline_dataset")
+    }
+
+    val exported_dataset = spark.read.load(s"$workingPath/baseline_dataset").as[PMArticle]
+    exported_dataset
+      .map(a => PubMedToOaf.convert(a, vocabularies)).as[Result]
+      .filter(p => p != null)
+      .write.mode(SaveMode.Overwrite).save(targetPath)
+  }
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/ebi/SparkDownloadEBILinks.scala

@@ -0,0 +1,118 @@
+package eu.dnetlib.dhp.sx.bio.ebi
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMAuthor, PMJournal}
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF.EBILinkItem
+import eu.dnetlib.dhp.sx.bio.pubmed.PMJournal
+import org.apache.commons.io.IOUtils
+import org.apache.http.client.config.RequestConfig
+import org.apache.http.client.methods.HttpGet
+import org.apache.http.impl.client.HttpClientBuilder
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.functions.max
+import org.apache.spark.sql._
+import org.slf4j.{Logger, LoggerFactory}
+
+object SparkDownloadEBILinks {
+
+  def createEBILinks(pmid: Long): EBILinkItem = {
+
+    val res = requestLinks(pmid)
+    if (res != null)
+      return EBILinkItem(pmid, res)
+    null
+  }
+
+  def requestPage(url: String): String = {
+    val r = new HttpGet(url)
+    val timeout = 60; // seconds
+    val config = RequestConfig.custom()
+      .setConnectTimeout(timeout * 1000)
+      .setConnectionRequestTimeout(timeout * 1000)
+      .setSocketTimeout(timeout * 1000).build()
+    val client = HttpClientBuilder.create().setDefaultRequestConfig(config).build()
+    try {
+      var tries = 4
+      while (tries > 0) {
+        println(s"requesting ${r.getURI}")
+        try {
+          val response = client.execute(r)
+          println(s"get response with status${response.getStatusLine.getStatusCode}")
+          if (response.getStatusLine.getStatusCode > 400) {
+            tries -= 1
+          }
+          else
+            return IOUtils.toString(response.getEntity.getContent)
+        } catch {
+          case e: Throwable =>
+            println(s"Error on requesting ${r.getURI}")
+            e.printStackTrace()
+            tries -= 1
+        }
+      }
+      ""
+    } finally {
+      if (client != null)
+        client.close()
+    }
+  }
+
+  def requestLinks(PMID: Long): String = {
+    requestPage(s"https://www.ebi.ac.uk/europepmc/webservices/rest/MED/$PMID/datalinks?format=json")
+
+  }
+
+  def main(args: Array[String]): Unit = {
+
+    val log: Logger = LoggerFactory.getLogger(getClass)
+    val MAX_ITEM_PER_PARTITION = 20000
+    val conf: SparkConf = new SparkConf()
+    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/bio/ebi/ebi_download_update.json")))
+    parser.parseArgument(args)
+    val spark: SparkSession =
+      SparkSession
+        .builder()
+        .config(conf)
+        .appName(SparkEBILinksToOaf.getClass.getSimpleName)
+        .master(parser.get("master")).getOrCreate()
+
+    import spark.implicits._
+
+    implicit val PMEncoder: Encoder[PMArticle] = Encoders.kryo(classOf[PMArticle])
+    implicit val PMJEncoder: Encoder[PMJournal] = Encoders.kryo(classOf[PMJournal])
+    implicit val PMAEncoder: Encoder[PMAuthor] = Encoders.kryo(classOf[PMAuthor])
+
+    val sourcePath = parser.get("sourcePath")
+    log.info(s"sourcePath  -> $sourcePath")
+    val workingPath = parser.get("workingPath")
+    log.info(s"workingPath  -> $workingPath")
+
+    log.info("Getting max pubmedId where the links have already requested")
+    val links: Dataset[EBILinkItem] = spark.read.load(s"$sourcePath/ebi_links_dataset").as[EBILinkItem]
+    val lastPMIDRequested = links.map(l => l.id).select(max("value")).first.getLong(0)
+
+    log.info("Retrieving PMID to request links")
+    val pubmed = spark.read.load(s"$sourcePath/baseline_dataset").as[PMArticle]
+    pubmed.map(p => p.getPmid.toLong).where(s"value > $lastPMIDRequested").write.mode(SaveMode.Overwrite).save(s"$workingPath/id_to_request")
+
+    val pmidToReq: Dataset[Long] = spark.read.load(s"$workingPath/id_to_request").as[Long]
+
+    val total = pmidToReq.count()
+
+    spark.createDataset(pmidToReq.rdd.repartition((total / MAX_ITEM_PER_PARTITION).toInt).map(pmid => createEBILinks(pmid)).filter(l => l != null)).write.mode(SaveMode.Overwrite).save(s"$workingPath/links_update")
+
+    val updates: Dataset[EBILinkItem] = spark.read.load(s"$workingPath/links_update").as[EBILinkItem]
+
+    links.union(updates).groupByKey(_.id)
+      .reduceGroups { (x, y) =>
+        if (x == null || x.links == null)
+          y
+        if (y == null || y.links == null)
+          x
+        if (x.links.length > y.links.length)
+          x
+        else
+          y
+      }.map(_._2).write.mode(SaveMode.Overwrite).save(s"$workingPath/links_final")
+  }
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/ebi/SparkEBILinksToOaf.scala

@@ -1,21 +1,22 @@
-package eu.dnetlib.dhp.sx.graph.ebi
+package eu.dnetlib.dhp.sx.bio.ebi
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.schema.oaf.Oaf
-import eu.dnetlib.dhp.sx.graph.bio
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF.EBILinkItem
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF.EBILinkItem
+import BioDBToOAF.EBILinkItem
+import eu.dnetlib.dhp.collection.CollectionUtils
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
+
 object SparkEBILinksToOaf {
 
   def main(args: Array[String]): Unit = {
     val log: Logger = LoggerFactory.getLogger(getClass)
     val conf: SparkConf = new SparkConf()
-    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/ebi/ebi_to_df_params.json")))
+    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/bio/ebi/ebi_to_df_params.json")))
     parser.parseArgument(args)
     val spark: SparkSession =
       SparkSession
@@ -24,24 +25,20 @@ object SparkEBILinksToOaf {
         .appName(SparkEBILinksToOaf.getClass.getSimpleName)
         .master(parser.get("master")).getOrCreate()
 
+
+    import spark.implicits._
     val sourcePath = parser.get("sourcePath")
     log.info(s"sourcePath  -> $sourcePath")
     val targetPath = parser.get("targetPath")
     log.info(s"targetPath  -> $targetPath")
+    implicit val PMEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
 
-    import spark.implicits._
-    implicit  val PMEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
-
-    val ebi_rdd:Dataset[EBILinkItem] = spark.createDataset(spark.sparkContext.textFile(sourcePath).map(s => BioDBToOAF.extractEBILinksFromDump(s))).as[EBILinkItem]
-
-    ebi_rdd.write.mode(SaveMode.Overwrite).save(s"${sourcePath}_dataset")
-
-    val ebLinks:Dataset[EBILinkItem] = spark.read.load(s"${sourcePath}_dataset").as[EBILinkItem].filter(l => l.links!= null)
+    val ebLinks: Dataset[EBILinkItem] = spark.read.load(sourcePath).as[EBILinkItem].filter(l => l.links != null && l.links.startsWith("{"))
 
-    ebLinks.flatMap(j =>BioDBToOAF.parse_ebi_links(j.links))
-      .repartition(4000)
+    ebLinks.flatMap(j => BioDBToOAF.parse_ebi_links(j.links))
       .filter(p => BioDBToOAF.EBITargetLinksFilter(p))
       .flatMap(p => BioDBToOAF.convertEBILinksToOaf(p))
+      .flatMap(i=> CollectionUtils.fixRelations(i)).filter(i => i != null)
       .write.mode(SaveMode.Overwrite).save(targetPath)
   }
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMArticle.java

@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 import java.util.ArrayList;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMAuthor.java

@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMGrant.java

@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 public class PMGrant {
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMJournal.java

@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 import java.io.Serializable;
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMParser.scala

@@ -1,4 +1,4 @@
-package eu.dnetlib.dhp.sx.graph.bio.pubmed
+package eu.dnetlib.dhp.sx.bio.pubmed
 
 import scala.xml.MetaData
 import scala.xml.pull.{EvElemEnd, EvElemStart, EvText, XMLEventReader}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PMSubject.java

@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.sx.graph.bio.pubmed;
+package eu.dnetlib.dhp.sx.bio.pubmed;
 
 public class PMSubject {
 	private String value;

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala

@@ -1,11 +1,12 @@
-package eu.dnetlib.dhp.sx.graph.bio.pubmed
+package eu.dnetlib.dhp.sx.bio.pubmed
+
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
 import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf._
 import eu.dnetlib.dhp.schema.oaf.utils.{GraphCleaningFunctions, IdentifierFactory, OafMapperUtils, PidType}
+import eu.dnetlib.dhp.schema.oaf._
+import scala.collection.JavaConverters._
 
 import java.util.regex.Pattern
-import scala.collection.JavaConverters._
 
 object PubMedToOaf {
 
@@ -15,7 +16,7 @@ object PubMedToOaf {
     "doi" -> "https://dx.doi.org/"
   )
 
-  def cleanDoi(doi:String):String = {
+  def cleanDoi(doi: String): String = {
 
     val regex = "^10.\\d{4,9}\\/[\\[\\]\\-\\<\\>._;()\\/:A-Z0-9]+$"
 
@@ -71,14 +72,14 @@ object PubMedToOaf {
     if (article.getPublicationTypes == null)
       return null
     val i = new Instance
-    var pidList: List[StructuredProperty] = List(OafMapperUtils.structuredProperty(article.getPmid, PidType.pmid.toString, PidType.pmid.toString, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, dataInfo))
+    val pidList: List[StructuredProperty] = List(OafMapperUtils.structuredProperty(article.getPmid, PidType.pmid.toString, PidType.pmid.toString, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, dataInfo))
     if (pidList == null)
       return null
 
-    var alternateIdentifier :StructuredProperty = null
+    var alternateIdentifier: StructuredProperty = null
     if (article.getDoi != null) {
       val normalizedPid = cleanDoi(article.getDoi)
-      if (normalizedPid!= null)
+      if (normalizedPid != null)
         alternateIdentifier = OafMapperUtils.structuredProperty(normalizedPid, PidType.doi.toString, PidType.doi.toString, ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, dataInfo)
     }
 
@@ -102,10 +103,10 @@ object PubMedToOaf {
       return result
     result.setDataInfo(dataInfo)
     i.setPid(pidList.asJava)
-    if (alternateIdentifier!= null)
+    if (alternateIdentifier != null)
       i.setAlternateIdentifier(List(alternateIdentifier).asJava)
     result.setInstance(List(i).asJava)
-    i.getPid.asScala.filter(p => "pmid".equalsIgnoreCase(p.getQualifier.getClassid)).map(p => p.getValue)(collection breakOut)
+    i.getPid.asScala.filter(p => "pmid".equalsIgnoreCase(p.getQualifier.getClassid)).map(p => p.getValue)(collection.breakOut)
     val urlLists: List[String] = pidList
       .map(s => (urlMap.getOrElse(s.getQualifier.getClassid, ""), s.getValue))
       .filter(t => t._1.nonEmpty)
@@ -136,7 +137,7 @@ object PubMedToOaf {
     }
 
 
-    val subjects: List[StructuredProperty] = article.getSubjects.asScala.map(s => OafMapperUtils.structuredProperty(s.getValue, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, dataInfo))(collection breakOut)
+    val subjects: List[StructuredProperty] = article.getSubjects.asScala.map(s => OafMapperUtils.structuredProperty(s.getValue, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, dataInfo))(collection.breakOut)
     if (subjects != null)
       result.setSubject(subjects.asJava)
 
@@ -148,7 +149,7 @@ object PubMedToOaf {
       author.setFullname(a.getFullName)
       author.setRank(index + 1)
       author
-    }(collection breakOut)
+    }(collection.breakOut)
 
 
     if (authors != null && authors.nonEmpty)

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/get_fos_parameters.json

@@ -0,0 +1,33 @@
+[
+  {
+    "paramName":"s",
+    "paramLongName":"sourcePath",
+    "paramDescription": "the path of the sequencial file to read",
+    "paramRequired": true
+  },
+  {
+    "paramName":"out",
+    "paramLongName":"outputPath",
+    "paramDescription": "the output path",
+    "paramRequired": true
+  },
+
+  {
+    "paramName": "ssm",
+    "paramLongName": "isSparkSessionManaged",
+    "paramDescription": "true if the spark session is managed, false otherwise",
+    "paramRequired": false
+  },
+  {
+    "paramName": "hnn",
+    "paramLongName": "hdfsNameNode",
+    "paramDescription": "the path used to store the HostedByMap",
+    "paramRequired": true
+  },
+  {
+    "paramName": "cfn",
+    "paramLongName": "classForName",
+    "paramDescription": "the path used to store the HostedByMap",
+    "paramRequired": true
+  }
+]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/config-default.xml

@@ -0,0 +1,30 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>hiveMetastoreUris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hiveJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
+    </property>
+    <property>
+        <name>hiveDbName</name>
+        <value>openaire</value>
+    </property>
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml

@@ -0,0 +1,174 @@
+
+<workflow-app name="UnresolvedEntities" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>fosPath</name>
+            <description>the input path of the resources to be extended</description>
+        </property>
+
+        <property>
+            <name>bipScorePath</name>
+            <description>the path where to find the bipFinder scores</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>the path where to store the actionset</description>
+        </property>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>oozieActionShareLibForSpark2</name>
+            <description>oozie action sharelib for spark 2.*</description>
+        </property>
+        <property>
+            <name>spark2ExtraListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+            <description>spark 2.* extra listeners classname</description>
+        </property>
+        <property>
+            <name>spark2SqlQueryExecutionListeners</name>
+            <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+            <description>spark 2.* sql query execution listeners classname</description>
+        </property>
+        <property>
+            <name>spark2YarnHistoryServerAddress</name>
+            <description>spark 2.* yarn history server address</description>
+        </property>
+        <property>
+            <name>spark2EventLogDir</name>
+            <description>spark 2.* event log dir location</description>
+        </property>
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+            <property>
+                <name>oozie.action.sharelib.for.spark</name>
+                <value>${oozieActionShareLibForSpark2}</value>
+            </property>
+
+        </configuration>
+    </global>
+    <start to="prepareInfo"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+
+    <fork name="prepareInfo">
+        <path start="prepareBip"/>
+        <path start="getFOS"/>
+    </fork>
+
+    <action name="prepareBip">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Produces the unresolved from bip finder!</name>
+            <class>eu.dnetlib.dhp.actionmanager.createunresolvedentities.PrepareBipFinder</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${bipScorePath}</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/prepared</arg>
+        </spark>
+        <ok to="join"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="getFOS">
+        <java>
+            <main-class>eu.dnetlib.dhp.actionmanager.createunresolvedentities.GetFOSData</main-class>
+            <arg>--hdfsNameNode</arg><arg>${nameNode}</arg>
+            <arg>--sourcePath</arg><arg>${fosPath}</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/input/fos</arg>
+            <arg>--classForName</arg><arg>eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.FOSDataModel</arg>
+        </java>
+        <ok to="prepareFos"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="prepareFos">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Produces the unresolved from FOS!</name>
+            <class>eu.dnetlib.dhp.actionmanager.createunresolvedentities.PrepareFOSSparkJob</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/input/fos</arg>
+            <arg>--outputPath</arg><arg>${workingDir}/prepared</arg>
+
+        </spark>
+        <ok to="join"/>
+        <error to="Kill"/>
+    </action>
+
+
+
+    <join name="join" to="produceUnresolved"/>
+
+    <action name="produceUnresolved">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Saves the result produced for bip and fos by grouping results with the same id</name>
+            <class>eu.dnetlib.dhp.actionmanager.createunresolvedentities.SparkSaveUnresolved</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${workingDir}/prepared</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/prepare_parameters.json

@@ -0,0 +1,20 @@
+[
+  {
+    "paramName": "issm",
+    "paramLongName": "isSparkSessionManaged",
+    "paramDescription": "when true will stop SparkSession after job execution",
+    "paramRequired": false
+  },
+  {
+    "paramName": "sp",
+    "paramLongName": "sourcePath",
+    "paramDescription": "the URL from where to get the programme file",
+    "paramRequired": true
+  },
+  {
+    "paramName": "o",
+    "paramLongName": "outputPath",
+    "paramDescription": "the path of the new ActionSet",
+    "paramRequired": true
+  }
+]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/produce_unresolved_parameters.json

@@ -0,0 +1,20 @@
+[
+  {
+    "paramName": "issm",
+    "paramLongName": "isSparkSessionManaged",
+    "paramDescription": "when true will stop SparkSession after job execution",
+    "paramRequired": false
+  },
+  {
+    "paramName": "sp",
+    "paramLongName": "sourcePath",
+    "paramDescription": "the URL from where to get the programme file",
+    "paramRequired": true
+  },
+  {
+    "paramName": "o",
+    "paramLongName": "outputPath",
+    "paramDescription": "the path of the new ActionSet",
+    "paramRequired": true
+  }
+]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/oozie_app/workflow.xml

@@ -1,81 +0,0 @@
-<workflow-app name="Import_Datacite_and_transform_to_OAF" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>mainPath</name>
-            <description>the working path of Datacite stores</description>
-        </property>
-        <property>
-            <name>isLookupUrl</name>
-            <description>The IS lookUp service endopoint</description>
-        </property>
-        <property>
-            <name>blocksize</name>
-            <value>100</value>
-            <description>The request block size</description>
-        </property>
-
-    </parameters>
-
-    <start to="ImportDatacite"/>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-
-    <action name="ImportDatacite">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>ImportDatacite</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.ImportDatacite</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--targetPath</arg><arg>${mainPath}/datacite_update</arg>
-            <arg>--dataciteDumpPath</arg><arg>${mainPath}/datacite_dump</arg>
-            <arg>--namenode</arg><arg>${nameNode}</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-            <arg>--blocksize</arg><arg>${blocksize}</arg>
-        </spark>
-        <ok to="TransformJob"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="TransformJob">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>TransformJob</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.GenerateDataciteDatasetSpark</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${mainPath}/datacite_dump</arg>
-            <arg>--targetPath</arg><arg>${mainPath}/datacite_oaf</arg>
-            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
-            <arg>--exportLinks</arg><arg>false</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-    <end name="End"/>
-</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/scholix/oozie_app/workflow.xml

@@ -1,84 +0,0 @@
-<workflow-app name="Generate_Datacite_and_Crossref_dump_for_Scholexplorer" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>datacitePath</name>
-            <description>the path of Datacite spark dataset</description>
-        </property>
-        <property>
-            <name>isLookupUrl</name>
-            <description>The IS lookUp service endopoint</description>
-        </property>
-        <property>
-            <name>crossrefPath</name>
-            <description>the path of Crossref spark dataset</description>
-        </property>
-
-        <property>
-            <name>targetPath</name>
-            <description>the path of Crossref spark dataset</description>
-        </property>
-
-    </parameters>
-
-    <start to="ImportDatacite"/>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-
-    <action name="ImportDatacite">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>ImportDatacite</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.GenerateDataciteDatasetSpark</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${datacitePath}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/datacite_oaf</arg>
-            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
-            <arg>--exportLinks</arg><arg>true</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="FilterCrossrefEntities"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="FilterCrossrefEntities">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>FilterCrossrefEntities</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.FilterCrossrefEntitiesSpark</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${crossrefPath}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/crossref_oaf</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-    <end name="End"/>
-</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/collection/oozie_app/workflow.xml

@@ -1,46 +1,52 @@
-<workflow-app name="Datacite_to_ActionSet_Workflow" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="Collect_Datacite" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
-            <name>sourcePath</name>
+            <name>mainPath</name>
             <description>the working path of Datacite stores</description>
         </property>
         <property>
-            <name>outputPath</name>
-            <description>the path of Datacite ActionSet</description>
+            <name>isLookupUrl</name>
+            <description>The IS lookUp service endopoint</description>
         </property>
+        <property>
+            <name>blocksize</name>
+            <value>100</value>
+            <description>The request block size</description>
+        </property>
+
     </parameters>
 
-    <start to="ExportDataset"/>
+    <start to="ImportDatacite"/>
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
 
-    <action name="ExportDataset">
+    <action name="ImportDatacite">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn-cluster</master>
             <mode>cluster</mode>
-            <name>ExportDataset</name>
-            <class>eu.dnetlib.dhp.actionmanager.datacite.ExportActionSetJobNode</class>
+            <name>ImportDatacite</name>
+            <class>eu.dnetlib.dhp.datacite.ImportDatacite</class>
             <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
-            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
-            <arg>--targetPath</arg><arg>${outputPath}</arg>
+            <arg>--targetPath</arg><arg>${mainPath}/datacite_update</arg>
+            <arg>--dataciteDumpPath</arg><arg>${mainPath}/datacite_dump</arg>
+            <arg>--namenode</arg><arg>${nameNode}</arg>
             <arg>--master</arg><arg>yarn-cluster</arg>
+            <arg>--blocksize</arg><arg>${blocksize}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
     </action>
-
     <end name="End"/>
 </workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/generate_dataset_params.json

@@ -7,8 +7,8 @@
   },
 
   {
-    "paramName": "t",
-    "paramLongName": "targetPath",
+    "paramName": "mo",
+    "paramLongName": "mdstoreOutputVersion",
     "paramDescription": "the target mdstore path",
     "paramRequired": true
   },

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/transformation/oozie_app/workflow.xml

@@ -0,0 +1,126 @@
+<workflow-app name="transform_Datacite" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>mainPath</name>
+            <description>the working path of Datacite stores</description>
+        </property>
+        <property>
+            <name>isLookupUrl</name>
+            <description>The IS lookUp service endopoint</description>
+        </property>
+        <property>
+            <name>mdStoreOutputId</name>
+            <description>the identifier of the cleaned MDStore</description>
+        </property>
+        <property>
+            <name>mdStoreManagerURI</name>
+            <description>the path of the cleaned mdstore</description>
+        </property>
+    </parameters>
+
+    <start to="StartTransaction"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <action name="StartTransaction">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>NEW_VERSION</arg>
+            <arg>--mdStoreID</arg><arg>${mdStoreOutputId}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            <capture-output/>
+        </java>
+        <ok to="TransformJob"/>
+        <error to="EndReadRollBack"/>
+    </action>
+
+    <action name="TransformJob">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>TransformJob</name>
+            <class>eu.dnetlib.dhp.datacite.GenerateDataciteDatasetSpark</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.sql.shuffle.partitions=3840
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${mainPath}/datacite_dump</arg>
+            <arg>--mdstoreOutputVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
+            <arg>--exportLinks</arg><arg>true</arg>
+            <arg>--master</arg><arg>yarn-cluster</arg>
+        </spark>
+        <ok to="CommitVersion"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="CommitVersion">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>COMMIT</arg>
+            <arg>--namenode</arg><arg>${nameNode}</arg>
+            <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+        </java>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="EndReadRollBack">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>READ_UNLOCK</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            <arg>--readMDStoreId</arg><arg>${wf:actionData('BeginRead')['mdStoreReadLockVersion']}</arg>
+            <capture-output/>
+        </java>
+        <ok to="RollBack"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="RollBack">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>ROLLBACK</arg>
+            <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+        </java>
+        <ok to="Kill"/>
+        <error to="Kill"/>
+    </action>
+
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/db/oozie_app/workflow.xml

@@ -1,42 +1,33 @@
-<workflow-app name="Create EBI Dataset" xmlns="uri:oozie:workflow:0.5">
+    <workflow-app name="Transform_BioEntity_Workflow" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
             <name>sourcePath</name>
-            <description>the Working Path</description>
+            <description>the PDB Database Working Path</description>
         </property>
         <property>
-            <name>workingPath</name>
-            <description>the Working Path</description>
-        </property>
-        <property>
-            <name>sparkDriverMemory</name>
-            <description>memory for driver process</description>
-        </property>
-        <property>
-            <name>sparkExecutorMemory</name>
-            <description>memory for individual executor</description>
+            <name>database</name>
+            <description>the PDB Database Working Path</description>
         </property>
+
         <property>
-            <name>sparkExecutorCores</name>
-            <description>number of cores used by single executor</description>
+            <name>targetPath</name>
+            <description>the Target Working dir path</description>
         </property>
     </parameters>
 
-    <start to="DownloadEBILinks"/>
-
+    <start to="ConvertDB"/>
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
-
-    <action name="DownloadEBILinks">
+    <action name="ConvertDB">
         <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
+            <master>yarn</master>
             <mode>cluster</mode>
-            <name>Incremental Download EBI Links</name>
-            <class>eu.dnetlib.dhp.sx.graph.ebi.SparkDownloadEBILinks</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <name>Convert Bio DB to OAF Dataset</name>
+            <class>eu.dnetlib.dhp.sx.bio.SparkTransformBioDatabaseToOAF</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
@@ -47,13 +38,14 @@
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
-            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
-            <arg>--workingPath</arg><arg>${workingPath}</arg>
             <arg>--master</arg><arg>yarn</arg>
+            <arg>--dbPath</arg><arg>${sourcePath}</arg>
+            <arg>--database</arg><arg>${database}</arg>
+            <arg>--targetPath</arg><arg>${targetPath}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
     </action>
-
     <end name="End"/>
+
 </workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json

@@ -0,0 +1,8 @@
+[
+  {"paramName":"mt",  "paramLongName":"master",         "paramDescription": "should be local or yarn",                  "paramRequired": true},
+  {"paramName":"i",   "paramLongName":"isLookupUrl",    "paramDescription": "isLookupUrl",                              "paramRequired": true},
+  {"paramName":"w",   "paramLongName":"workingPath",    "paramDescription": "the path of the sequencial file to read",  "paramRequired": true},
+  {"paramName":"t",   "paramLongName":"targetPath",     "paramDescription": "the oaf path ",                            "paramRequired": true},
+  {"paramName":"s",   "paramLongName":"skipUpdate",     "paramDescription": "skip update ",                             "paramRequired": false},
+  {"paramName":"h",   "paramLongName":"hdfsServerUri",  "paramDescription": "the  working path ",                       "paramRequired": true}
+]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/ebi_download_update.json

@@ -0,0 +1,5 @@
+[
+  {"paramName":"mt",  "paramLongName":"master",         "paramDescription": "should be local or yarn",    "paramRequired": true},
+  {"paramName":"s",   "paramLongName":"sourcePath",     "paramDescription": "the source Path",            "paramRequired": true},
+  {"paramName":"w",   "paramLongName":"workingPath",    "paramDescription": "the  working path ",         "paramRequired": true}
+]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/oozie_app/workflow.xml

@@ -0,0 +1,105 @@
+<workflow-app name="Create EBI Dataset" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>sourcePath</name>
+            <description>the Working Path</description>
+        </property>
+        <property>
+            <name>workingPath</name>
+            <description>the Working Path</description>
+        </property>
+        <property>
+            <name>targetPath</name>
+            <description>the OAF MDStore Path</description>
+        </property>
+        <property>
+            <name>sparkDriverMemory</name>
+            <description>memory for driver process</description>
+        </property>
+        <property>
+            <name>sparkExecutorMemory</name>
+            <description>memory for individual executor</description>
+        </property>
+        <property>
+            <name>sparkExecutorCores</name>
+            <description>number of cores used by single executor</description>
+        </property>
+        <property>
+            <name>resumeFrom</name>
+            <value>DownloadEBILinks</value>
+            <description>node to start</description>
+        </property>
+    </parameters>
+
+    <start to="resume_from"/>
+
+    <decision name="resume_from">
+        <switch>
+            <case to="DownloadEBILinks">${wf:conf('resumeFrom') eq 'DownloadEBILinks'}</case>
+            <case to="CreateEBIDataSet">${wf:conf('resumeFrom') eq 'CreateEBIDataSet'}</case>
+            <default to="DownloadEBILinks"/>
+        </switch>
+    </decision>
+
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+
+    <action name="DownloadEBILinks">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>Incremental Download EBI Links</name>
+            <class>eu.dnetlib.dhp.sx.bio.ebi.SparkDownloadEBILinks</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.shuffle.partitions=2000
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+            <arg>--workingPath</arg><arg>${workingPath}</arg>
+            <arg>--master</arg><arg>yarn</arg>
+        </spark>
+        <ok to="OverrideFolders"/>
+        <error to="Kill"/>
+    </action>
+    <action name="OverrideFolders">
+        <fs>
+            <delete path="${sourcePath}/ebi_links_dataset_old"/>
+            <move source="${sourcePath}/ebi_links_dataset" target="${sourcePath}/ebi_links_dataset_old"/>
+            <move source="${workingPath}/links_final" target="${sourcePath}/ebi_links_dataset"/>
+        </fs>
+        <ok to="CreateEBIDataSet"/>
+        <error to="Kill"/>
+    </action>
+    <action name="CreateEBIDataSet">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>Create OAF DataSet</name>
+            <class>eu.dnetlib.dhp.sx.bio.ebi.SparkEBILinksToOaf</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.sql.shuffle.partitions=2000
+                ${sparkExtraOPT}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}/ebi_links_dataset</arg>
+            <arg>--targetPath</arg><arg>${targetPath}</arg>
+            <arg>--master</arg><arg>yarn</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/pubmed/oozie_app/workflow.xml

@@ -1,16 +1,21 @@
-    <workflow-app name="Transform_Pubmed_Workflow" xmlns="uri:oozie:workflow:0.5">
+    <workflow-app name="Download_Transform_Pubmed_Workflow" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
             <name>baselineWorkingPath</name>
             <description>the Baseline Working Path</description>
         </property>
+        <property>
+            <name>isLookupUrl</name>
+            <description>The IS lookUp service endopoint</description>
+        </property>
         <property>
             <name>targetPath</name>
-            <description>the Target Path</description>
+            <description>The target path</description>
         </property>
         <property>
-            <name>isLookupUrl</name>
-            <description>The IS lookUp service endopoint</description>
+            <name>skipUpdate</name>
+            <value>false</value>
+            <description>The request block size</description>
         </property>
     </parameters>
 
@@ -24,9 +29,9 @@
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
             <mode>cluster</mode>
-            <name>Convert Baseline to Dataset</name>
-            <class>eu.dnetlib.dhp.sx.graph.ebi.SparkCreateBaselineDataFrame</class>
-            <jar>dhp-graph-mapper-${projectVersion}.jar</jar>
+            <name>Convert Baseline to OAF Dataset</name>
+            <class>eu.dnetlib.dhp.sx.bio.ebi.SparkCreateBaselineDataFrame</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
             <spark-opts>
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
@@ -41,6 +46,8 @@
             <arg>--targetPath</arg><arg>${targetPath}</arg>
             <arg>--master</arg><arg>yarn</arg>
             <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
+            <arg>--hdfsServerUri</arg><arg>${nameNode}</arg>
+            <arg>--skipUpdate</arg><arg>${skipUpdate}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java

@@ -0,0 +1,250 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.FOSDataModel;
+import eu.dnetlib.dhp.common.collection.CollectorException;
+import eu.dnetlib.dhp.schema.oaf.Result;
+
+public class PrepareTest {
+
+	private static final Logger log = LoggerFactory.getLogger(ProduceTest.class);
+
+	private static Path workingDir;
+	private static SparkSession spark;
+	private static LocalFileSystem fs;
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	@BeforeAll
+	public static void beforeAll() throws IOException {
+		workingDir = Files.createTempDirectory(PrepareTest.class.getSimpleName());
+
+		fs = FileSystem.getLocal(new Configuration());
+		log.info("using work dir {}", workingDir);
+
+		SparkConf conf = new SparkConf();
+		conf.setAppName(ProduceTest.class.getSimpleName());
+
+		conf.setMaster("local[*]");
+		conf.set("spark.driver.host", "localhost");
+		conf.set("hive.metastore.local", "true");
+		conf.set("spark.ui.enabled", "false");
+		conf.set("spark.sql.warehouse.dir", workingDir.toString());
+		conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
+
+		spark = SparkSession
+			.builder()
+			.appName(PrepareTest.class.getSimpleName())
+			.config(conf)
+			.getOrCreate();
+	}
+
+	@AfterAll
+	public static void afterAll() throws IOException {
+		FileUtils.deleteDirectory(workingDir.toFile());
+		spark.stop();
+	}
+
+	@Test
+	void bipPrepareTest() throws Exception {
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json")
+			.getPath();
+
+		PrepareBipFinder
+			.main(
+				new String[] {
+					"--isSparkSessionManaged", Boolean.FALSE.toString(),
+					"--sourcePath", sourcePath,
+					"--outputPath", workingDir.toString() + "/work"
+
+				});
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<Result> tmp = sc
+			.textFile(workingDir.toString() + "/work/bip")
+			.map(item -> OBJECT_MAPPER.readValue(item, Result.class));
+
+		Assertions.assertEquals(86, tmp.count());
+
+		String doi1 = "unresolved::10.0000/096020199389707::doi";
+
+		Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi1)).count());
+		Assertions.assertEquals(3, tmp.filter(r -> r.getId().equals(doi1)).collect().get(0).getMeasures().size());
+		Assertions
+			.assertEquals(
+				"6.34596412687e-09", tmp
+					.filter(r -> r.getId().equals(doi1))
+					.collect()
+					.get(0)
+					.getMeasures()
+					.stream()
+					.filter(sl -> sl.getId().equals("influence"))
+					.collect(Collectors.toList())
+					.get(0)
+					.getUnit()
+					.get(0)
+					.getValue());
+		Assertions
+			.assertEquals(
+				"0.641151896994", tmp
+					.filter(r -> r.getId().equals(doi1))
+					.collect()
+					.get(0)
+					.getMeasures()
+					.stream()
+					.filter(sl -> sl.getId().equals("popularity_alt"))
+					.collect(Collectors.toList())
+					.get(0)
+					.getUnit()
+					.get(0)
+					.getValue());
+		Assertions
+			.assertEquals(
+				"2.33375102921e-09", tmp
+					.filter(r -> r.getId().equals(doi1))
+					.collect()
+					.get(0)
+					.getMeasures()
+					.stream()
+					.filter(sl -> sl.getId().equals("popularity"))
+					.collect(Collectors.toList())
+					.get(0)
+					.getUnit()
+					.get(0)
+					.getValue());
+
+	}
+
+	@Test
+	void getFOSFileTest() throws IOException, ClassNotFoundException {
+
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/h2020_fos_sbs.csv")
+			.getPath();
+		final String outputPath = workingDir.toString() + "/fos.json";
+
+		new GetFOSData()
+			.doRewrite(
+				sourcePath, outputPath, "eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.FOSDataModel",
+				'\t', fs);
+
+		BufferedReader in = new BufferedReader(
+			new InputStreamReader(fs.open(new org.apache.hadoop.fs.Path(outputPath))));
+
+		String line;
+		int count = 0;
+		while ((line = in.readLine()) != null) {
+			FOSDataModel fos = new ObjectMapper().readValue(line, FOSDataModel.class);
+
+			System.out.println(new ObjectMapper().writeValueAsString(fos));
+			count += 1;
+		}
+
+		assertEquals(38, count);
+
+	}
+
+	@Test
+	void fosPrepareTest() throws Exception {
+		final String sourcePath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos.json")
+			.getPath();
+
+		PrepareFOSSparkJob
+			.main(
+				new String[] {
+					"--isSparkSessionManaged", Boolean.FALSE.toString(),
+					"--sourcePath", sourcePath,
+
+					"-outputPath", workingDir.toString() + "/work"
+
+				});
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<Result> tmp = sc
+			.textFile(workingDir.toString() + "/work/fos")
+			.map(item -> OBJECT_MAPPER.readValue(item, Result.class));
+
+		String doi1 = "unresolved::10.3390/s18072310::doi";
+
+		assertEquals(50, tmp.count());
+		assertEquals(1, tmp.filter(row -> row.getId().equals(doi1)).count());
+		assertTrue(
+			tmp
+				.filter(r -> r.getId().equals(doi1))
+				.flatMap(r -> r.getSubject().iterator())
+				.map(sbj -> sbj.getValue())
+				.collect()
+				.contains("engineering and technology"));
+
+		assertTrue(
+			tmp
+				.filter(r -> r.getId().equals(doi1))
+				.flatMap(r -> r.getSubject().iterator())
+				.map(sbj -> sbj.getValue())
+				.collect()
+				.contains("nano-technology"));
+		assertTrue(
+			tmp
+				.filter(r -> r.getId().equals(doi1))
+				.flatMap(r -> r.getSubject().iterator())
+				.map(sbj -> sbj.getValue())
+				.collect()
+				.contains("nanoscience & nanotechnology"));
+
+		String doi = "unresolved::10.1111/1365-2656.12831::doi";
+		assertEquals(1, tmp.filter(row -> row.getId().equals(doi)).count());
+		assertTrue(
+			tmp
+				.filter(r -> r.getId().equals(doi))
+				.flatMap(r -> r.getSubject().iterator())
+				.map(sbj -> sbj.getValue())
+				.collect()
+				.contains("psychology and cognitive sciences"));
+
+		assertTrue(
+			tmp
+				.filter(r -> r.getId().equals(doi))
+				.flatMap(r -> r.getSubject().iterator())
+				.map(sbj -> sbj.getValue())
+				.collect()
+				.contains("social sciences"));
+		assertFalse(
+			tmp
+				.filter(r -> r.getId().equals(doi))
+				.flatMap(r -> r.getSubject().iterator())
+				.map(sbj -> sbj.getValue())
+				.collect()
+				.contains("NULL"));
+
+	}
+
+}

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java

@@ -0,0 +1,234 @@
+
+package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.*;
+
+public class ProduceTest {
+	private static final Logger log = LoggerFactory.getLogger(ProduceTest.class);
+
+	private static Path workingDir;
+	private static SparkSession spark;
+	private static LocalFileSystem fs;
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+	private static final String ID_PREFIX = "50|doi_________";
+
+	@BeforeAll
+	public static void beforeAll() throws IOException {
+		workingDir = Files.createTempDirectory(ProduceTest.class.getSimpleName());
+
+		fs = FileSystem.getLocal(new Configuration());
+		log.info("using work dir {}", workingDir);
+
+		SparkConf conf = new SparkConf();
+		conf.setAppName(ProduceTest.class.getSimpleName());
+
+		conf.setMaster("local[*]");
+		conf.set("spark.driver.host", "localhost");
+		conf.set("hive.metastore.local", "true");
+		conf.set("spark.ui.enabled", "false");
+		conf.set("spark.sql.warehouse.dir", workingDir.toString());
+		conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
+
+		spark = SparkSession
+			.builder()
+			.appName(ProduceTest.class.getSimpleName())
+			.config(conf)
+			.getOrCreate();
+	}
+
+	@AfterAll
+	public static void afterAll() throws IOException {
+		FileUtils.deleteDirectory(workingDir.toFile());
+		spark.stop();
+	}
+
+	@Test
+	void produceTest() throws Exception {
+
+		final String bipPath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json")
+			.getPath();
+
+		PrepareBipFinder
+			.main(
+				new String[] {
+					"--isSparkSessionManaged", Boolean.FALSE.toString(),
+					"--sourcePath", bipPath,
+					"--outputPath", workingDir.toString() + "/work"
+
+				});
+		final String fosPath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos.json")
+			.getPath();
+
+		PrepareFOSSparkJob
+			.main(
+				new String[] {
+					"--isSparkSessionManaged", Boolean.FALSE.toString(),
+					"--sourcePath", fosPath,
+					"-outputPath", workingDir.toString() + "/work"
+				});
+
+		SparkSaveUnresolved.main(new String[] {
+			"--isSparkSessionManaged", Boolean.FALSE.toString(),
+			"--sourcePath", workingDir.toString() + "/work",
+
+			"-outputPath", workingDir.toString() + "/unresolved"
+
+		});
+
+		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		JavaRDD<Result> tmp = sc
+			.textFile(workingDir.toString() + "/unresolved")
+			.map(item -> OBJECT_MAPPER.readValue(item, Result.class));
+
+		Assertions.assertEquals(135, tmp.count());
+
+		Assertions.assertEquals(1, tmp.filter(row -> row.getId().equals("unresolved::10.3390/s18072310::doi")).count());
+
+		Assertions
+			.assertEquals(
+				3, tmp
+					.filter(row -> row.getId().equals("unresolved::10.3390/s18072310::doi"))
+					.collect()
+					.get(0)
+					.getSubject()
+					.size());
+
+		Assertions
+			.assertEquals(
+				3, tmp
+					.filter(row -> row.getId().equals("unresolved::10.3390/s18072310::doi"))
+					.collect()
+					.get(0)
+					.getMeasures()
+					.size());
+
+		List<StructuredProperty> sbjs = tmp
+			.filter(row -> row.getId().equals("unresolved::10.3390/s18072310::doi"))
+			.flatMap(row -> row.getSubject().iterator())
+			.collect();
+
+		sbjs.forEach(sbj -> Assertions.assertEquals("FOS", sbj.getQualifier().getClassid()));
+		sbjs
+			.forEach(
+				sbj -> Assertions
+					.assertEquals(
+						"Fields of Science and Technology classification", sbj.getQualifier().getClassname()));
+		sbjs
+			.forEach(
+				sbj -> Assertions
+					.assertEquals(ModelConstants.DNET_SUBJECT_TYPOLOGIES, sbj.getQualifier().getSchemeid()));
+		sbjs
+			.forEach(
+				sbj -> Assertions
+					.assertEquals(ModelConstants.DNET_SUBJECT_TYPOLOGIES, sbj.getQualifier().getSchemename()));
+
+		sbjs.forEach(sbj -> Assertions.assertEquals(false, sbj.getDataInfo().getDeletedbyinference()));
+		sbjs.forEach(sbj -> Assertions.assertEquals(true, sbj.getDataInfo().getInferred()));
+		sbjs.forEach(sbj -> Assertions.assertEquals(false, sbj.getDataInfo().getInvisible()));
+		sbjs.forEach(sbj -> Assertions.assertEquals("", sbj.getDataInfo().getTrust()));
+		sbjs.forEach(sbj -> Assertions.assertEquals("update", sbj.getDataInfo().getInferenceprovenance()));
+		sbjs
+			.forEach(
+				sbj -> Assertions.assertEquals("subject:fos", sbj.getDataInfo().getProvenanceaction().getClassid()));
+		sbjs
+			.forEach(
+				sbj -> Assertions
+					.assertEquals("Inferred by OpenAIRE", sbj.getDataInfo().getProvenanceaction().getClassname()));
+		sbjs
+			.forEach(
+				sbj -> Assertions
+					.assertEquals(
+						ModelConstants.DNET_PROVENANCE_ACTIONS, sbj.getDataInfo().getProvenanceaction().getSchemeid()));
+		sbjs
+			.forEach(
+				sbj -> Assertions
+					.assertEquals(
+						ModelConstants.DNET_PROVENANCE_ACTIONS,
+						sbj.getDataInfo().getProvenanceaction().getSchemename()));
+
+		sbjs.stream().anyMatch(sbj -> sbj.getValue().equals("engineering and technology"));
+		sbjs.stream().anyMatch(sbj -> sbj.getValue().equals("nano-technology"));
+		sbjs.stream().anyMatch(sbj -> sbj.getValue().equals("nanoscience & nanotechnology"));
+
+		List<Measure> measures = tmp
+			.filter(row -> row.getId().equals("unresolved::10.3390/s18072310::doi"))
+			.flatMap(row -> row.getMeasures().iterator())
+			.collect();
+		Assertions
+			.assertEquals(
+				"7.5597134689e-09", measures
+					.stream()
+					.filter(mes -> mes.getId().equals("influence"))
+					.collect(Collectors.toList())
+					.get(0)
+					.getUnit()
+					.get(0)
+					.getValue());
+
+		Assertions
+			.assertEquals(
+				"4.903880192", measures
+					.stream()
+					.filter(mes -> mes.getId().equals("popularity_alt"))
+					.collect(Collectors.toList())
+					.get(0)
+					.getUnit()
+					.get(0)
+					.getValue());
+
+		Assertions
+			.assertEquals(
+				"1.17977512835e-08", measures
+					.stream()
+					.filter(mes -> mes.getId().equals("popularity"))
+					.collect(Collectors.toList())
+					.get(0)
+					.getUnit()
+					.get(0)
+					.getValue());
+
+		Assertions
+			.assertEquals(
+				49, tmp
+					.filter(row -> !row.getId().equals("unresolved::10.3390/s18072310::doi"))
+					.filter(row -> row.getSubject() != null)
+					.count());
+
+		Assertions
+			.assertEquals(
+				85,
+				tmp
+					.filter(row -> !row.getId().equals("unresolved::10.3390/s18072310::doi"))
+					.filter(r -> r.getMeasures() != null)
+					.count());
+
+	}
+
+}

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/datacite/DataciteToOAFTest.scala

@@ -1,8 +1,7 @@
-package eu.dnetlib.dhp.actionmanager.datacite
+package eu.dnetlib.dhp.datacite
 
 
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.databind.SerializationFeature
+import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
 import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest
 import eu.dnetlib.dhp.schema.oaf.Oaf
 import org.junit.jupiter.api.extension.ExtendWith

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/sx/bio/BioScholixTest.scala

@@ -1,12 +1,10 @@
-package eu.dnetlib.dhp.sx.graph.bio.pubmed
+package eu.dnetlib.dhp.sx.bio
 
 import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
-import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.utils.{CleaningFunctions, OafMapperUtils, PidType}
+import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest
 import eu.dnetlib.dhp.schema.oaf.{Oaf, Relation, Result}
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF.ScholixResolved
-import eu.dnetlib.dhp.sx.graph.bio.BioDBToOAF
-import eu.dnetlib.dhp.sx.graph.bio.pubmed.PubMedToOaf.dataInfo
+import eu.dnetlib.dhp.sx.bio.BioDBToOAF.ScholixResolved
+import eu.dnetlib.dhp.sx.bio.pubmed.{PMArticle, PMParser, PubMedToOaf}
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.parse
@@ -50,9 +48,11 @@ class BioScholixTest extends AbstractVocabularyTest{
   }
 
 
+ 
+
   @Test
   def testEBIData() = {
-    val inputXML = Source.fromInputStream(getClass.getResourceAsStream("pubmed.xml")).mkString
+    val inputXML = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")).mkString
     val xml = new XMLEventReader(Source.fromBytes(inputXML.getBytes()))
     new PMParser(xml).foreach(s =>println(mapper.writeValueAsString(s)))
   }
@@ -62,7 +62,7 @@ class BioScholixTest extends AbstractVocabularyTest{
   def testPubmedToOaf(): Unit = {
     assertNotNull(vocabularies)
     assertTrue(vocabularies.vocabularyExists("dnet:publication_resource"))
-    val records:String =Source.fromInputStream(getClass.getResourceAsStream("pubmed_dump")).mkString
+    val records:String =Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed_dump")).mkString
     val r:List[Oaf] = records.lines.toList.map(s=>mapper.readValue(s, classOf[PMArticle])).map(a => PubMedToOaf.convert(a, vocabularies))
     assertEquals(10, r.size)
     assertTrue(r.map(p => p.asInstanceOf[Result]).flatMap(p => p.getInstance().asScala.map(i => i.getInstancetype.getClassid)).exists(p => "0037".equalsIgnoreCase(p)))

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json

@@ -0,0 +1,86 @@
+{"10.3390/s18072310": [{"id": "influence", "unit": [{"value": "7.5597134689e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "4.903880192", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "1.17977512835e-08", "key": "score"}]}]}
+{"10.0000/096020199389707": [{"id": "influence", "unit": [{"value": "6.34596412687e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.641151896994", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "2.33375102921e-09", "key": "score"}]}]}
+{"10.00000/jpmc.2017.106": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
+{"10.0000/9781845416881": [{"id": "influence", "unit": [{"value": "5.96492048955e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "1.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "1.12641925838e-08", "key": "score"}]}]}
+{"10.0000/anziamj.v0i0.266": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.76260934675e-10", "key": "score"}]}]}
+{"10.0000/anziamj.v48i0.79": [{"id": "influence", "unit": [{"value": "6.93311506443e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.002176782336", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "1.7668105708e-09", "key": "score"}]}]}
+{"10.0000/anziamj.v50i0.1472": [{"id": "influence", "unit": [{"value": "6.26777280882e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.406656", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.39745193285e-09", "key": "score"}]}]}
+{"10.0000/cja5553": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/czastest.16": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/czastest.17": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
+{"10.0000/czastest.18": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
+{"10.0000/czastest.20": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/czastest.21": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
+{"10.0000/czastest.28": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "3.47956715615e-09", "key": "score"}]}]}
+{"10.0000/czastest.60": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/czt.2019.1.2.15": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v4i02.36": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v4i02.37": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v4i02.38": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v5i01.32": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v6i01.24": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v6i01.27": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v6i02.41": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v6i02.44": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i01.40": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.01810569717e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i01.42": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i01.47": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i01.51": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i01.52": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i02.86": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i02.88": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v7i02.91": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v8i01.129": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v8i01.180": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
+{"10.0000/geoekonomi.v8i01.87": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "4.65008652949e-09", "key": "score"}]}]}
+{"10.0000/hbv2004w010": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hbv2101w001": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2101w002": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2101w003": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2101w004": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2101w005": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2101w006": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2101w007": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2102w001": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hbv2102w010": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "9.88840807598e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v1i1.13207": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v1i1.13208": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v1i1.13209": [{"id": "influence", "unit": [{"value": "6.32078461509e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "1.6", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.3168486939e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v1i1.13210": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v1i1.13211": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v1i1.13212": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "5.39172290649e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v1i2.13231": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i2.28782": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i2.28783": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i2.28784": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i2.28786": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i2.28787": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i2.28788": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i3.28234": [{"id": "influence", "unit": [{"value": "6.40470414877e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.6", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.89465099068e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i3.28236": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i3.28238": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i3.28239": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i3.28242": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v2i3.28243": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "6.26204125721e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i4.38186": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i4.38187": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i4.38190": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i4.38207": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i4.38209": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i5.41163": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i5.41166": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i5.41167": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i5.41168": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "7.28336930301e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v3i5.41229": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.36360": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.40796": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.41153": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.42511": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.42555": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.42752": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.42768": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i6.42795": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i7.41295": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i7.42830": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i7.42861": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}
+{"10.0000/hoplos.v4i7.43096": [{"id": "influence", "unit": [{"value": "5.91019644836e-09", "key": "score"}]}, {"id": "popularity_alt", "unit": [{"value": "0.0", "key": "score"}]}, {"id": "popularity", "unit": [{"value": "8.48190886761e-09", "key": "score"}]}]}

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos.json

@@ -0,0 +1,38 @@
+{"doi":"10.3390/s18072310","level1":"engineering and technology","level2":"nano-technology","level3":"nanoscience & nanotechnology"}
+{"doi":"10.1111/1365-2656.12831\u000210.17863/cam.24369","level1":"social sciences","level2":"psychology and cognitive sciences","level3":"NULL"}
+{"doi":"10.3929/ethz-b-000187584\u000210.1002/chem.201701644","level1":"natural sciences","level2":"NULL","level3":"NULL"}
+{"doi":"10.1080/01913123.2017.1367361","level1":"medical and health sciences","level2":"clinical medicine","level3":"oncology & carcinogenesis"}
+{"doi":"10.1051/e3sconf/20199207011","level1":"natural sciences","level2":"earth and related environmental sciences","level3":"environmental sciences"}
+{"doi":"10.1038/onc.2015.333","level1":"medical and health sciences","level2":"clinical medicine","level3":"oncology & carcinogenesis"}
+{"doi":"10.1093/mnras/staa256","level1":"natural sciences","level2":"physical sciences","level3":"NULL"}
+{"doi":"10.1016/j.jclepro.2018.07.166","level1":"engineering and technology","level2":"other engineering and technologies","level3":"building & construction"}
+{"doi":"10.1103/physrevlett.125.037403","level1":"natural sciences","level2":"physical sciences","level3":"nuclear & particles physics"}
+{"doi":"10.1080/03602532.2017.1316285","level1":"natural sciences","level2":"NULL","level3":"NULL"}
+{"doi":"10.1001/jamanetworkopen.2019.1868","level1":"medical and health sciences","level2":"other medical science","level3":"health policy & services"}
+{"doi":"10.1128/mra.00874-18","level1":"natural sciences","level2":"biological sciences","level3":"plant biology & botany"}
+{"doi":"10.1016/j.nancom.2018.03.001","level1":"engineering and technology","level2":"NULL","level3":"NULL"}
+{"doi":"10.1112/topo.12174","level1":"natural sciences","level2":"NULL","level3":"NULL"}
+{"doi":"10.12688/wellcomeopenres.15846.1","level1":"medical and health sciences","level2":"health sciences","level3":"NULL"}
+{"doi":"10.21468/scipostphys.3.1.001","level1":"natural sciences","level2":"physical sciences","level3":"NULL"}
+{"doi":"10.1088/1741-4326/ab6c77","level1":"natural sciences","level2":"physical sciences","level3":"nuclear & particles physics"}
+{"doi":"10.1109/tpwrs.2019.2944747","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"electrical & electronic engineering"}
+{"doi":"10.1016/j.expthermflusci.2019.109994\u000210.17863/cam.46212","level1":"engineering and technology","level2":"mechanical engineering","level3":"mechanical engineering & transports"}
+{"doi":"10.1109/tc.2018.2860012","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"computer hardware & architecture"}
+{"doi":"10.1002/mma.6622","level1":"natural sciences","level2":"mathematics","level3":"numerical & computational mathematics"}
+{"doi":"10.1051/radiopro/2020020","level1":"natural sciences","level2":"chemical sciences","level3":"NULL"}
+{"doi":"10.1007/s12268-019-1003-4","level1":"medical and health sciences","level2":"basic medicine","level3":"NULL"}
+{"doi":"10.3390/cancers12010236","level1":"medical and health sciences","level2":"health sciences","level3":"biochemistry & molecular biology"}
+{"doi":"10.6084/m9.figshare.9912614\u000210.6084/m9.figshare.9912614.v1\u000210.1080/00268976.2019.1665199","level1":"natural sciences","level2":"chemical sciences","level3":"physical chemistry"}
+{"doi":"10.1175/jpo-d-17-0239.1","level1":"natural sciences","level2":"biological sciences","level3":"marine biology & hydrobiology"}
+{"doi":"10.1007/s13218-020-00674-7","level1":"engineering and technology","level2":"industrial biotechnology","level3":"industrial engineering & automation"}
+{"doi":"10.1016/j.psyneuen.2016.02.003\u000210.1016/j.psyneuen.2016.02.00310.7892/boris.78886\u000210.7892/boris.78886","level1":"medical and health sciences","level2":"basic medicine","level3":"NULL"}
+{"doi":"10.1109/ted.2018.2813542","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"electrical & electronic engineering"}
+{"doi":"10.3989/scimar.04739.25a","level1":"natural sciences","level2":"biological sciences","level3":"NULL"}
+{"doi":"10.3390/su12187503","level1":"natural sciences","level2":"earth and related environmental sciences","level3":"NULL"}
+{"doi":"10.1016/j.ccell.2018.08.017","level1":"medical and health sciences","level2":"basic medicine","level3":"biochemistry & molecular biology"}
+{"doi":"10.1103/physrevresearch.2.023322","level1":"natural sciences","level2":"physical sciences","level3":"nuclear & particles physics"}
+{"doi":"10.1039/c8cp03234c","level1":"natural sciences","level2":"NULL","level3":"NULL"}
+{"doi":"10.5281/zenodo.3696557\u000210.5281/zenodo.3696556\u000210.1109/jsac.2016.2545384","level1":"engineering and technology","level2":"electrical engineering, electronic engineering, information engineering","level3":"networking & telecommunications"}
+{"doi":"10.1038/ng.3667\u000210.1038/ng.3667.\u000210.17615/tct6-4m26\u000210.17863/cam.15649","level1":"medical and health sciences","level2":"health sciences","level3":"genetics & heredity"}
+{"doi":"10.1016/j.jclepro.2019.119065","level1":"engineering and technology","level2":"other engineering and technologies","level3":"building & construction"}
+{"doi":"10.1111/pce.13392","level1":"agricultural and veterinary sciences","level2":"agriculture, forestry, and fisheries","level3":"agronomy & agriculture"}

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/h2020_fos_sbs.csv

@@ -0,0 +1,38 @@
+dedup_wf_001::ddcc7a56fa13e49bcc59c6bdd19ad26c	10.3390/s18072310	engineering and technology	nano-technology	nanoscience & nanotechnology
+dedup_wf_001::b76062d56e28224eac56111a4e1e5ecf	10.1111/1365-2656.1283110.17863/cam.24369	social sciences	psychology and cognitive sciences	NULL
+dedup_wf_001::bb752acb8f403a25fa7851a302f7b7ac	10.3929/ethz-b-00018758410.1002/chem.201701644	natural sciences	NULL	NULL
+dedup_wf_001::2f1435a9201ecf5cbbcb12c9b2d971cd	10.1080/01913123.2017.1367361	medical and health sciences	clinical medicine	oncology & carcinogenesis
+dedup_wf_001::fc9e47ec16c67b101724320d4b030514	10.1051/e3sconf/20199207011	natural sciences	earth and related environmental sciences	environmental sciences
+dedup_wf_001::caa1e5b4de387cb31751552f4f0f5d72	10.1038/onc.2015.333	medical and health sciences	clinical medicine	oncology & carcinogenesis
+dedup_wf_001::c2a98df5637d69bf0524eaf40fe6bf11	10.1093/mnras/staa256	natural sciences	physical sciences	NULL
+dedup_wf_001::c221262bdc77cbfd59859a402f0e3991	10.1016/j.jclepro.2018.07.166	engineering and technology	other engineering and technologies	building & construction
+doiboost____::d56d9dc21f317b3e009d5b6c8ea87212	10.1103/physrevlett.125.037403	natural sciences	physical sciences	nuclear & particles physics
+dedup_wf_001::8a7269c8ee6470b2fb4fd384bc389e08	10.1080/03602532.2017.1316285	natural sciences	NULL	NULL
+dedup_wf_001::28342ebbc19833e4e1f4a2b23cf5ee20	10.1001/jamanetworkopen.2019.1868	medical and health sciences	other medical science	health policy & services
+dedup_wf_001::c1e1daf2b55dd9ec8e1c7c7458bbc7bc	10.1128/mra.00874-18	natural sciences	biological sciences	plant biology & botany
+dedup_wf_001::a2ef4a2720c71907180750e5871298ef	10.1016/j.nancom.2018.03.001	engineering and technology	NULL	NULL
+dedup_wf_001::676f46a31519e83a89efcb1c626286fb	10.1112/topo.12174	natural sciences	NULL	NULL
+dedup_wf_001::6f2761642f1e39313388e2c4060657dd	10.12688/wellcomeopenres.15846.1	medical and health sciences	health sciences	NULL
+dedup_wf_001::e414c1dec599521a9635a60de0f6755b	10.21468/scipostphys.3.1.001	natural sciences	physical sciences	NULL
+dedup_wf_001::f3395fe0f330164ea424dc61c86c9a3d	10.1088/1741-4326/ab6c77	natural sciences	physical sciences	nuclear & particles physics
+dedup_wf_001::a4f32a97a783117012f1de11797e73f2	10.1109/tpwrs.2019.2944747	engineering and technology	electrical engineering, electronic engineering, information engineering	electrical & electronic engineering
+dedup_wf_001::313ae1cd083ae1696d12dd1909f97df8	10.1016/j.expthermflusci.2019.10999410.17863/cam.46212	engineering and technology	mechanical engineering	mechanical engineering & transports
+dedup_wf_001::2a300a7d3ca7347791ebcef986bc0682	10.1109/tc.2018.2860012	engineering and technology	electrical engineering, electronic engineering, information engineering	computer hardware & architecture
+doiboost____::5b79bd7bd9f87361b4a4abc3cbb2df75	10.1002/mma.6622	natural sciences	mathematics	numerical & computational mathematics
+dedup_wf_001::6a3f61f217a2519fbaddea1094e3bfc2	10.1051/radiopro/2020020	natural sciences	chemical sciences	NULL
+dedup_wf_001::a3f0430309a639f4234a0e57b10f2dee	10.1007/s12268-019-1003-4	medical and health sciences	basic medicine	NULL
+dedup_wf_001::b6b8a3a1cccbee459cf3343485efdb12	10.3390/cancers12010236	medical and health sciences	health sciences	biochemistry & molecular biology
+dedup_wf_001::dd06ee7974730e7b09a4f03c83b3f9bd	10.6084/m9.figshare.991261410.6084/m9.figshare.9912614.v110.1080/00268976.2019.1665199	natural sciences	chemical sciences	physical chemistry
+dedup_wf_001::027c78bef6f972b5e26dfea55d30fbe3	10.1175/jpo-d-17-0239.1	natural sciences	biological sciences	marine biology & hydrobiology
+dedup_wf_001::43edc179aa9e1fbaf582c5203b18b519	10.1007/s13218-020-00674-7	engineering and technology	industrial biotechnology	industrial engineering & automation
+dedup_wf_001::e7770e11cd6eb514bb52c07b5a8a80f0	10.1016/j.psyneuen.2016.02.00310.1016/j.psyneuen.2016.02.00310.7892/boris.7888610.7892/boris.78886	medical and health sciences	basic medicine	NULL
+dedup_wf_001::80bc15d69bdc589149631f3439dde5aa	10.1109/ted.2018.2813542	engineering and technology	electrical engineering, electronic engineering, information engineering	electrical & electronic engineering
+dedup_wf_001::42c1cfa33e7872944b920cff90f4d99e	10.3989/scimar.04739.25a	natural sciences	biological sciences	NULL
+dedup_wf_001::9bacdbbaa9da3658b7243d5de8e3ce14	10.3390/su12187503	natural sciences	earth and related environmental sciences	NULL
+dedup_wf_001::59e43d3527dcfecb6097fbd5740c8950	10.1016/j.ccell.2018.08.017	medical and health sciences	basic medicine	biochemistry & molecular biology
+doiboost____::e024d1b738df3b24bc58fa0228542571	10.1103/physrevresearch.2.023322	natural sciences	physical sciences	nuclear & particles physics
+dedup_wf_001::66e9a3237fa8178886d26d3c2d5b9e66	10.1039/c8cp03234c	natural sciences	NULL	NULL
+dedup_wf_001::83737ab4205bae751571bb3b166efa18	10.5281/zenodo.369655710.5281/zenodo.369655610.1109/jsac.2016.2545384	engineering and technology	electrical engineering, electronic engineering, information engineering	networking & telecommunications
+dedup_wf_001::e3f892db413a689e572dd256acad55fe	10.1038/ng.366710.1038/ng.3667.10.17615/tct6-4m2610.17863/cam.15649	medical and health sciences	health sciences	genetics & heredity
+dedup_wf_001::14ba594e8fd081847bc3f50f56335003	10.1016/j.jclepro.2019.119065	engineering and technology	other engineering and technologies	building & construction
+dedup_wf_001::08ac7b33a41bcea2d055ecd8585d632e	10.1111/pce.13392	agricultural and veterinary sciences	agriculture, forestry, and fisheries	agronomy & agriculture

dhp-workflows/dhp-blacklist/src/main/java/eu/dnetlib/dhp/blacklist/ReadBlacklistFromDB.java

@@ -91,8 +91,8 @@ public class ReadBlacklistFromDB implements Closeable {
 
 			String encoding = rs.getString("relationship");
 			RelationInverse ri = ModelSupport.relationInverseMap.get(encoding);
-			direct.setRelClass(ri.getRelation());
-			inverse.setRelClass(ri.getInverse());
+			direct.setRelClass(ri.getRelClass());
+			inverse.setRelClass(ri.getInverseRelClass());
 			direct.setRelType(ri.getRelType());
 			inverse.setRelType(ri.getRelType());
 			direct.setSubRelType(ri.getSubReltype());

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java

@@ -0,0 +1,154 @@
+
+package eu.dnetlib.dhp.oa.dedup;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.api.java.function.PairFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.SparkSession;
+import org.dom4j.DocumentException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.schema.oaf.DataInfo;
+import eu.dnetlib.dhp.schema.oaf.Relation;
+import eu.dnetlib.dhp.utils.ISLookupClientFactory;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.pace.config.DedupConfig;
+import eu.dnetlib.pace.model.MapDocument;
+import eu.dnetlib.pace.util.MapDocumentUtil;
+import scala.Tuple2;
+
+public class SparkWhitelistSimRels extends AbstractSparkAction {
+
+	private static final Logger log = LoggerFactory.getLogger(SparkCreateSimRels.class);
+
+	private static final String WHITELIST_SEPARATOR = "####";
+
+	public SparkWhitelistSimRels(ArgumentApplicationParser parser, SparkSession spark) {
+		super(parser, spark);
+	}
+
+	public static void main(String[] args) throws Exception {
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateSimRels.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/whitelistSimRels_parameters.json")));
+		parser.parseArgument(args);
+
+		SparkConf conf = new SparkConf();
+		new SparkWhitelistSimRels(parser, getSparkSession(conf))
+			.run(ISLookupClientFactory.getLookUpService(parser.get("isLookUpUrl")));
+	}
+
+	@Override
+	public void run(ISLookUpService isLookUpService)
+		throws DocumentException, IOException, ISLookUpException, SAXException {
+
+		// read oozie parameters
+		final String graphBasePath = parser.get("graphBasePath");
+		final String isLookUpUrl = parser.get("isLookUpUrl");
+		final String actionSetId = parser.get("actionSetId");
+		final String workingPath = parser.get("workingPath");
+		final int numPartitions = Optional
+			.ofNullable(parser.get("numPartitions"))
+			.map(Integer::valueOf)
+			.orElse(NUM_PARTITIONS);
+		final String whiteListPath = parser.get("whiteListPath");
+
+		log.info("numPartitions: '{}'", numPartitions);
+		log.info("graphBasePath: '{}'", graphBasePath);
+		log.info("isLookUpUrl:   '{}'", isLookUpUrl);
+		log.info("actionSetId:   '{}'", actionSetId);
+		log.info("workingPath:   '{}'", workingPath);
+		log.info("whiteListPath: '{}'", whiteListPath);
+
+		JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		// file format: source####target
+		Dataset<Tuple2<String, String>> whiteListRels = spark
+			.createDataset(
+				sc
+					.textFile(whiteListPath)
+					// check if the line is in the correct format: id1####id2
+					.filter(s -> s.contains(WHITELIST_SEPARATOR) && s.split(WHITELIST_SEPARATOR).length == 2)
+					.map(s -> new Tuple2<>(s.split(WHITELIST_SEPARATOR)[0], s.split(WHITELIST_SEPARATOR)[1]))
+					.rdd(),
+				Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+
+		// for each dedup configuration
+		for (DedupConfig dedupConf : getConfigurations(isLookUpService, actionSetId)) {
+
+			final String entity = dedupConf.getWf().getEntityType();
+			final String subEntity = dedupConf.getWf().getSubEntityValue();
+			log.info("Adding whitelist simrels for: '{}'", subEntity);
+
+			final String outputPath = DedupUtility.createSimRelPath(workingPath, actionSetId, subEntity);
+
+			Dataset<Tuple2<String, String>> entities = spark
+				.createDataset(
+					sc
+						.textFile(DedupUtility.createEntityPath(graphBasePath, subEntity))
+						.repartition(numPartitions)
+						.mapToPair(
+							(PairFunction<String, String, String>) s -> {
+								MapDocument d = MapDocumentUtil.asMapDocumentWithJPath(dedupConf, s);
+								return new Tuple2<>(d.getIdentifier(), "present");
+							})
+						.rdd(),
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+
+			Dataset<Tuple2<String, String>> whiteListRels1 = whiteListRels
+				.joinWith(entities, whiteListRels.col("_1").equalTo(entities.col("_1")), "inner")
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+
+			Dataset<Tuple2<String, String>> whiteListRels2 = whiteListRels1
+				.joinWith(entities, whiteListRels1.col("_2").equalTo(entities.col("_1")), "inner")
+				.map(
+					(MapFunction<Tuple2<Tuple2<String, String>, Tuple2<String, String>>, Tuple2<String, String>>) Tuple2::_1,
+					Encoders.tuple(Encoders.STRING(), Encoders.STRING()));
+
+			Dataset<Relation> whiteListSimRels = whiteListRels2
+				.map(
+					(MapFunction<Tuple2<String, String>, Relation>) r -> createSimRel(r._1(), r._2(), entity),
+					Encoders.bean(Relation.class));
+
+			saveParquet(whiteListSimRels, outputPath, SaveMode.Append);
+		}
+	}
+
+	private Relation createSimRel(String source, String target, String entity) {
+		final Relation r = new Relation();
+		r.setSource(source);
+		r.setTarget(target);
+		r.setSubRelType("dedupSimilarity");
+		r.setRelClass("isSimilarTo");
+		r.setDataInfo(new DataInfo());
+
+		switch (entity) {
+			case "result":
+				r.setRelType("resultResult");
+				break;
+			case "organization":
+				r.setRelType("organizationOrganization");
+				break;
+			default:
+				throw new IllegalArgumentException("unmanaged entity type: " + entity);
+		}
+		return r;
+	}
+}

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/UpdateOpenorgsJob.java

@@ -0,0 +1,117 @@
+
+package eu.dnetlib.dhp.oa.dedup;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+
+public class UpdateOpenorgsJob {
+
+	private static final Logger log = LoggerFactory.getLogger(UpdateOpenorgsJob.class);
+
+	public static void main(String[] args) throws Exception {
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkCreateSimRels.class
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json")));
+		parser.parseArgument(args);
+
+		final String apiUrl = parser.get("apiUrl");
+		final int delay = Integer.parseInt(parser.get("delay"));
+
+		log.info("apiUrl: '{}'", apiUrl);
+		log.info("delay: '{}'", delay);
+
+		APIResponse res = httpCall(apiUrl);
+		while (res != null && res.getStatus().equals(ImportStatus.RUNNING)) {
+			TimeUnit.MINUTES.sleep(delay);
+			res = httpCall(apiUrl + "/status");
+		}
+
+		if (res == null) {
+			log.error("Openorgs Update FAILED: No response");
+			throw new RuntimeException("Openorgs Update FAILED: No response");
+		}
+
+		if (res.getStatus() == null || !res.getStatus().equals(ImportStatus.SUCCESS)) {
+			log.error("Openorgs Update FAILED: '{}' - '{}'", res.getStatus(), res.getMessage());
+			throw new RuntimeException(res.getMessage());
+		}
+
+	}
+
+	private static APIResponse httpCall(final String url) throws Exception {
+		final HttpGet req = new HttpGet(url);
+
+		try (final CloseableHttpClient client = HttpClients.createDefault()) {
+			try (final CloseableHttpResponse response = client.execute(req)) {
+				final String s = IOUtils.toString(response.getEntity().getContent());
+				return (new ObjectMapper()).readValue(s, APIResponse.class);
+			}
+		}
+	}
+
+}
+
+class APIResponse {
+	private String id;
+	private Long dateStart;
+	private Long dateEnd;
+	private ImportStatus status;
+	private String message;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public Long getDateStart() {
+		return dateStart;
+	}
+
+	public void setDateStart(Long dateStart) {
+		this.dateStart = dateStart;
+	}
+
+	public Long getDateEnd() {
+		return dateEnd;
+	}
+
+	public void setDateEnd(Long dateEnd) {
+		this.dateEnd = dateEnd;
+	}
+
+	public ImportStatus getStatus() {
+		return status;
+	}
+
+	public void setStatus(ImportStatus status) {
+		this.status = status;
+	}
+
+	public String getMessage() {
+		return message;
+	}
+
+	public void setMessage(String message) {
+		this.message = message;
+	}
+}
+
+enum ImportStatus {
+	SUCCESS, FAILED, RUNNING, NOT_LAUNCHED, NOT_YET_STARTED
+}

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml

@@ -28,6 +28,11 @@
             <name>dbPwd</name>
             <description>password to access the OpenOrgs database</description>
         </property>
+        <property>
+            <name>dbConnections</name>
+            <value>10</value>
+            <description>number of connections to the postgres db</description>
+        </property>
         <property>
             <name>workingPath</name>
             <description>path for the working directory</description>
@@ -223,7 +228,7 @@
             <arg>--dbTable</arg><arg>${dbTable}</arg>
             <arg>--dbUser</arg><arg>${dbUser}</arg>
             <arg>--dbPwd</arg><arg>${dbPwd}</arg>
-            <arg>--numConnections</arg><arg>20</arg>
+            <arg>--numConnections</arg><arg>${dbConnections}</arg>
         </spark>
         <ok to="PrepareNewOrgs"/>
         <error to="Kill"/>
@@ -254,19 +259,24 @@
             <arg>--dbTable</arg><arg>${dbTable}</arg>
             <arg>--dbUser</arg><arg>${dbUser}</arg>
             <arg>--dbPwd</arg><arg>${dbPwd}</arg>
-            <arg>--numConnections</arg><arg>20</arg>
+            <arg>--numConnections</arg><arg>${dbConnections}</arg>
         </spark>
         <ok to="update_openorgs"/>
         <error to="Kill"/>
     </action>
 
     <action name="update_openorgs">
-        <shell xmlns="uri:oozie:shell-action:0.1">
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <exec>/usr/bin/curl</exec>
-            <argument>${apiUrl}</argument>
-        </shell>
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.oa.dedup.UpdateOpenorgsJob</main-class>
+            <arg>--apiUrl</arg><arg>${apiUrl}</arg>
+            <arg>--delay</arg><arg>5</arg>
+        </java>
         <ok to="End"/>
         <error to="Kill"/>
     </action>

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/scan/oozie_app/workflow.xml

@@ -20,6 +20,10 @@
             <name>workingPath</name>
             <description>path for the working directory</description>
         </property>
+        <property>
+            <name>whiteListPath</name>
+            <description>path for the whitelist of similarity relations</description>
+        </property>
         <property>
             <name>dedupGraphPath</name>
             <description>path for the output graph</description>
@@ -130,6 +134,34 @@
             <arg>--workingPath</arg><arg>${workingPath}</arg>
             <arg>--numPartitions</arg><arg>8000</arg>
         </spark>
+        <ok to="WhitelistSimRels"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="WhitelistSimRels">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>Add Whitelist Similarity Relations</name>
+            <class>eu.dnetlib.dhp.oa.dedup.SparkWhitelistSimRels</class>
+            <jar>dhp-dedup-openaire-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.shuffle.partitions=3840
+            </spark-opts>
+            <arg>--graphBasePath</arg><arg>${graphBasePath}</arg>
+            <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
+            <arg>--actionSetId</arg><arg>${actionSetId}</arg>
+            <arg>--workingPath</arg><arg>${workingPath}</arg>
+            <arg>--whiteListPath</arg><arg>${whiteListPath}</arg>
+            <arg>--numPartitions</arg><arg>8000</arg>
+        </spark>
         <ok to="CreateMergeRel"/>
         <error to="Kill"/>
     </action>

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/updateOpenorgsJob_parameters.json

@@ -0,0 +1,14 @@
+[
+  {
+    "paramName": "api",
+    "paramLongName": "apiUrl",
+    "paramDescription": "the url of the API",
+    "paramRequired": true
+  },
+  {
+    "paramName": "d",
+    "paramLongName": "delay",
+    "paramDescription": "delay for the HTTP call in minutes",
+    "paramRequired": true
+  }
+]

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/whitelistSimRels_parameters.json

@@ -0,0 +1,38 @@
+[
+  {
+    "paramName": "la",
+    "paramLongName": "isLookUpUrl",
+    "paramDescription": "address for the LookUp",
+    "paramRequired": true
+  },
+  {
+    "paramName": "asi",
+    "paramLongName": "actionSetId",
+    "paramDescription": "action set identifier (name of the orchestrator)",
+    "paramRequired": true
+  },
+  {
+    "paramName": "i",
+    "paramLongName": "graphBasePath",
+    "paramDescription": "the base path of the raw graph",
+    "paramRequired": true
+  },
+  {
+    "paramName": "w",
+    "paramLongName": "workingPath",
+    "paramDescription": "path of the working directory",
+    "paramRequired": true
+  },
+  {
+    "paramName": "np",
+    "paramLongName": "numPartitions",
+    "paramDescription": "number of partitions for the similarity relations intermediate phases",
+    "paramRequired": false
+  },
+  {
+    "paramName": "wl",
+    "paramLongName": "whiteListPath",
+    "paramDescription": "whitelist file path for the addition of custom simrels",
+    "paramRequired": true
+  }
+]

dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java

@@ -5,13 +5,16 @@ import static java.nio.file.Files.createTempDirectory;
 
 import static org.apache.spark.sql.functions.count;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.lenient;
 
 import java.io.File;
+import java.io.FileReader;
 import java.io.IOException;
 import java.io.Serializable;
 import java.net.URISyntaxException;
 import java.nio.file.Paths;
+import java.util.List;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
@@ -55,6 +58,10 @@ public class SparkDedupTest implements Serializable {
 	private static String testOutputBasePath;
 	private static String testDedupGraphBasePath;
 	private static final String testActionSetId = "test-orchestrator";
+	private static String whitelistPath;
+	private static List<String> whiteList;
+
+	private static String WHITELIST_SEPARATOR = "####";
 
 	@BeforeAll
 	public static void cleanUp() throws IOException, URISyntaxException {
@@ -71,6 +78,12 @@ public class SparkDedupTest implements Serializable {
 			.toAbsolutePath()
 			.toString();
 
+		whitelistPath = Paths
+			.get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/whitelist.simrels.txt").toURI())
+			.toFile()
+			.getAbsolutePath();
+		whiteList = IOUtils.readLines(new FileReader(whitelistPath));
+
 		FileUtils.deleteDirectory(new File(testOutputBasePath));
 		FileUtils.deleteDirectory(new File(testDedupGraphBasePath));
 
@@ -202,6 +215,84 @@ public class SparkDedupTest implements Serializable {
 
 	@Test
 	@Order(2)
+	void whitelistSimRelsTest() throws Exception {
+
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					SparkWhitelistSimRels.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/whitelistSimRels_parameters.json")));
+
+		parser
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath,
+					"-asi", testActionSetId,
+					"-la", "lookupurl",
+					"-w", testOutputBasePath,
+					"-np", "50",
+					"-wl", whitelistPath
+				});
+
+		new SparkWhitelistSimRels(parser, spark).run(isLookUpService);
+
+		long orgs_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization"))
+			.count();
+
+		long pubs_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "publication"))
+			.count();
+
+		long ds_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "dataset"))
+			.count();
+
+		long orp_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
+			.count();
+
+		// entities simrels supposed to be equal to the number of previous step (no rels in whitelist)
+		assertEquals(3082, orgs_simrel);
+		assertEquals(7036, pubs_simrel);
+		assertEquals(442, ds_simrel);
+		assertEquals(6750, orp_simrel);
+
+		// entities simrels to be different from the number of previous step (new simrels in the whitelist)
+		Dataset<Row> sw_simrel = spark
+			.read()
+			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"));
+
+		// check if the first relation in the whitelist exists
+		assertTrue(
+			sw_simrel
+				.as(Encoders.bean(Relation.class))
+				.toJavaRDD()
+				.filter(
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[0])
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(0).split(WHITELIST_SEPARATOR)[1]))
+				.count() > 0);
+		// check if the second relation in the whitelist exists
+		assertTrue(
+			sw_simrel
+				.as(Encoders.bean(Relation.class))
+				.toJavaRDD()
+				.filter(
+					rel -> rel.getSource().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[0])
+						&& rel.getTarget().equalsIgnoreCase(whiteList.get(1).split(WHITELIST_SEPARATOR)[1]))
+				.count() > 0);
+
+		assertEquals(338, sw_simrel.count());
+
+	}
+
+	@Test
+	@Order(3)
 	void cutMergeRelsTest() throws Exception {
 
 		ArgumentApplicationParser parser = new ArgumentApplicationParser(
@@ -297,7 +388,7 @@ public class SparkDedupTest implements Serializable {
 	}
 
 	@Test
-	@Order(3)
+	@Order(4)
 	void createMergeRelsTest() throws Exception {
 
 		ArgumentApplicationParser parser = new ArgumentApplicationParser(
@@ -353,7 +444,7 @@ public class SparkDedupTest implements Serializable {
 	}
 
 	@Test
-	@Order(4)
+	@Order(5)
 	void createDedupRecordTest() throws Exception {
 
 		ArgumentApplicationParser parser = new ArgumentApplicationParser(
@@ -394,13 +485,13 @@ public class SparkDedupTest implements Serializable {
 
 		assertEquals(85, orgs_deduprecord);
 		assertEquals(65, pubs_deduprecord);
-		assertEquals(51, sw_deduprecord);
+		assertEquals(49, sw_deduprecord);
 		assertEquals(97, ds_deduprecord);
 		assertEquals(89, orp_deduprecord);
 	}
 
 	@Test
-	@Order(5)
+	@Order(6)
 	void updateEntityTest() throws Exception {
 
 		ArgumentApplicationParser parser = new ArgumentApplicationParser(
@@ -479,7 +570,7 @@ public class SparkDedupTest implements Serializable {
 		assertEquals(838, organizations);
 		assertEquals(100, projects);
 		assertEquals(100, datasource);
-		assertEquals(200, softwares);
+		assertEquals(198, softwares);
 		assertEquals(389, dataset);
 		assertEquals(517, otherresearchproduct);
 
@@ -516,7 +607,7 @@ public class SparkDedupTest implements Serializable {
 	}
 
 	@Test
-	@Order(6)
+	@Order(7)
 	void propagateRelationTest() throws Exception {
 
 		ArgumentApplicationParser parser = new ArgumentApplicationParser(
@@ -566,7 +657,7 @@ public class SparkDedupTest implements Serializable {
 	}
 
 	@Test
-	@Order(7)
+	@Order(8)
 	void testRelations() throws Exception {
 		testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_1.json", 12, 10);
 		testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_2.json", 10, 2);

dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/whitelist.simrels.txt

@@ -0,0 +1,2 @@
+50|r37b0ad08687::f645b9729d1e1025a72c57883f0f2cac####50|r37b0ad08687::4c55b436743b5c49fa32cd582fd9e1aa
+50|datacite____::a90f49f9fde5393c00633bea6e4e374a####50|datacite____::5f55cdee77303ba8a2bf9996c32a330c

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala

@@ -13,10 +13,30 @@ import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.JavaConverters._
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST.{JField, JObject, JString,JArray}
+import org.json4s.jackson.JsonMethods.parse
 
 object SparkGenerateDoiBoost {
 
 
+  def extractIdGRID(input:String):List[(String,String)] = {
+    implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
+    lazy val json: org.json4s.JValue = parse(input)
+
+    val id:String = (json \ "id").extract[String]
+
+    val grids:List[String] = for {
+
+      JObject(pid) <- json \ "pid"
+      JField("qualifier", JObject(qualifier)) <- pid
+      JField("classid",   JString(classid)) <-qualifier
+      JField("value", JString(vl)) <- pid
+      if classid == "GRID"
+    } yield vl
+    grids.map(g => (id, s"unresolved::grid::${g.toLowerCase}"))(collection.breakOut)
+  }
+
 
 
   def main(args: Array[String]): Unit = {
@@ -36,6 +56,7 @@ object SparkGenerateDoiBoost {
 
     val hostedByMapPath = parser.get("hostedByMapPath")
     val workingDirPath = parser.get("workingPath")
+    val openaireOrganizationPath = parser.get("openaireOrganizationPath")
 
     val crossrefAggregator = new Aggregator[(String, Publication), Publication, Publication] with Serializable {
       override def zero: Publication = new Publication
@@ -156,7 +177,7 @@ object SparkGenerateDoiBoost {
     magPubs.joinWith(a,magPubs("_1").equalTo(a("PaperId"))).flatMap(item => {
       val pub:Publication = item._1._2
       val affiliation = item._2
-      val affId:String = if (affiliation.GridId.isDefined) DoiBoostMappingUtil.generateGridAffiliationId(affiliation.GridId.get) else  DoiBoostMappingUtil.generateMAGAffiliationId(affiliation.AffiliationId.toString)
+      val affId:String = if (affiliation.GridId.isDefined) s"unresolved::grid::${affiliation.GridId.get.toLowerCase}" else  DoiBoostMappingUtil.generateMAGAffiliationId(affiliation.AffiliationId.toString)
       val r:Relation = new Relation
       r.setSource(pub.getId)
       r.setTarget(affId)
@@ -174,8 +195,34 @@ object SparkGenerateDoiBoost {
       r1.setDataInfo(pub.getDataInfo)
       r1.setCollectedfrom(List(DoiBoostMappingUtil.createMAGCollectedFrom()).asJava)
       List(r, r1)
-    })(mapEncoderRel).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationAffiliation")
+    })(mapEncoderRel).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationAffiliation_unresolved")
+
+
+
+
+    val unresolvedRels:Dataset[(String, Relation)] = spark.read.load(s"$workingDirPath/doiBoostPublicationAffiliation_unresolved").as[Relation].map(r => {
+
+      if (r.getSource.startsWith("unresolved"))
+        (r.getSource, r)
+      else if (r.getTarget.startsWith("unresolved"))
+        (r.getTarget,r)
+        else
+        ("resolved", r)
+    })(Encoders.tuple(Encoders.STRING, mapEncoderRel))
+
+    val openaireOrganization:Dataset[(String,String)] = spark.read.text(openaireOrganizationPath).as[String].flatMap(s => extractIdGRID(s)).groupByKey(_._2).reduceGroups((x,y) => if (x != null) x else y ).map(_._2)
 
+    unresolvedRels.joinWith(openaireOrganization,unresolvedRels("_1").equalTo(openaireOrganization("_2")))
+      .map { x =>
+        val currentRels = x._1._2
+        val currentOrgs = x._2
+        if (currentOrgs!= null)
+          if(currentRels.getSource.startsWith("unresolved"))
+            currentRels.setSource(currentOrgs._1)
+          else
+            currentRels.setTarget(currentOrgs._1)
+      currentRels
+      }.filter(r=> !r.getSource.startsWith("unresolved") && !r.getTarget.startsWith("unresolved")).write.mode(SaveMode.Overwrite).save(s"$workingDirPath/doiBoostPublicationAffiliation")
 
     magPubs.joinWith(a,magPubs("_1").equalTo(a("PaperId"))).map( item => {
       val affiliation = item._2

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala

@@ -70,7 +70,7 @@ case object Crossref2Oaf {
     "reference-book" -> "0002 Book",
     "monograph" -> "0002 Book",
     "journal-article" -> "0001 Article",
-    "dissertation" -> "0006 Doctoral thesis",
+    "dissertation" -> "0044 Thesis",
     "other" -> "0038 Other literature type",
     "peer-review" -> "0015 Review",
     "proceedings" -> "0004 Conference object",
@@ -206,11 +206,16 @@ case object Crossref2Oaf {
     else {
       instance.setDateofacceptance(asField(createdDate.getValue))
     }
-    val s: String = (json \ "URL").extract[String]
-    val links: List[String] = ((for {JString(url) <- json \ "link" \ "URL"} yield url) ::: List(s)).filter(p => p != null).distinct
-    if (links.nonEmpty) {
-      instance.setUrl(links.asJava)
-    }
+    val s: List[String] = List("https://doi.org/" + doi)
+//    val links: List[String] = ((for {JString(url) <- json \ "link" \ "URL"} yield url) ::: List(s)).filter(p => p != null && p.toLowerCase().contains(doi.toLowerCase())).distinct
+//    if (links.nonEmpty) {
+//      instance.setUrl(links.asJava)
+//    }
+    if(s.nonEmpty)
+      {
+        instance.setUrl(s.asJava)
+      }
+
     result.setInstance(List(instance).asJava)
 
     //IMPORTANT

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/SparkProcessMAG.scala

@@ -111,26 +111,9 @@ object SparkProcessMAG {
       .map(item => ConversionUtil.updatePubsWithConferenceInfo(item))
       .write
       .mode(SaveMode.Overwrite)
-      .save(s"$workingPath/merge_step_2_conference")
-
-
-    magPubs= spark.read.load(s"$workingPath/merge_step_2_conference").as[Publication]
-      .map(p => (ConversionUtil.extractMagIdentifier(p.getOriginalId.asScala), p)).as[(String, Publication)]
-
-    val paperUrlDataset = spark.read.load(s"$sourcePath/PaperUrls").as[MagPaperUrl].groupBy("PaperId").agg(collect_list(struct("sourceUrl")).as("instances")).as[MagUrl]
-
-
-    logger.info("Phase 5) enrich publication with URL and Instances")
-    magPubs.joinWith(paperUrlDataset, col("_1").equalTo(paperUrlDataset("PaperId")), "left")
-      .map { a: ((String, Publication), MagUrl) => ConversionUtil.addInstances((a._1._2, a._2)) }
-      .write.mode(SaveMode.Overwrite)
       .save(s"$workingPath/merge_step_3")
 
 
-//    logger.info("Phase 6) Enrich Publication with description")
-//    val pa = spark.read.load(s"${parser.get("sourcePath")}/PaperAbstractsInvertedIndex").as[MagPaperAbstract]
-//    pa.map(ConversionUtil.transformPaperAbstract).write.mode(SaveMode.Overwrite).save(s"${parser.get("targetPath")}/PaperAbstract")
-
     val paperAbstract = spark.read.load((s"$workingPath/PaperAbstract")).as[MagPaperAbstract]
 
 
@@ -162,12 +145,14 @@ object SparkProcessMAG {
       .write.mode(SaveMode.Overwrite)
       .save(s"$workingPath/mag_publication")
 
+    spark.read.load(s"$workingPath/mag_publication").as[Publication]
+      .filter(p => p.getId == null)
+      .groupByKey(p => p.getId)
+      .reduceGroups((a:Publication, b:Publication) => ConversionUtil.mergePublication(a,b))
+      .map(_._2)
+      .write.mode(SaveMode.Overwrite).save(s"$targetPath/magPublication")
 
-    val s:RDD[Publication] = spark.read.load(s"$workingPath/mag_publication").as[Publication]
-      .map(p=>Tuple2(p.getId, p)).rdd.reduceByKey((a:Publication, b:Publication) => ConversionUtil.mergePublication(a,b))
-    .map(_._2)
 
-    spark.createDataset(s).as[Publication].write.mode(SaveMode.Overwrite).save(s"$targetPath/magPublication")
 
   }
 }

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/generate_doiboost_params.json

@@ -1,7 +1,8 @@
 [
   {"paramName": "m",    "paramLongName":"master",                    "paramDescription": "the master name",                 "paramRequired": true},
   {"paramName": "hb",   "paramLongName":"hostedByMapPath",            "paramDescription": "the hosted By Map Path",         "paramRequired": true},
+  {"paramName": "oo",   "paramLongName":"openaireOrganizationPath",  "paramDescription": "the openaire Organization Path",  "paramRequired": true},
   {"paramName": "ap",   "paramLongName":"affiliationPath",            "paramDescription": "the Affliation Path",            "paramRequired": true},
   {"paramName": "pa",   "paramLongName":"paperAffiliationPath",      "paramDescription": "the paperAffiliation Path",       "paramRequired": true},
-  {"paramName": "w",    "paramLongName":"workingPath",                "paramDescription": "the Working Path",                "paramRequired": true}
+  {"paramName": "w",    "paramLongName":"workingPath",                "paramDescription": "the Working Path",               "paramRequired": true}
 ]

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml

@@ -107,7 +107,6 @@
     <action name="ResetMagWorkingPath">
         <fs>
             <delete path="${inputPathMAG}/dataset"/>
-            <delete path="${inputPathMAG}/process"/>
         </fs>
         <ok to="ConvertMagToDataset"/>
         <error to="Kill"/>

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml

@@ -27,6 +27,12 @@
             <name>hostedByMapPath</name>
             <description>the hostedByMap Path</description>
         </property>
+        <property>
+            <name>openaireOrganizationPath</name>
+            <description>the OpenAire Organizations Path</description>
+        </property>
+
+
         <property>
             <name>outputPath</name>
             <description>the Path of the sequence file action set</description>
@@ -42,7 +48,7 @@
         <!--    MAG Parameters    -->
         <property>
             <name>inputPathMAG</name>
-            <description>the MAG working path</description>
+            <description>the MAG input path</description>
         </property>
 
 
@@ -132,7 +138,7 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${inputPathMAG}/dataset</arg>
-            <arg>--workingPath</arg><arg>${inputPathMAG}/process_p</arg>
+            <arg>--workingPath</arg><arg>${workingPath}/MAG</arg>
             <arg>--targetPath</arg><arg>${workingPath}</arg>
             <arg>--master</arg><arg>yarn-cluster</arg>
         </spark>
@@ -214,6 +220,7 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
             </spark-opts>
             <arg>--hostedByMapPath</arg><arg>${hostedByMapPath}</arg>
+            <arg>--openaireOrganizationPath</arg><arg>${openaireOrganizationPath}</arg>
             <arg>--affiliationPath</arg><arg>${inputPathMAG}/dataset/Affiliations</arg>
             <arg>--paperAffiliationPath</arg><arg>${inputPathMAG}/dataset/PaperAuthorAffiliations</arg>
             <arg>--workingPath</arg><arg>${workingPath}</arg>

dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/crossref/CrossrefMappingTest.scala

@@ -612,4 +612,26 @@ class CrossrefMappingTest {
 
   }
 
+  @Test
+  def testMultipleURLs() :Unit = {
+    val json = Source.fromInputStream(getClass.getResourceAsStream("multiple_urls.json")).mkString
+
+
+    assertNotNull(json)
+    assertFalse(json.isEmpty);
+
+    val resultList: List[Oaf] = Crossref2Oaf.convert(json)
+
+    assertTrue(resultList.nonEmpty)
+
+
+    val item : Result = resultList.filter(p => p.isInstanceOf[Result]).head.asInstanceOf[Result]
+
+    assertEquals(1, item.getInstance().size())
+    assertEquals(1, item.getInstance().get(0).getUrl().size())
+    assertEquals("https://doi.org/10.1016/j.jas.2019.105013", item.getInstance().get(0).getUrl().get(0))
+    //println(mapper.writeValueAsString(item))
+
+  }
+
 }

dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/doiboost/crossref/multiple_urls.json

@@ -0,0 +1,614 @@
+
+{
+"indexed": {
+"date-parts": [
+[
+2021,
+10,
+31
+]
+],
+"date-time": "2021-10-31T15:48:01Z",
+"timestamp": 1635695281393
+},
+"reference-count": 39,
+"publisher": "Elsevier BV",
+"license": [
+{
+"start": {
+"date-parts": [
+[
+2019,
+12,
+1
+]
+],
+"date-time": "2019-12-01T00:00:00Z",
+"timestamp": 1575158400000
+},
+"content-version": "tdm",
+"delay-in-days": 0,
+"URL": "https://www.elsevier.com/tdm/userlicense/1.0/"
+},
+{
+"start": {
+"date-parts": [
+[
+2019,
+9,
+13
+]
+],
+"date-time": "2019-09-13T00:00:00Z",
+"timestamp": 1568332800000
+},
+"content-version": "vor",
+"delay-in-days": 0,
+"URL": "http://creativecommons.org/licenses/by/4.0/"
+}
+],
+"funder": [
+{
+"DOI": "10.13039/100001182",
+"name": "INSTAP",
+"doi-asserted-by": "publisher"
+},
+{
+"DOI": "10.13039/100014440",
+"name": "Ministry of Science, Innovation and Universities",
+"doi-asserted-by": "publisher",
+"award": [
+"RYC-2016-19637"
+]
+},
+{
+"DOI": "10.13039/100010661",
+"name": "European Union’s Horizon 2020",
+"doi-asserted-by": "publisher",
+"award": [
+"746446"
+]
+}
+],
+"content-domain": {
+"domain": [
+"elsevier.com",
+"sciencedirect.com"
+],
+"crossmark-restriction": true
+},
+"short-container-title": [
+"Journal of Archaeological Science"
+],
+"published-print": {
+"date-parts": [
+[
+2019,
+12
+]
+]
+},
+"DOI": "10.1016/j.jas.2019.105013",
+"type": "journal-article",
+"created": {
+"date-parts": [
+[
+2019,
+9,
+25
+]
+],
+"date-time": "2019-09-25T20:05:08Z",
+"timestamp": 1569441908000
+},
+"page": "105013",
+"update-policy": "http://dx.doi.org/10.1016/elsevier_cm_policy",
+"source": "Crossref",
+"is-referenced-by-count": 21,
+"title": [
+"A brave new world for archaeological survey: Automated machine learning-based potsherd detection using high-resolution drone imagery"
+],
+"prefix": "10.1016",
+"volume": "112",
+"author": [
+{
+"given": "H.A.",
+"family": "Orengo",
+"sequence": "first",
+"affiliation": [
+
+]
+},
+{
+"given": "A.",
+"family": "Garcia-Molsosa",
+"sequence": "additional",
+"affiliation": [
+
+]
+}
+],
+"member": "78",
+"reference": [
+{
+"key": "10.1016/j.jas.2019.105013_bib1",
+"doi-asserted-by": "crossref",
+"first-page": "85",
+"DOI": "10.1080/17538947.2016.1250829",
+"article-title": "Remote sensing heritage in a petabyte-scale: satellite data and heritage Earth Engine© applications",
+"volume": "10",
+"author": "Agapiou",
+"year": "2017",
+"journal-title": "Int. J. Digit. Earth"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib2",
+"series-title": "Extracting Meaning from Ploughsoil Assemblages",
+"first-page": "1",
+"article-title": "Extracting meaning from ploughsoil assemblages: assessments of the past, strategies for the future",
+"author": "Alcock",
+"year": "2000"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib3",
+"series-title": "Side-by-Side Survey. Comparative Regional Studies in the Mediterranean World",
+"first-page": "1",
+"article-title": "Introduction",
+"author": "Alcock",
+"year": "2004"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib4",
+"doi-asserted-by": "crossref",
+"first-page": "93",
+"DOI": "10.1111/j.1538-4632.1995.tb00338.x",
+"article-title": "Local indicators of spatial association—LISA",
+"volume": "27",
+"author": "Anselin",
+"year": "1995",
+"journal-title": "Geogr. Anal."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib5",
+"series-title": "Archaeological Survey",
+"author": "Banning",
+"year": "2002"
+},
+{
+"issue": "1/2",
+"key": "10.1016/j.jas.2019.105013_bib6",
+"doi-asserted-by": "crossref",
+"first-page": "123",
+"DOI": "10.2307/3181488",
+"article-title": "GIS, archaeological survey and landscape archaeology on the island of Kythera, Greece",
+"volume": "29",
+"author": "Bevan",
+"year": "2004",
+"journal-title": "J. Field Archaeol."
+},
+{
+"issue": "1",
+"key": "10.1016/j.jas.2019.105013_bib8",
+"doi-asserted-by": "crossref",
+"first-page": "5",
+"DOI": "10.1023/A:1010933404324",
+"article-title": "Random forests",
+"volume": "45",
+"author": "Breiman",
+"year": "2001",
+"journal-title": "Mach. Learn."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib9",
+"series-title": "Sampling in Contemporary British Archaeology",
+"author": "Cherry",
+"year": "1978"
+},
+{
+"issue": "3",
+"key": "10.1016/j.jas.2019.105013_bib10",
+"doi-asserted-by": "crossref",
+"first-page": "273",
+"DOI": "10.1016/0734-189X(84)90197-X",
+"article-title": "Segmentation of a high-resolution urban scene using texture operators",
+"volume": "25",
+"author": "Conners",
+"year": "1984",
+"journal-title": "Comput. Vis. Graph Image Process"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib11",
+"first-page": "31",
+"article-title": "Old land surfaces and modern ploughsoil: implications of recent work at Maxey, Cambridgeshire",
+"volume": "2",
+"author": "Crowther",
+"year": "1983",
+"journal-title": "Scott. Archaeol. Rev."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib12",
+"series-title": "Settlement Pattern Studies in the Americas: Fifty Years since Virú",
+"first-page": "203",
+"article-title": "Conclusions: the settlement pattern concept from an Americanist perspective",
+"author": "Fish",
+"year": "1999"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib13",
+"doi-asserted-by": "crossref",
+"first-page": "21",
+"DOI": "10.3390/geosciences9010021",
+"article-title": "Remote sensing and historical morphodynamics of alluvial plains. The 1909 indus flood and the city of Dera Gazhi Khan (province of Punjab, Pakistan)",
+"volume": "9",
+"author": "Garcia",
+"year": "2019",
+"journal-title": "Geosciences"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib14",
+"unstructured": "Georgiadis, M.; Garcia-Molsosa, A.; Orengo, H.A.; Kefalidou, E. and Kallintzi, K. In Preparation. APAX Project 2015-2018: A Preliminary Report. (Hesperia)."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib15",
+"series-title": "Geographical Information Systems and Landscape Archaeology",
+"first-page": "35",
+"article-title": "Regional survey and GIS: the boeotia project",
+"author": "Gillings",
+"year": "1999"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib16",
+"doi-asserted-by": "crossref",
+"first-page": "18",
+"DOI": "10.1016/j.rse.2017.06.031",
+"article-title": "Google Earth engine: planetary-scale geospatial analysis for everyone",
+"volume": "202",
+"author": "Gorelick",
+"year": "2017",
+"journal-title": "Remote Sens. Environ."
+},
+{
+"issue": "107",
+"key": "10.1016/j.jas.2019.105013_bib17",
+"doi-asserted-by": "crossref",
+"first-page": "177",
+"DOI": "10.1111/j.0031-868X.2004.00278.x",
+"article-title": "Photogrammetric reconstruction of the great buddha of Bamiyan, Afghanistan",
+"volume": "19",
+"author": "Grün",
+"year": "2004",
+"journal-title": "Photogramm. Rec."
+},
+{
+"issue": "6",
+"key": "10.1016/j.jas.2019.105013_bib18",
+"doi-asserted-by": "crossref",
+"first-page": "610",
+"DOI": "10.1109/TSMC.1973.4309314",
+"article-title": "Textural features for image classification",
+"author": "Haralick",
+"year": "1973",
+"journal-title": "IEEE Trans. Syst., Man, Cybernet., SMC-3"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib19",
+"doi-asserted-by": "crossref",
+"first-page": "76",
+"DOI": "10.1558/jmea.v14i1.76",
+"article-title": "Excavating to excess? Implications of the last decade of archaeology in Israel",
+"volume": "14",
+"author": "Kletter",
+"year": "2001",
+"journal-title": "J. Mediterr. Archaeol."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib20",
+"first-page": "299",
+"article-title": "Testing Google Earth Engine for the automatic identification and vectorization of archaeological features: a case study from Faynan, Jordan",
+"volume": "15",
+"author": "Liss",
+"year": "2017",
+"journal-title": "J. Archaeol. Sci.: Report"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib21",
+"series-title": "Geographical Information Systems and Landscape Archaeology",
+"first-page": "55",
+"article-title": "Towards a methodology for modelling surface survey data: the sangro valley project",
+"author": "Lock",
+"year": "1999"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib22",
+"series-title": "Extracting Meaning from Ploughsoil Assemblages",
+"first-page": "5",
+"article-title": "Methods of collection recording and quantification",
+"author": "Mattingly",
+"year": "2000"
+},
+{
+"issue": "14",
+"key": "10.1016/j.jas.2019.105013_bib23",
+"doi-asserted-by": "crossref",
+"first-page": "E778",
+"DOI": "10.1073/pnas.1115472109",
+"article-title": "Mapping patterns of long-term settlement in Northern Mesopotamia at a large scale",
+"volume": "109",
+"author": "Menze",
+"year": "2012",
+"journal-title": "Proc. Natl. Acad. Sci."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib24",
+"doi-asserted-by": "crossref",
+"first-page": "80",
+"DOI": "10.1016/j.jas.2015.04.002",
+"article-title": "A supervised machine-learning approach towards geochemical predictive modelling in archaeology",
+"volume": "59",
+"author": "Oonk",
+"year": "2015",
+"journal-title": "J. Archaeol. Sci."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib25",
+"doi-asserted-by": "crossref",
+"first-page": "49",
+"DOI": "10.1016/j.isprsjprs.2012.07.005",
+"article-title": "Combining terrestrial stereophotogrammetry, DGPS and GIS-based 3D voxel modelling in the volumetric recording of archaeological features",
+"volume": "76",
+"author": "Orengo",
+"year": "2013",
+"journal-title": "ISPRS J. Photogrammetry Remote Sens."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib26",
+"doi-asserted-by": "crossref",
+"first-page": "100",
+"DOI": "10.1016/j.jas.2015.10.008",
+"article-title": "Photogrammetric re-discovery of the Eastern Thessalian hidden long-term landscapes",
+"volume": "64",
+"author": "Orengo",
+"year": "2015",
+"journal-title": "J. Archaeol. Sci."
+},
+{
+"issue": "3",
+"key": "10.1016/j.jas.2019.105013_bib27",
+"doi-asserted-by": "crossref",
+"first-page": "479",
+"DOI": "10.3764/aja.122.3.0479",
+"article-title": "Towards a definition of Minoan agro-pastoral landscapes: results of the survey at Palaikastro (Crete)",
+"volume": "122",
+"author": "Orengo",
+"year": "2018",
+"journal-title": "Am. J. Archaeol."
+},
+{
+"issue": "7",
+"key": "10.1016/j.jas.2019.105013_bib28",
+"doi-asserted-by": "crossref",
+"first-page": "735",
+"DOI": "10.3390/rs9070735",
+"article-title": "Large-scale, multi-temporal remote sensing of palaeo-river networks: a case study from Northwest India and its implications for the Indus civilisation",
+"volume": "9",
+"author": "Orengo",
+"year": "2017",
+"journal-title": "Remote Sens."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib29",
+"doi-asserted-by": "crossref",
+"first-page": "1361",
+"DOI": "10.1002/esp.4317",
+"article-title": "Multi-scale relief model (MSRM): a new algorithm for the visualization of subtle topographic change of variable size in digital elevation models",
+"volume": "43",
+"author": "Orengo",
+"year": "2018",
+"journal-title": "Earth Surf. Process. Landforms"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib30",
+"series-title": "Submitted to Proceedings of the National Academy of Sciences",
+"article-title": "Living on the edge of the desert: automated detection of archaeological mounds in Cholistan (Pakistan) using machine learning classification of multi-sensor multi-temporal satellite data",
+"author": "Orengo",
+"year": "2019"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib31",
+"first-page": "154",
+"article-title": "How many trees in a random forest?",
+"volume": "vol. 7376",
+"author": "Oshiro",
+"year": "2012"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib32",
+"article-title": "Decision-making in modern surveys",
+"volume": "ume 1",
+"author": "Plog",
+"year": "1978"
+},
+{
+"issue": "4",
+"key": "10.1016/j.jas.2019.105013_bib33",
+"doi-asserted-by": "crossref",
+"first-page": "100",
+"DOI": "10.3390/geosciences7040100",
+"article-title": "From above and on the ground: geospatial methods for recording endangered archaeology in the Middle East and north africa",
+"volume": "7",
+"author": "Rayne",
+"year": "2017",
+"journal-title": "Geosciences"
+},
+{
+"issue": "1",
+"key": "10.1016/j.jas.2019.105013_bib34",
+"doi-asserted-by": "crossref",
+"first-page": "1",
+"DOI": "10.1080/00438243.1978.9979712",
+"article-title": "The design of archaeological surveys",
+"volume": "10",
+"author": "Schiffer",
+"year": "1978",
+"journal-title": "World Archaeol."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib35",
+"series-title": "Experiments in the Collection and Analysis of Archaeological Survey Data: the East Hampshire Survey",
+"author": "Shennan",
+"year": "1985"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib36",
+"doi-asserted-by": "crossref",
+"first-page": "1066",
+"DOI": "10.1016/j.culher.2016.06.006",
+"article-title": "Drones over Mediterranean landscapes. The potential of small UAV's (drones) for site detection and heritage management in archaeological survey projects: a case study from Le Pianelle in the Tappino Valley, Molise (Italy)",
+"volume": "22",
+"author": "Stek",
+"year": "2016",
+"journal-title": "J. Cult. Herit."
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib37",
+"series-title": "Side-by-Side Survey. Comparative Regional Studies in the Mediterranean World",
+"first-page": "65",
+"article-title": "Side-by-side and back to front: exploring intra-regional latitudinal and longitudinal comparability in survey data. Three case studies from Metaponto, southern Italy",
+"author": "Thomson",
+"year": "2004"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib38",
+"series-title": "Digital Discovery. Exploring New Frontiers in Human Heritage. Computer Applications and Quantitative Methods in Archaeology",
+"article-title": "Computer vision and machine learning for archaeology",
+"author": "van der Maaten",
+"year": "2007"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib39",
+"doi-asserted-by": "crossref",
+"first-page": "1114",
+"DOI": "10.1111/j.1475-4754.2012.00667.x",
+"article-title": "Computer vision-based orthophoto mapping of complex archaeological sites: the ancient quarry of Pitaranha (Portugal-Spain)",
+"volume": "54",
+"author": "Verhoeven",
+"year": "2012",
+"journal-title": "Archaeometry"
+},
+{
+"key": "10.1016/j.jas.2019.105013_bib40",
+"series-title": "A Guide for Salvage Archeology",
+"author": "Wendorf",
+"year": "1962"
+}
+],
+"container-title": [
+"Journal of Archaeological Science"
+],
+"original-title": [
+
+],
+"language": "en",
+"link": [
+{
+"URL": "https://api.elsevier.com/content/article/PII:S0305440319301001?httpAccept=text/xml",
+"content-type": "text/xml",
+"content-version": "vor",
+"intended-application": "text-mining"
+},
+{
+"URL": "https://api.elsevier.com/content/article/PII:S0305440319301001?httpAccept=text/plain",
+"content-type": "text/plain",
+"content-version": "vor",
+"intended-application": "text-mining"
+}
+],
+"deposited": {
+"date-parts": [
+[
+2019,
+11,
+25
+]
+],
+"date-time": "2019-11-25T06:46:34Z",
+"timestamp": 1574664394000
+},
+"score": 1,
+"subtitle": [
+
+],
+"short-title": [
+
+],
+"issued": {
+"date-parts": [
+[
+2019,
+12
+]
+]
+},
+"references-count": 39,
+"alternative-id": [
+"S0305440319301001"
+],
+"URL": "http://dx.doi.org/10.1016/j.jas.2019.105013",
+"relation": {
+
+},
+"ISSN": [
+"0305-4403"
+],
+"issn-type": [
+{
+"value": "0305-4403",
+"type": "print"
+}
+],
+"subject": [
+"Archaeology",
+"Archaeology"
+],
+"published": {
+"date-parts": [
+[
+2019,
+12
+]
+]
+},
+"assertion": [
+{
+"value": "Elsevier",
+"name": "publisher",
+"label": "This article is maintained by"
+},
+{
+"value": "A brave new world for archaeological survey: Automated machine learning-based potsherd detection using high-resolution drone imagery",
+"name": "articletitle",
+"label": "Article Title"
+},
+{
+"value": "Journal of Archaeological Science",
+"name": "journaltitle",
+"label": "Journal Title"
+},
+{
+"value": "https://doi.org/10.1016/j.jas.2019.105013",
+"name": "articlelink",
+"label": "CrossRef DOI link to publisher maintained version"
+},
+{
+"value": "article",
+"name": "content_type",
+"label": "Content Type"
+},
+{
+"value": "© 2019 The Authors. Published by Elsevier Ltd.",
+"name": "copyright",
+"label": "Copyright"
+}
+],
+"article-number": "105013"
+}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java

@@ -25,6 +25,24 @@ public class PropagationConstant {
 	private PropagationConstant() {
 	}
 
+	public static final String DOI = "doi";
+	public static final String REF_DOI = ".refs";
+
+	public static final String UPDATE_DATA_INFO_TYPE = "update";
+	public static final String UPDATE_SUBJECT_FOS_CLASS_ID = "subject:fos";
+	public static final String UPDATE_CLASS_NAME = "Inferred  by OpenAIRE";
+	public static final String UPDATE_MEASURE_BIP_CLASS_ID = "measure:bip";
+
+	public static final String FOS_CLASS_ID = "FOS";
+	public static final String FOS_CLASS_NAME = "Fields of Science and Technology classification";
+
+	public static final String OPENCITATIONS_CLASSID = "sysimport:crosswalk:opencitations";
+	public static final String OPENCITATIONS_CLASSNAME = "Imported from OpenCitations";
+	public static final String ID_PREFIX = "50|doi_________::";
+	public static final String OC_TRUST = "0.91";
+
+	public final static String NULL = "NULL";
+
 	public static final String INSTITUTIONAL_REPO_TYPE = "pubsrepository::institutional";
 
 	public static final String PROPAGATION_DATA_INFO_TYPE = "propagation";
@@ -75,10 +93,25 @@ public class PropagationConstant {
 
 	public static DataInfo getDataInfo(
 		String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema) {
+
+		return getDataInfo(inference_provenance, inference_class_id, inference_class_name, qualifierSchema, "0.85");
+	}
+
+	public static DataInfo getDataInfo(
+		String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema,
+		String trust) {
+		return getDataInfo(
+			inference_provenance, inference_class_id, inference_class_name, qualifierSchema, trust, true);
+
+	}
+
+	public static DataInfo getDataInfo(
+		String inference_provenance, String inference_class_id, String inference_class_name, String qualifierSchema,
+		String trust, boolean inferred) {
 		DataInfo di = new DataInfo();
-		di.setInferred(true);
+		di.setInferred(inferred);
 		di.setDeletedbyinference(false);
-		di.setTrust("0.85");
+		di.setTrust(trust);
 		di.setInferenceprovenance(inference_provenance);
 		di.setProvenanceaction(getQualifier(inference_class_id, inference_class_name, qualifierSchema));
 		return di;

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Constants.java

@@ -5,37 +5,40 @@ import java.util.Map;
 
 import com.google.common.collect.Maps;
 
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+
 public class Constants {
 
-	public static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
-	public static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
+	protected static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
+	protected static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
 
 	public static final String INFERRED = "Inferred by OpenAIRE";
+	public static final String CABF2 = "c_abf2";
 
 	public static final String HARVESTED = "Harvested";
 	public static final String DEFAULT_TRUST = "0.9";
 	public static final String USER_CLAIM = "Linked by user";
 
-	public static String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
+	public static final String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
 
-	public static String ZENODO_COMMUNITY_PREFIX = "https://zenodo.org/communities/";
+	public static final String ZENODO_COMMUNITY_PREFIX = "https://zenodo.org/communities/";
 
-	public static String RESEARCH_COMMUNITY = "Research Community";
+	public static final String RESEARCH_COMMUNITY = "Research Community";
 
-	public static String RESEARCH_INFRASTRUCTURE = "Research Infrastructure/Initiative";
+	public static final String RESEARCH_INFRASTRUCTURE = "Research Infrastructure/Initiative";
 
 	static {
-		accessRightsCoarMap.put("OPEN", "c_abf2");
+		accessRightsCoarMap.put(ModelConstants.ACCESS_RIGHT_OPEN, CABF2);
 		accessRightsCoarMap.put("RESTRICTED", "c_16ec");
-		accessRightsCoarMap.put("OPEN SOURCE", "c_abf2");
-		accessRightsCoarMap.put("CLOSED", "c_14cb");
-		accessRightsCoarMap.put("EMBARGO", "c_f1cf");
+		accessRightsCoarMap.put("OPEN SOURCE", CABF2);
+		accessRightsCoarMap.put(ModelConstants.ACCESS_RIGHT_CLOSED, "c_14cb");
+		accessRightsCoarMap.put(ModelConstants.ACCESS_RIGHT_EMBARGO, "c_f1cf");
 	}
 
 	static {
-		coarCodeLabelMap.put("c_abf2", "OPEN");
+		coarCodeLabelMap.put(CABF2, ModelConstants.ACCESS_RIGHT_OPEN);
 		coarCodeLabelMap.put("c_16ec", "RESTRICTED");
-		coarCodeLabelMap.put("c_14cb", "CLOSED");
+		coarCodeLabelMap.put("c_14cb", ModelConstants.ACCESS_RIGHT_CLOSED);
 		coarCodeLabelMap.put("c_f1cf", "EMBARGO");
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/DumpProducts.java

@@ -11,12 +11,14 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.FilterFunction;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SaveMode;
 import org.apache.spark.sql.SparkSession;
 
 import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
+import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
 import eu.dnetlib.dhp.schema.oaf.*;
 
 /**
@@ -37,7 +39,8 @@ public class DumpProducts implements Serializable {
 			isSparkSessionManaged,
 			spark -> {
 				Utils.removeOutputDir(spark, outputPath);
-				execDump(spark, inputPath, outputPath, communityMapPath, inputClazz, outputClazz, dumpType);
+				execDump(
+					spark, inputPath, outputPath, communityMapPath, inputClazz, outputClazz, dumpType);
 			});
 	}
 
@@ -55,7 +58,7 @@ public class DumpProducts implements Serializable {
 		Utils
 			.readPath(spark, inputPath, inputClazz)
 			.map((MapFunction<I, O>) value -> execMap(value, communityMap, dumpType), Encoders.bean(outputClazz))
-			.filter(Objects::nonNull)
+			.filter((FilterFunction<O>) value -> value != null)
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")
@@ -65,7 +68,7 @@ public class DumpProducts implements Serializable {
 
 	private static <I extends OafEntity, O extends eu.dnetlib.dhp.schema.dump.oaf.Result> O execMap(I value,
 		CommunityMap communityMap,
-		String dumpType) {
+		String dumpType) throws NoAvailableEntityTypeException {
 
 		Optional<DataInfo> odInfo = Optional.ofNullable(value.getDataInfo());
 		if (odInfo.isPresent()) {
@@ -89,11 +92,11 @@ public class DumpProducts implements Serializable {
 					return c.getId();
 				}
 				if (c.getId().contains("::") && communities.contains(c.getId().substring(0, c.getId().indexOf("::")))) {
-					return c.getId().substring(0, 3);
+					return c.getId().substring(0, c.getId().indexOf("::"));
 				}
 				return null;
 			}).filter(Objects::nonNull).collect(Collectors.toList());
-			if (toDumpFor.size() == 0) {
+			if (toDumpFor.isEmpty()) {
 				return null;
 			}
 		}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/MakeTar.java

@@ -57,16 +57,16 @@ public class MakeTar implements Serializable {
 	public static void makeTArArchive(FileSystem fileSystem, String inputPath, String outputPath, int gBperSplit)
 		throws IOException {
 
-		RemoteIterator<LocatedFileStatus> dir_iterator = fileSystem.listLocatedStatus(new Path(inputPath));
+		RemoteIterator<LocatedFileStatus> dirIterator = fileSystem.listLocatedStatus(new Path(inputPath));
 
-		while (dir_iterator.hasNext()) {
-			LocatedFileStatus fileStatus = dir_iterator.next();
+		while (dirIterator.hasNext()) {
+			LocatedFileStatus fileStatus = dirIterator.next();
 
 			Path p = fileStatus.getPath();
-			String p_string = p.toString();
-			String entity = p_string.substring(p_string.lastIndexOf("/") + 1);
+			String pathString = p.toString();
+			String entity = pathString.substring(pathString.lastIndexOf("/") + 1);
 
-			MakeTarArchive.tarMaxSize(fileSystem, p_string, outputPath + "/" + entity, entity, gBperSplit);
+			MakeTarArchive.tarMaxSize(fileSystem, pathString, outputPath + "/" + entity, entity, gBperSplit);
 		}
 
 	}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/QueryInformationSystem.java

@@ -18,10 +18,10 @@ public class QueryInformationSystem {
 
 	private ISLookUpService isLookUp;
 
-	private static final String XQUERY = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') "
+	private static final String XQUERY_ALL = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') "
 		+
 		"  where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] " +
-		" and ($x//context/param[./@name = 'status']/text() = 'manager'  or $x//context/param[./@name = 'status']/text() = 'all') "
+		" and ($x//context/param[./@name = 'status']/text() = 'all') "
 		+
 		"  return " +
 		"<community> " +
@@ -29,9 +29,22 @@ public class QueryInformationSystem {
 		"{$x//CONFIGURATION/context/@label}" +
 		"</community>";
 
-	public CommunityMap getCommunityMap()
+	private static final String XQUERY_CI = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') "
+		+
+		"  where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] " +
+		" and  $x//CONFIGURATION/context[./@id=%s]  "
+		+
+		"  return " +
+		"<community> " +
+		"{$x//CONFIGURATION/context/@id}" +
+		"{$x//CONFIGURATION/context/@label}" +
+		"</community>";
+
+	public CommunityMap getCommunityMap(boolean singleCommunity, String communityId)
 		throws ISLookUpException, DocumentException, SAXException {
-		return getMap(isLookUp.quickSearchProfile(XQUERY));
+		if (singleCommunity)
+			return getMap(isLookUp.quickSearchProfile(XQUERY_CI.replace("%s", "'" + communityId + "'")));
+		return getMap(isLookUp.quickSearchProfile(XQUERY_ALL));
 
 	}
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java

@@ -7,21 +7,28 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
+import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.dump.oaf.*;
+import eu.dnetlib.dhp.schema.dump.oaf.AccessRight;
+import eu.dnetlib.dhp.schema.dump.oaf.Author;
+import eu.dnetlib.dhp.schema.dump.oaf.Country;
+import eu.dnetlib.dhp.schema.dump.oaf.GeoLocation;
+import eu.dnetlib.dhp.schema.dump.oaf.Instance;
+import eu.dnetlib.dhp.schema.dump.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.dump.oaf.OpenAccessRoute;
+import eu.dnetlib.dhp.schema.dump.oaf.Qualifier;
+import eu.dnetlib.dhp.schema.dump.oaf.Result;
 import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityInstance;
 import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
 import eu.dnetlib.dhp.schema.dump.oaf.community.Context;
 import eu.dnetlib.dhp.schema.dump.oaf.graph.GraphResult;
-import eu.dnetlib.dhp.schema.oaf.DataInfo;
-import eu.dnetlib.dhp.schema.oaf.Field;
-import eu.dnetlib.dhp.schema.oaf.Journal;
-import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
+import eu.dnetlib.dhp.schema.oaf.*;
 
 public class ResultMapper implements Serializable {
 
 	public static <E extends eu.dnetlib.dhp.schema.oaf.OafEntity> Result map(
-		E in, Map<String, String> communityMap, String dumpType) {
+		E in, Map<String, String> communityMap, String dumpType) throws NoAvailableEntityTypeException {
 
 		Result out;
 		if (Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
@@ -33,113 +40,28 @@ public class ResultMapper implements Serializable {
 		eu.dnetlib.dhp.schema.oaf.Result input = (eu.dnetlib.dhp.schema.oaf.Result) in;
 		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> ort = Optional.ofNullable(input.getResulttype());
 		if (ort.isPresent()) {
-			switch (ort.get().getClassid()) {
-				case "publication":
-					Optional<Journal> journal = Optional
-						.ofNullable(((eu.dnetlib.dhp.schema.oaf.Publication) input).getJournal());
-					if (journal.isPresent()) {
-						Journal j = journal.get();
-						Container c = new Container();
-						c.setConferencedate(j.getConferencedate());
-						c.setConferenceplace(j.getConferenceplace());
-						c.setEdition(j.getEdition());
-						c.setEp(j.getEp());
-						c.setIss(j.getIss());
-						c.setIssnLinking(j.getIssnLinking());
-						c.setIssnOnline(j.getIssnOnline());
-						c.setIssnPrinted(j.getIssnPrinted());
-						c.setName(j.getName());
-						c.setSp(j.getSp());
-						c.setVol(j.getVol());
-						out.setContainer(c);
-						out.setType(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE.getClassname());
-					}
-					break;
-				case "dataset":
-					eu.dnetlib.dhp.schema.oaf.Dataset id = (eu.dnetlib.dhp.schema.oaf.Dataset) input;
-					Optional.ofNullable(id.getSize()).ifPresent(v -> out.setSize(v.getValue()));
-					Optional.ofNullable(id.getVersion()).ifPresent(v -> out.setVersion(v.getValue()));
-
-					out
-						.setGeolocation(
-							Optional
-								.ofNullable(id.getGeolocation())
-								.map(
-									igl -> igl
-										.stream()
-										.filter(Objects::nonNull)
-										.map(gli -> {
-											GeoLocation gl = new GeoLocation();
-											gl.setBox(gli.getBox());
-											gl.setPlace(gli.getPlace());
-											gl.setPoint(gli.getPoint());
-											return gl;
-										})
-										.collect(Collectors.toList()))
-								.orElse(null));
-
-					out.setType(ModelConstants.DATASET_DEFAULT_RESULTTYPE.getClassname());
-					break;
-				case "software":
-
-					eu.dnetlib.dhp.schema.oaf.Software is = (eu.dnetlib.dhp.schema.oaf.Software) input;
-					Optional
-						.ofNullable(is.getCodeRepositoryUrl())
-						.ifPresent(value -> out.setCodeRepositoryUrl(value.getValue()));
-					Optional
-						.ofNullable(is.getDocumentationUrl())
-						.ifPresent(
-							value -> out
-								.setDocumentationUrl(
-									value
-										.stream()
-										.map(Field::getValue)
-										.collect(Collectors.toList())));
-
-					Optional
-						.ofNullable(is.getProgrammingLanguage())
-						.ifPresent(value -> out.setProgrammingLanguage(value.getClassid()));
-
-					out.setType(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE.getClassname());
-					break;
-				case "other":
-
-					eu.dnetlib.dhp.schema.oaf.OtherResearchProduct ir = (eu.dnetlib.dhp.schema.oaf.OtherResearchProduct) input;
-					out
-						.setContactgroup(
-							Optional
-								.ofNullable(ir.getContactgroup())
-								.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
-								.orElse(null));
-
-					out
-						.setContactperson(
-							Optional
-								.ofNullable(ir.getContactperson())
-								.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
-								.orElse(null));
-					out
-						.setTool(
-							Optional
-								.ofNullable(ir.getTool())
-								.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
-								.orElse(null));
-
-					out.setType(ModelConstants.ORP_DEFAULT_RESULTTYPE.getClassname());
-
-					break;
-			}
+			try {
+
+				addTypeSpecificInformation(out, input, ort);
+				Optional<List<Measure>> mes = Optional.ofNullable(input.getMeasures());
+				if (mes.isPresent()) {
+					List<KeyValue> measure = new ArrayList<>();
+					mes
+						.get()
+						.forEach(
+							m -> m.getUnit().forEach(u -> measure.add(KeyValue.newInstance(m.getId(), u.getValue()))));
+					out.setMeasures(measure);
+				}
 
-			Optional
-				.ofNullable(input.getAuthor())
-				.ifPresent(
-					ats -> out.setAuthor(ats.stream().map(ResultMapper::getAuthor).collect(Collectors.toList())));
+				Optional
+					.ofNullable(input.getAuthor())
+					.ifPresent(
+						ats -> out.setAuthor(ats.stream().map(ResultMapper::getAuthor).collect(Collectors.toList())));
 
-			// I do not map Access Right UNKNOWN or OTHER
+				// I do not map Access Right UNKNOWN or OTHER
 
-			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oar = Optional.ofNullable(input.getBestaccessright());
-			if (oar.isPresent()) {
-				if (Constants.accessRightsCoarMap.containsKey(oar.get().getClassid())) {
+				Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oar = Optional.ofNullable(input.getBestaccessright());
+				if (oar.isPresent() && Constants.accessRightsCoarMap.containsKey(oar.get().getClassid())) {
 					String code = Constants.accessRightsCoarMap.get(oar.get().getClassid());
 					out
 						.setBestaccessright(
@@ -149,226 +71,340 @@ public class ResultMapper implements Serializable {
 									Constants.coarCodeLabelMap.get(code),
 									Constants.COAR_ACCESS_RIGHT_SCHEMA));
 				}
-			}
 
-			final List<String> contributorList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getContributor())
-				.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
-			out.setContributor(contributorList);
-
-			Optional
-				.ofNullable(input.getCountry())
-				.ifPresent(
-					value -> out
-						.setCountry(
-							value
-								.stream()
-								.map(
-									c -> {
-										if (c.getClassid().equals((ModelConstants.UNKNOWN))) {
-											return null;
-										}
-										Country country = new Country();
-										country.setCode(c.getClassid());
-										country.setLabel(c.getClassname());
-										Optional
-											.ofNullable(c.getDataInfo())
-											.ifPresent(
-												provenance -> country
-													.setProvenance(
-														Provenance
-															.newInstance(
-																provenance
-																	.getProvenanceaction()
-																	.getClassname(),
-																c.getDataInfo().getTrust())));
-										return country;
-									})
-								.filter(Objects::nonNull)
-								.collect(Collectors.toList())));
-
-			final List<String> coverageList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getCoverage())
-				.ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue())));
-			out.setCoverage(coverageList);
-
-			out.setDateofcollection(input.getDateofcollection());
-
-			final List<String> descriptionList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getDescription())
-				.ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue())));
-			out.setDescription(descriptionList);
-			Optional<Field<String>> oStr = Optional.ofNullable(input.getEmbargoenddate());
-			if (oStr.isPresent()) {
-				out.setEmbargoenddate(oStr.get().getValue());
-			}
+				final List<String> contributorList = new ArrayList<>();
+				Optional
+					.ofNullable(input.getContributor())
+					.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
+				out.setContributor(contributorList);
+
+				Optional
+					.ofNullable(input.getCountry())
+					.ifPresent(
+						value -> out
+							.setCountry(
+								value
+									.stream()
+									.map(
+										c -> {
+											if (c.getClassid().equals((ModelConstants.UNKNOWN))) {
+												return null;
+											}
+											Country country = new Country();
+											country.setCode(c.getClassid());
+											country.setLabel(c.getClassname());
+											Optional
+												.ofNullable(c.getDataInfo())
+												.ifPresent(
+													provenance -> country
+														.setProvenance(
+															Provenance
+																.newInstance(
+																	provenance
+																		.getProvenanceaction()
+																		.getClassname(),
+																	c.getDataInfo().getTrust())));
+											return country;
+										})
+									.filter(Objects::nonNull)
+									.collect(Collectors.toList())));
+
+				final List<String> coverageList = new ArrayList<>();
+				Optional
+					.ofNullable(input.getCoverage())
+					.ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue())));
+				out.setCoverage(coverageList);
+
+				out.setDateofcollection(input.getDateofcollection());
+
+				final List<String> descriptionList = new ArrayList<>();
+				Optional
+					.ofNullable(input.getDescription())
+					.ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue())));
+				out.setDescription(descriptionList);
+				Optional<Field<String>> oStr = Optional.ofNullable(input.getEmbargoenddate());
+				if (oStr.isPresent()) {
+					out.setEmbargoenddate(oStr.get().getValue());
+				}
 
-			final List<String> formatList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getFormat())
-				.ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue())));
-			out.setFormat(formatList);
-			out.setId(input.getId());
-			out.setOriginalId(input.getOriginalId());
-
-			Optional<List<eu.dnetlib.dhp.schema.oaf.Instance>> oInst = Optional
-				.ofNullable(input.getInstance());
-
-			if (oInst.isPresent()) {
-				if (Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
-					((GraphResult) out)
-						.setInstance(
-							oInst.get().stream().map(ResultMapper::getGraphInstance).collect(Collectors.toList()));
-				} else {
-					((CommunityResult) out)
-						.setInstance(
-							oInst.get().stream().map(ResultMapper::getCommunityInstance).collect(Collectors.toList()));
+				final List<String> formatList = new ArrayList<>();
+				Optional
+					.ofNullable(input.getFormat())
+					.ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue())));
+				out.setFormat(formatList);
+				out.setId(input.getId());
+				out.setOriginalId(input.getOriginalId());
+
+				Optional<List<eu.dnetlib.dhp.schema.oaf.Instance>> oInst = Optional
+					.ofNullable(input.getInstance());
+
+				if (oInst.isPresent()) {
+					if (Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
+						((GraphResult) out)
+							.setInstance(
+								oInst.get().stream().map(ResultMapper::getGraphInstance).collect(Collectors.toList()));
+					} else {
+						((CommunityResult) out)
+							.setInstance(
+								oInst
+									.get()
+									.stream()
+									.map(ResultMapper::getCommunityInstance)
+									.collect(Collectors.toList()));
+					}
 				}
-			}
 
-			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oL = Optional.ofNullable(input.getLanguage());
-			if (oL.isPresent()) {
-				eu.dnetlib.dhp.schema.oaf.Qualifier language = oL.get();
-				out.setLanguage(Qualifier.newInstance(language.getClassid(), language.getClassname()));
-			}
-			Optional<Long> oLong = Optional.ofNullable(input.getLastupdatetimestamp());
-			if (oLong.isPresent()) {
-				out.setLastupdatetimestamp(oLong.get());
-			}
-			Optional<List<StructuredProperty>> otitle = Optional.ofNullable(input.getTitle());
-			if (otitle.isPresent()) {
-				List<StructuredProperty> iTitle = otitle
-					.get()
-					.stream()
-					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title"))
-					.collect(Collectors.toList());
-				if (iTitle.size() > 0) {
-					out.setMaintitle(iTitle.get(0).getValue());
+				Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oL = Optional.ofNullable(input.getLanguage());
+				if (oL.isPresent()) {
+					eu.dnetlib.dhp.schema.oaf.Qualifier language = oL.get();
+					out.setLanguage(Qualifier.newInstance(language.getClassid(), language.getClassname()));
+				}
+				Optional<Long> oLong = Optional.ofNullable(input.getLastupdatetimestamp());
+				if (oLong.isPresent()) {
+					out.setLastupdatetimestamp(oLong.get());
 				}
+				Optional<List<StructuredProperty>> otitle = Optional.ofNullable(input.getTitle());
+				if (otitle.isPresent()) {
+					List<StructuredProperty> iTitle = otitle
+						.get()
+						.stream()
+						.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title"))
+						.collect(Collectors.toList());
+					if (!iTitle.isEmpty()) {
+						out.setMaintitle(iTitle.get(0).getValue());
+					}
+
+					iTitle = otitle
+						.get()
+						.stream()
+						.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle"))
+						.collect(Collectors.toList());
+					if (!iTitle.isEmpty()) {
+						out.setSubtitle(iTitle.get(0).getValue());
+					}
 
-				iTitle = otitle
-					.get()
-					.stream()
-					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle"))
-					.collect(Collectors.toList());
-				if (iTitle.size() > 0) {
-					out.setSubtitle(iTitle.get(0).getValue());
 				}
 
-			}
+				Optional
+					.ofNullable(input.getPid())
+					.ifPresent(
+						value -> out
+							.setPid(
+								value
+									.stream()
+									.map(
+										p -> ControlledField
+											.newInstance(p.getQualifier().getClassid(), p.getValue()))
+									.collect(Collectors.toList())));
+
+				oStr = Optional.ofNullable(input.getDateofacceptance());
+				if (oStr.isPresent()) {
+					out.setPublicationdate(oStr.get().getValue());
+				}
+				oStr = Optional.ofNullable(input.getPublisher());
+				if (oStr.isPresent()) {
+					out.setPublisher(oStr.get().getValue());
+				}
 
-			List<ControlledField> pids = new ArrayList<>();
-			Optional
-				.ofNullable(input.getPid())
-				.ifPresent(
-					value -> value
-						.stream()
-						.forEach(
-							p -> pids
-								.add(
-									ControlledField
-										.newInstance(p.getQualifier().getClassid(), p.getValue()))));
-			out.setPid(pids);
-			oStr = Optional.ofNullable(input.getDateofacceptance());
-			if (oStr.isPresent()) {
-				out.setPublicationdate(oStr.get().getValue());
-			}
-			oStr = Optional.ofNullable(input.getPublisher());
-			if (oStr.isPresent()) {
-				out.setPublisher(oStr.get().getValue());
-			}
+				Optional
+					.ofNullable(input.getSource())
+					.ifPresent(
+						value -> out.setSource(value.stream().map(Field::getValue).collect(Collectors.toList())));
 
-			List<String> sourceList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getSource())
-				.ifPresent(value -> value.stream().forEach(s -> sourceList.add(s.getValue())));
-			// out.setSource(input.getSource().stream().map(s -> s.getValue()).collect(Collectors.toList()));
-			List<Subject> subjectList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getSubject())
-				.ifPresent(
-					value -> value
-						.forEach(s -> subjectList.add(getSubject(s))));
-
-			out.setSubjects(subjectList);
-
-			out.setType(input.getResulttype().getClassid());
-		}
+				List<Subject> subjectList = new ArrayList<>();
+				Optional
+					.ofNullable(input.getSubject())
+					.ifPresent(
+						value -> value
+							.forEach(s -> subjectList.add(getSubject(s))));
 
-		if (!Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
-			((CommunityResult) out)
-				.setCollectedfrom(
-					input
-						.getCollectedfrom()
-						.stream()
-						.map(cf -> KeyValue.newInstance(cf.getKey(), cf.getValue()))
-						.collect(Collectors.toList()));
-
-			Set<String> communities = communityMap.keySet();
-			List<Context> contextList = Optional
-				.ofNullable(
-					input
-						.getContext())
-				.map(
-					value -> value
-						.stream()
-						.map(c -> {
-							String community_id = c.getId();
-							if (community_id.indexOf("::") > 0) {
-								community_id = community_id.substring(0, community_id.indexOf("::"));
-							}
-							if (communities.contains(community_id)) {
-								Context context = new Context();
-								context.setCode(community_id);
-								context.setLabel(communityMap.get(community_id));
-								Optional<List<DataInfo>> dataInfo = Optional.ofNullable(c.getDataInfo());
-								if (dataInfo.isPresent()) {
-									List<Provenance> provenance = new ArrayList<>();
-									provenance
-										.addAll(
-											dataInfo
-												.get()
-												.stream()
-												.map(
-													di -> Optional
-														.ofNullable(di.getProvenanceaction())
+				out.setSubjects(subjectList);
+
+				out.setType(input.getResulttype().getClassid());
+
+				if (!Constants.DUMPTYPE.COMPLETE.getType().equals(dumpType)) {
+					((CommunityResult) out)
+						.setCollectedfrom(
+							input
+								.getCollectedfrom()
+								.stream()
+								.map(cf -> KeyValue.newInstance(cf.getKey(), cf.getValue()))
+								.collect(Collectors.toList()));
+
+					Set<String> communities = communityMap.keySet();
+					List<Context> contextList = Optional
+						.ofNullable(
+							input
+								.getContext())
+						.map(
+							value -> value
+								.stream()
+								.map(c -> {
+									String communityId = c.getId();
+									if (communityId.contains("::")) {
+										communityId = communityId.substring(0, communityId.indexOf("::"));
+									}
+									if (communities.contains(communityId)) {
+										Context context = new Context();
+										context.setCode(communityId);
+										context.setLabel(communityMap.get(communityId));
+										Optional<List<DataInfo>> dataInfo = Optional.ofNullable(c.getDataInfo());
+										if (dataInfo.isPresent()) {
+											List<Provenance> provenance = new ArrayList<>();
+											provenance
+												.addAll(
+													dataInfo
+														.get()
+														.stream()
 														.map(
-															provenanceaction -> Provenance
-																.newInstance(
-																	provenanceaction.getClassname(), di.getTrust()))
-														.orElse(null))
-												.filter(Objects::nonNull)
-												.collect(Collectors.toSet()));
-
-									context.setProvenance(getUniqueProvenance(provenance));
-								}
-								return context;
+															di -> Optional
+																.ofNullable(di.getProvenanceaction())
+																.map(
+																	provenanceaction -> Provenance
+																		.newInstance(
+																			provenanceaction.getClassname(),
+																			di.getTrust()))
+																.orElse(null))
+														.filter(Objects::nonNull)
+														.collect(Collectors.toSet()));
+
+											try {
+												context.setProvenance(getUniqueProvenance(provenance));
+											} catch (NoAvailableEntityTypeException e) {
+												e.printStackTrace();
+											}
+										}
+										return context;
+									}
+									return null;
+								})
+								.filter(Objects::nonNull)
+								.collect(Collectors.toList()))
+						.orElse(new ArrayList<>());
+
+					if (!contextList.isEmpty()) {
+						Set<Integer> hashValue = new HashSet<>();
+						List<Context> remainigContext = new ArrayList<>();
+						contextList.forEach(c -> {
+							if (!hashValue.contains(c.hashCode())) {
+								remainigContext.add(c);
+								hashValue.add(c.hashCode());
 							}
-							return null;
-						})
-						.filter(Objects::nonNull)
-						.collect(Collectors.toList()))
-				.orElse(new ArrayList<>());
-
-			if (contextList.size() > 0) {
-				Set<Integer> hashValue = new HashSet<>();
-				List<Context> remainigContext = new ArrayList<>();
-				contextList.forEach(c -> {
-					if (!hashValue.contains(c.hashCode())) {
-						remainigContext.add(c);
-						hashValue.add(c.hashCode());
+						});
+						((CommunityResult) out).setContext(remainigContext);
 					}
-				});
-				((CommunityResult) out).setContext(remainigContext);
+				}
+			} catch (ClassCastException cce) {
+				return out;
 			}
 		}
+
 		return out;
 
 	}
 
+	private static void addTypeSpecificInformation(Result out, eu.dnetlib.dhp.schema.oaf.Result input,
+		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> ort) throws NoAvailableEntityTypeException {
+		switch (ort.get().getClassid()) {
+			case "publication":
+				Optional<Journal> journal = Optional
+					.ofNullable(((Publication) input).getJournal());
+				if (journal.isPresent()) {
+					Journal j = journal.get();
+					Container c = new Container();
+					c.setConferencedate(j.getConferencedate());
+					c.setConferenceplace(j.getConferenceplace());
+					c.setEdition(j.getEdition());
+					c.setEp(j.getEp());
+					c.setIss(j.getIss());
+					c.setIssnLinking(j.getIssnLinking());
+					c.setIssnOnline(j.getIssnOnline());
+					c.setIssnPrinted(j.getIssnPrinted());
+					c.setName(j.getName());
+					c.setSp(j.getSp());
+					c.setVol(j.getVol());
+					out.setContainer(c);
+					out.setType(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE.getClassname());
+				}
+				break;
+			case "dataset":
+				Dataset id = (Dataset) input;
+				Optional.ofNullable(id.getSize()).ifPresent(v -> out.setSize(v.getValue()));
+				Optional.ofNullable(id.getVersion()).ifPresent(v -> out.setVersion(v.getValue()));
+
+				out
+					.setGeolocation(
+						Optional
+							.ofNullable(id.getGeolocation())
+							.map(
+								igl -> igl
+									.stream()
+									.filter(Objects::nonNull)
+									.map(gli -> {
+										GeoLocation gl = new GeoLocation();
+										gl.setBox(gli.getBox());
+										gl.setPlace(gli.getPlace());
+										gl.setPoint(gli.getPoint());
+										return gl;
+									})
+									.collect(Collectors.toList()))
+							.orElse(null));
+
+				out.setType(ModelConstants.DATASET_DEFAULT_RESULTTYPE.getClassname());
+				break;
+			case "software":
+
+				Software is = (Software) input;
+				Optional
+					.ofNullable(is.getCodeRepositoryUrl())
+					.ifPresent(value -> out.setCodeRepositoryUrl(value.getValue()));
+				Optional
+					.ofNullable(is.getDocumentationUrl())
+					.ifPresent(
+						value -> out
+							.setDocumentationUrl(
+								value
+									.stream()
+									.map(Field::getValue)
+									.collect(Collectors.toList())));
+
+				Optional
+					.ofNullable(is.getProgrammingLanguage())
+					.ifPresent(value -> out.setProgrammingLanguage(value.getClassid()));
+
+				out.setType(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE.getClassname());
+				break;
+			case "other":
+
+				OtherResearchProduct ir = (OtherResearchProduct) input;
+				out
+					.setContactgroup(
+						Optional
+							.ofNullable(ir.getContactgroup())
+							.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
+							.orElse(null));
+
+				out
+					.setContactperson(
+						Optional
+							.ofNullable(ir.getContactperson())
+							.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
+							.orElse(null));
+				out
+					.setTool(
+						Optional
+							.ofNullable(ir.getTool())
+							.map(value -> value.stream().map(Field::getValue).collect(Collectors.toList()))
+							.orElse(null));
+
+				out.setType(ModelConstants.ORP_DEFAULT_RESULTTYPE.getClassname());
+
+				break;
+			default:
+				throw new NoAvailableEntityTypeException();
+		}
+	}
+
 	private static Instance getGraphInstance(eu.dnetlib.dhp.schema.oaf.Instance i) {
 		Instance instance = new Instance();
 
@@ -397,21 +433,58 @@ public class ResultMapper implements Serializable {
 	}
 
 	private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {
-		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
-			.ofNullable(i.getAccessright());
-		if (opAr.isPresent()) {
-			if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
-				String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
-				instance
-					.setAccessright(
-						AccessRight
-							.newInstance(
-								code,
-								Constants.coarCodeLabelMap.get(code),
-								Constants.COAR_ACCESS_RIGHT_SCHEMA));
+		Optional<eu.dnetlib.dhp.schema.oaf.AccessRight> opAr = Optional.ofNullable(i.getAccessright());
+
+		if (opAr.isPresent() && Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
+			String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
+
+			instance
+				.setAccessright(
+					AccessRight
+						.newInstance(
+							code,
+							Constants.coarCodeLabelMap.get(code),
+							Constants.COAR_ACCESS_RIGHT_SCHEMA));
+			if (opAr.get().getOpenAccessRoute() != null) {
+				switch (opAr.get().getOpenAccessRoute()) {
+					case hybrid:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.hybrid);
+						break;
+					case gold:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.gold);
+						break;
+					case green:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.green);
+						break;
+					case bronze:
+						instance.getAccessright().setOpenAccessRoute(OpenAccessRoute.bronze);
+						break;
+
+				}
 			}
+
 		}
 
+		Optional
+			.ofNullable(i.getPid())
+			.ifPresent(
+				pid -> instance
+					.setPid(
+						pid
+							.stream()
+							.map(p -> ControlledField.newInstance(p.getQualifier().getClassid(), p.getValue()))
+							.collect(Collectors.toList())));
+
+		Optional
+			.ofNullable(i.getAlternateIdentifier())
+			.ifPresent(
+				ai -> instance
+					.setAlternateIdentifier(
+						ai
+							.stream()
+							.map(p -> ControlledField.newInstance(p.getQualifier().getClassid(), p.getValue()))
+							.collect(Collectors.toList())));
+
 		Optional
 			.ofNullable(i.getLicense())
 			.ifPresent(value -> instance.setLicense(value.getValue()));
@@ -424,11 +497,26 @@ public class ResultMapper implements Serializable {
 		Optional
 			.ofNullable(i.getInstancetype())
 			.ifPresent(value -> instance.setType(value.getClassname()));
+		Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
+		Optional<Field<String>> oPca = Optional.ofNullable(i.getProcessingchargeamount());
+		Optional<Field<String>> oPcc = Optional.ofNullable(i.getProcessingchargecurrency());
+		if (oPca.isPresent() && oPcc.isPresent()) {
+			Field<String> pca = oPca.get();
+			Field<String> pcc = oPcc.get();
+			if (!pca.getValue().trim().equals("") && !pcc.getValue().trim().equals("")) {
+				APC apc = new APC();
+				apc.setCurrency(oPcc.get().getValue());
+				apc.setAmount(oPca.get().getValue());
+				instance.setArticleprocessingcharge(apc);
+			}
+
+		}
 		Optional.ofNullable(i.getUrl()).ifPresent(instance::setUrl);
 
 	}
 
-	private static List<Provenance> getUniqueProvenance(List<Provenance> provenance) {
+	private static List<Provenance> getUniqueProvenance(List<Provenance> provenance)
+		throws NoAvailableEntityTypeException {
 		Provenance iProv = new Provenance();
 
 		Provenance hProv = new Provenance();
@@ -450,6 +538,8 @@ public class ResultMapper implements Serializable {
 				case Constants.USER_CLAIM:
 					lProv = getHighestTrust(lProv, p);
 					break;
+				default:
+					throw new NoAvailableEntityTypeException();
 			}
 
 		}
@@ -503,34 +593,67 @@ public class ResultMapper implements Serializable {
 		return a;
 	}
 
-	private static Pid getOrcid(List<StructuredProperty> p) {
-		for (StructuredProperty pid : p) {
-			if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID)) {
-				Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
-				if (di.isPresent()) {
-					return Pid
+	private static Pid getAuthorPid(StructuredProperty pid) {
+		Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
+		if (di.isPresent()) {
+			return Pid
+				.newInstance(
+					ControlledField
 						.newInstance(
-							ControlledField
-								.newInstance(
-									pid.getQualifier().getClassid(),
-									pid.getValue()),
-							Provenance
-								.newInstance(
-									di.get().getProvenanceaction().getClassname(),
-									di.get().getTrust()));
-				} else {
-					return Pid
+							pid.getQualifier().getClassid(),
+							pid.getValue()),
+					Provenance
 						.newInstance(
-							ControlledField
-								.newInstance(
-									pid.getQualifier().getClassid(),
-									pid.getValue())
+							di.get().getProvenanceaction().getClassname(),
+							di.get().getTrust()));
+		} else {
+			return Pid
+				.newInstance(
+					ControlledField
+						.newInstance(
+							pid.getQualifier().getClassid(),
+							pid.getValue())
 
-						);
-				}
+				);
+		}
+	}
 
+	private static Pid getOrcid(List<StructuredProperty> p) {
+		List<StructuredProperty> pidList = p.stream().map(pid -> {
+			if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID) ||
+				(pid.getQualifier().getClassid().equals(ModelConstants.ORCID_PENDING))) {
+				return pid;
 			}
+			return null;
+		}).filter(Objects::nonNull).collect(Collectors.toList());
+
+		if (pidList.size() == 1) {
+			return getAuthorPid(pidList.get(0));
+		}
+
+		List<StructuredProperty> orcid = pidList
+			.stream()
+			.filter(
+				ap -> ap
+					.getQualifier()
+					.getClassid()
+					.equals(ModelConstants.ORCID))
+			.collect(Collectors.toList());
+		if (orcid.size() == 1) {
+			return getAuthorPid(orcid.get(0));
+		}
+		orcid = pidList
+			.stream()
+			.filter(
+				ap -> ap
+					.getQualifier()
+					.getClassid()
+					.equals(ModelConstants.ORCID_PENDING))
+			.collect(Collectors.toList());
+		if (orcid.size() == 1) {
+			return getAuthorPid(orcid.get(0));
 		}
+
 		return null;
 	}