Merge pull request 'Sequential ActionSet promotion' (#512) from actionset_promote_sequential into main

Reviewed-on: #512

.gitignore

@@ -27,3 +27,4 @@ spark-warehouse
 /**/.factorypath
 /**/.scalafmt.conf
 /.java-version
+/dhp-shade-package/dependency-reduced-pom.xml

dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java

@@ -80,7 +80,15 @@ class WritePredefinedProjectPropertiesTest {
 		mojo.outputFile = testFolder;
 
 		// execute
-		Assertions.assertThrows(MojoExecutionException.class, () -> mojo.execute());
+		try {
+			mojo.execute();
+			Assertions.assertTrue(false); // not reached
+		} catch (Exception e) {
+			Assertions
+				.assertTrue(
+					MojoExecutionException.class.isAssignableFrom(e.getClass()) ||
+						IllegalArgumentException.class.isAssignableFrom(e.getClass()));
+		}
 	}
 
 	@Test

dhp-common/pom.xml

@@ -70,10 +70,7 @@
 			<groupId>com.ibm.icu</groupId>
 			<artifactId>icu4j</artifactId>
 		</dependency>
-		<dependency>
+
-			<groupId>org.apache.hadoop</groupId>
-			<artifactId>hadoop-common</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>com.github.sisyphsu</groupId>
 			<artifactId>dateparser</artifactId>
@@ -163,7 +160,7 @@
 
 		<dependency>
 			<groupId>eu.dnetlib.dhp</groupId>
-			<artifactId>${dhp-schemas.artifact}</artifactId>
+			<artifactId>dhp-schemas</artifactId>
 		</dependency>
 
 		<dependency>
@@ -172,4 +169,23 @@
 		</dependency>
 	</dependencies>
 
+	<!-- dependencies required on JDK9+ because J2EE has been removed -->
+	<profiles>
+		<profile>
+			<id>spark-34</id>
+			<dependencies>
+				<dependency>
+					<groupId>javax.xml.bind</groupId>
+					<artifactId>jaxb-api</artifactId>
+					<version>2.2.11</version>
+				</dependency>
+				<dependency>
+					<groupId>com.sun.xml.ws</groupId>
+					<artifactId>jaxws-ri</artifactId>
+					<version>2.3.3</version>
+					<type>pom</type>
+				</dependency>
+			</dependencies>
+		</profile>
+	</profiles>
 </project>

dhp-common/src/main/java/eu/dnetlib/dhp/common/DbClient.java

@@ -7,12 +7,12 @@ import java.sql.*;
 import java.util.function.Consumer;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
+import org.slf4j.Logger;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.LoggerFactory;
 
 public class DbClient implements Closeable {
 
-	private static final Log log = LogFactory.getLog(DbClient.class);
+	private static final Logger log = LoggerFactory.getLogger(DbClient.class);
 
 	private final Connection connection;
 
@@ -37,6 +37,8 @@ public class DbClient implements Closeable {
 		try (final Statement stmt = connection.createStatement()) {
 			stmt.setFetchSize(100);
 
+			log.info("running SQL:\n\n{}\n\n", sql);
+
 			try (final ResultSet rs = stmt.executeQuery(sql)) {
 				while (rs.next()) {
 					consumer.accept(rs);

dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java

@@ -38,7 +38,7 @@ public class PacePerson {
 					PacePerson.class
 						.getResourceAsStream(
 							"/eu/dnetlib/dhp/common/name_particles.txt")));
-		} catch (IOException e) {
+		} catch (Exception e) {
 			throw new RuntimeException(e);
 		}
 	}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java

@@ -1,53 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import okhttp3.MediaType;
-import okhttp3.RequestBody;
-import okhttp3.internal.Util;
-import okio.BufferedSink;
-import okio.Okio;
-import okio.Source;
-
-public class InputStreamRequestBody extends RequestBody {
-
-	private final InputStream inputStream;
-	private final MediaType mediaType;
-	private final long lenght;
-
-	public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
-
-		return new InputStreamRequestBody(inputStream, mediaType, len);
-	}
-
-	private InputStreamRequestBody(InputStream inputStream, MediaType mediaType, long len) {
-		this.inputStream = inputStream;
-		this.mediaType = mediaType;
-		this.lenght = len;
-	}
-
-	@Override
-	public MediaType contentType() {
-		return mediaType;
-	}
-
-	@Override
-	public long contentLength() {
-
-		return lenght;
-
-	}
-
-	@Override
-	public void writeTo(BufferedSink sink) throws IOException {
-		Source source = null;
-		try {
-			source = Okio.source(inputStream);
-			sink.writeAll(source);
-		} finally {
-			Util.closeQuietly(source);
-		}
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java

@@ -1,8 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-public class MissingConceptDoiException extends Throwable {
-	public MissingConceptDoiException(String message) {
-		super(message);
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java

@@ -1,365 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.*;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.http.HttpHeaders;
-import org.apache.http.entity.ContentType;
-import org.jetbrains.annotations.NotNull;
-
-import com.google.gson.Gson;
-
-import eu.dnetlib.dhp.common.api.zenodo.ZenodoModel;
-import eu.dnetlib.dhp.common.api.zenodo.ZenodoModelList;
-import okhttp3.*;
-
-public class ZenodoAPIClient implements Serializable {
-
-	String urlString;
-	String bucket;
-
-	String deposition_id;
-	String access_token;
-
-	public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
-
-	private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip");
-
-	public String getUrlString() {
-		return urlString;
-	}
-
-	public void setUrlString(String urlString) {
-		this.urlString = urlString;
-	}
-
-	public String getBucket() {
-		return bucket;
-	}
-
-	public void setBucket(String bucket) {
-		this.bucket = bucket;
-	}
-
-	public void setDeposition_id(String deposition_id) {
-		this.deposition_id = deposition_id;
-	}
-
-	public ZenodoAPIClient(String urlString, String access_token) {
-
-		this.urlString = urlString;
-		this.access_token = access_token;
-	}
-
-	/**
-	 * Brand new deposition in Zenodo. It sets the deposition_id and the bucket where to store the files to upload
-	 *
-	 * @return response code
-	 * @throws IOException
-	 */
-	public int newDeposition() throws IOException {
-		String json = "{}";
-
-		URL url = new URL(urlString);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setRequestMethod("POST");
-		conn.setDoOutput(true);
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = json.getBytes("utf-8");
-			os.write(input, 0, input.length);
-		}
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-		conn.disconnect();
-
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel newSubmission = new Gson().fromJson(body, ZenodoModel.class);
-		this.bucket = newSubmission.getLinks().getBucket();
-		this.deposition_id = newSubmission.getId();
-
-		return responseCode;
-	}
-
-	/**
-	 * Upload files in Zenodo.
-	 *
-	 * @param is the inputStream for the file to upload
-	 * @param file_name the name of the file as it will appear on Zenodo
-	 * @return the response code
-	 */
-	public int uploadIS(InputStream is, String file_name) throws IOException {
-
-		URL url = new URL(bucket + "/" + file_name);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, "application/zip");
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("PUT");
-
-		byte[] buf = new byte[8192];
-		int length;
-		try (OutputStream os = conn.getOutputStream()) {
-			while ((length = is.read(buf)) != -1) {
-				os.write(buf, 0, length);
-			}
-
-		}
-		int responseCode = conn.getResponseCode();
-		if (!checkOKStatus(responseCode)) {
-			throw new IOException("Unexpected code " + responseCode + getBody(conn));
-		}
-
-		return responseCode;
-	}
-
-	@NotNull
-	private String getBody(HttpURLConnection conn) throws IOException {
-		String body = "{}";
-		try (BufferedReader br = new BufferedReader(
-			new InputStreamReader(conn.getInputStream(), "utf-8"))) {
-			StringBuilder response = new StringBuilder();
-			String responseLine = null;
-			while ((responseLine = br.readLine()) != null) {
-				response.append(responseLine.trim());
-			}
-
-			body = response.toString();
-
-		}
-		return body;
-	}
-
-	/**
-	 * Associates metadata information to the current deposition
-	 *
-	 * @param metadata the metadata
-	 * @return response code
-	 * @throws IOException
-	 */
-	public int sendMretadata(String metadata) throws IOException {
-
-		URL url = new URL(urlString + "/" + deposition_id);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("PUT");
-
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = metadata.getBytes("utf-8");
-			os.write(input, 0, input.length);
-
-		}
-
-		final int responseCode = conn.getResponseCode();
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + getBody(conn));
-
-		return responseCode;
-
-	}
-
-	private boolean checkOKStatus(int responseCode) {
-
-		if (HttpURLConnection.HTTP_OK != responseCode ||
-			HttpURLConnection.HTTP_CREATED != responseCode)
-			return true;
-		return false;
-	}
-
-	/**
-	 * To publish the current deposition. It works for both new deposition or new version of an old deposition
-	 *
-	 * @return response code
-	 * @throws IOException
-	 */
-	@Deprecated
-	public int publish() throws IOException {
-
-		String json = "{}";
-
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
-
-		Request request = new Request.Builder()
-			.url(urlString + "/" + deposition_id + "/actions/publish")
-			.addHeader("Authorization", "Bearer " + access_token)
-			.post(body)
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			return response.code();
-
-		}
-	}
-
-	/**
-	 * To create a new version of an already published deposition. It sets the deposition_id and the bucket to be used
-	 * for the new version.
-	 *
-	 * @param concept_rec_id the concept record id of the deposition for which to create a new version. It is the last
-	 *            part of the url for the DOI Zenodo suggests to use to cite all versions: DOI: 10.xxx/zenodo.656930
-	 *            concept_rec_id = 656930
-	 * @return response code
-	 * @throws IOException
-	 * @throws MissingConceptDoiException
-	 */
-	public int newVersion(String concept_rec_id) throws IOException, MissingConceptDoiException {
-		setDepositionId(concept_rec_id, 1);
-		String json = "{}";
-
-		URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion");
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("POST");
-
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = json.getBytes("utf-8");
-			os.write(input, 0, input.length);
-
-		}
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
-		String latest_draft = zenodoModel.getLinks().getLatest_draft();
-		deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
-		bucket = getBucket(latest_draft);
-
-		return responseCode;
-
-	}
-
-	/**
-	 * To finish uploading a version or new deposition not published
-	 * It sets the deposition_id and the bucket to be used
-	 *
-	 *
-	 * @param deposition_id the deposition id of the not yet published upload
-	 *            concept_rec_id = 656930
-	 * @return response code
-	 * @throws IOException
-	 * @throws MissingConceptDoiException
-	 */
-	public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
-
-		this.deposition_id = deposition_id;
-
-		String json = "{}";
-
-		URL url = new URL(urlString + "/" + deposition_id);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setRequestMethod("POST");
-		conn.setDoOutput(true);
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = json.getBytes("utf-8");
-			os.write(input, 0, input.length);
-		}
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-		conn.disconnect();
-
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
-		bucket = zenodoModel.getLinks().getBucket();
-
-		return responseCode;
-
-	}
-
-	private void setDepositionId(String concept_rec_id, Integer page) throws IOException, MissingConceptDoiException {
-
-		ZenodoModelList zenodoModelList = new Gson()
-			.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);
-
-		for (ZenodoModel zm : zenodoModelList) {
-			if (zm.getConceptrecid().equals(concept_rec_id)) {
-				deposition_id = zm.getId();
-				return;
-			}
-		}
-		if (zenodoModelList.size() == 0)
-			throw new MissingConceptDoiException(
-				"The concept record id specified was missing in the list of depositions");
-		setDepositionId(concept_rec_id, page + 1);
-
-	}
-
-	private String getPrevDepositions(String page) throws IOException {
-
-		HttpUrl.Builder urlBuilder = HttpUrl.parse(urlString).newBuilder();
-		urlBuilder.addQueryParameter("page", page);
-
-		URL url = new URL(urlBuilder.build().toString());
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("GET");
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		return body;
-
-	}
-
-	private String getBucket(String inputUurl) throws IOException {
-
-		URL url = new URL(inputUurl);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("GET");
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
-
-		return zenodoModel.getLinks().getBucket();
-
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java

@@ -1,14 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-public class Community {
-	private String identifier;
-
-	public String getIdentifier() {
-		return identifier;
-	}
-
-	public void setIdentifier(String identifier) {
-		this.identifier = identifier;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java

@@ -1,47 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-public class Creator {
-	private String affiliation;
-	private String name;
-	private String orcid;
-
-	public String getAffiliation() {
-		return affiliation;
-	}
-
-	public void setAffiliation(String affiliation) {
-		this.affiliation = affiliation;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getOrcid() {
-		return orcid;
-	}
-
-	public void setOrcid(String orcid) {
-		this.orcid = orcid;
-	}
-
-	public static Creator newInstance(String name, String affiliation, String orcid) {
-		Creator c = new Creator();
-		if (name != null) {
-			c.name = name;
-		}
-		if (affiliation != null) {
-			c.affiliation = affiliation;
-		}
-		if (orcid != null) {
-			c.orcid = orcid;
-		}
-
-		return c;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java

@@ -1,44 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class File implements Serializable {
-	private String checksum;
-	private String filename;
-	private long filesize;
-	private String id;
-
-	public String getChecksum() {
-		return checksum;
-	}
-
-	public void setChecksum(String checksum) {
-		this.checksum = checksum;
-	}
-
-	public String getFilename() {
-		return filename;
-	}
-
-	public void setFilename(String filename) {
-		this.filename = filename;
-	}
-
-	public long getFilesize() {
-		return filesize;
-	}
-
-	public void setFilesize(long filesize) {
-		this.filesize = filesize;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java

@@ -1,23 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class Grant implements Serializable {
-	private String id;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public static Grant newInstance(String id) {
-		Grant g = new Grant();
-		g.id = id;
-
-		return g;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java

@@ -1,92 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class Links implements Serializable {
-
-	private String bucket;
-
-	private String discard;
-
-	private String edit;
-	private String files;
-	private String html;
-	private String latest_draft;
-	private String latest_draft_html;
-	private String publish;
-
-	private String self;
-
-	public String getBucket() {
-		return bucket;
-	}
-
-	public void setBucket(String bucket) {
-		this.bucket = bucket;
-	}
-
-	public String getDiscard() {
-		return discard;
-	}
-
-	public void setDiscard(String discard) {
-		this.discard = discard;
-	}
-
-	public String getEdit() {
-		return edit;
-	}
-
-	public void setEdit(String edit) {
-		this.edit = edit;
-	}
-
-	public String getFiles() {
-		return files;
-	}
-
-	public void setFiles(String files) {
-		this.files = files;
-	}
-
-	public String getHtml() {
-		return html;
-	}
-
-	public void setHtml(String html) {
-		this.html = html;
-	}
-
-	public String getLatest_draft() {
-		return latest_draft;
-	}
-
-	public void setLatest_draft(String latest_draft) {
-		this.latest_draft = latest_draft;
-	}
-
-	public String getLatest_draft_html() {
-		return latest_draft_html;
-	}
-
-	public void setLatest_draft_html(String latest_draft_html) {
-		this.latest_draft_html = latest_draft_html;
-	}
-
-	public String getPublish() {
-		return publish;
-	}
-
-	public void setPublish(String publish) {
-		this.publish = publish;
-	}
-
-	public String getSelf() {
-		return self;
-	}
-
-	public void setSelf(String self) {
-		this.self = self;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java

@@ -1,153 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Metadata implements Serializable {
-
-	private String access_right;
-	private List<Community> communities;
-	private List<Creator> creators;
-	private String description;
-	private String doi;
-	private List<Grant> grants;
-	private List<String> keywords;
-	private String language;
-	private String license;
-	private PrereserveDoi prereserve_doi;
-	private String publication_date;
-	private List<String> references;
-	private List<RelatedIdentifier> related_identifiers;
-	private String title;
-	private String upload_type;
-	private String version;
-
-	public String getUpload_type() {
-		return upload_type;
-	}
-
-	public void setUpload_type(String upload_type) {
-		this.upload_type = upload_type;
-	}
-
-	public String getVersion() {
-		return version;
-	}
-
-	public void setVersion(String version) {
-		this.version = version;
-	}
-
-	public String getAccess_right() {
-		return access_right;
-	}
-
-	public void setAccess_right(String access_right) {
-		this.access_right = access_right;
-	}
-
-	public List<Community> getCommunities() {
-		return communities;
-	}
-
-	public void setCommunities(List<Community> communities) {
-		this.communities = communities;
-	}
-
-	public List<Creator> getCreators() {
-		return creators;
-	}
-
-	public void setCreators(List<Creator> creators) {
-		this.creators = creators;
-	}
-
-	public String getDescription() {
-		return description;
-	}
-
-	public void setDescription(String description) {
-		this.description = description;
-	}
-
-	public String getDoi() {
-		return doi;
-	}
-
-	public void setDoi(String doi) {
-		this.doi = doi;
-	}
-
-	public List<Grant> getGrants() {
-		return grants;
-	}
-
-	public void setGrants(List<Grant> grants) {
-		this.grants = grants;
-	}
-
-	public List<String> getKeywords() {
-		return keywords;
-	}
-
-	public void setKeywords(List<String> keywords) {
-		this.keywords = keywords;
-	}
-
-	public String getLanguage() {
-		return language;
-	}
-
-	public void setLanguage(String language) {
-		this.language = language;
-	}
-
-	public String getLicense() {
-		return license;
-	}
-
-	public void setLicense(String license) {
-		this.license = license;
-	}
-
-	public PrereserveDoi getPrereserve_doi() {
-		return prereserve_doi;
-	}
-
-	public void setPrereserve_doi(PrereserveDoi prereserve_doi) {
-		this.prereserve_doi = prereserve_doi;
-	}
-
-	public String getPublication_date() {
-		return publication_date;
-	}
-
-	public void setPublication_date(String publication_date) {
-		this.publication_date = publication_date;
-	}
-
-	public List<String> getReferences() {
-		return references;
-	}
-
-	public void setReferences(List<String> references) {
-		this.references = references;
-	}
-
-	public List<RelatedIdentifier> getRelated_identifiers() {
-		return related_identifiers;
-	}
-
-	public void setRelated_identifiers(List<RelatedIdentifier> related_identifiers) {
-		this.related_identifiers = related_identifiers;
-	}
-
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java

@@ -1,25 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class PrereserveDoi implements Serializable {
-	private String doi;
-	private String recid;
-
-	public String getDoi() {
-		return doi;
-	}
-
-	public void setDoi(String doi) {
-		this.doi = doi;
-	}
-
-	public String getRecid() {
-		return recid;
-	}
-
-	public void setRecid(String recid) {
-		this.recid = recid;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java

@@ -1,43 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class RelatedIdentifier implements Serializable {
-	private String identifier;
-	private String relation;
-	private String resource_type;
-	private String scheme;
-
-	public String getIdentifier() {
-		return identifier;
-	}
-
-	public void setIdentifier(String identifier) {
-		this.identifier = identifier;
-	}
-
-	public String getRelation() {
-		return relation;
-	}
-
-	public void setRelation(String relation) {
-		this.relation = relation;
-	}
-
-	public String getResource_type() {
-		return resource_type;
-	}
-
-	public void setResource_type(String resource_type) {
-		this.resource_type = resource_type;
-	}
-
-	public String getScheme() {
-		return scheme;
-	}
-
-	public void setScheme(String scheme) {
-		this.scheme = scheme;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java

@@ -1,118 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class ZenodoModel implements Serializable {
-
-	private String conceptrecid;
-	private String created;
-
-	private List<File> files;
-	private String id;
-	private Links links;
-	private Metadata metadata;
-	private String modified;
-	private String owner;
-	private String record_id;
-	private String state;
-	private boolean submitted;
-	private String title;
-
-	public String getConceptrecid() {
-		return conceptrecid;
-	}
-
-	public void setConceptrecid(String conceptrecid) {
-		this.conceptrecid = conceptrecid;
-	}
-
-	public String getCreated() {
-		return created;
-	}
-
-	public void setCreated(String created) {
-		this.created = created;
-	}
-
-	public List<File> getFiles() {
-		return files;
-	}
-
-	public void setFiles(List<File> files) {
-		this.files = files;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public Links getLinks() {
-		return links;
-	}
-
-	public void setLinks(Links links) {
-		this.links = links;
-	}
-
-	public Metadata getMetadata() {
-		return metadata;
-	}
-
-	public void setMetadata(Metadata metadata) {
-		this.metadata = metadata;
-	}
-
-	public String getModified() {
-		return modified;
-	}
-
-	public void setModified(String modified) {
-		this.modified = modified;
-	}
-
-	public String getOwner() {
-		return owner;
-	}
-
-	public void setOwner(String owner) {
-		this.owner = owner;
-	}
-
-	public String getRecord_id() {
-		return record_id;
-	}
-
-	public void setRecord_id(String record_id) {
-		this.record_id = record_id;
-	}
-
-	public String getState() {
-		return state;
-	}
-
-	public void setState(String state) {
-		this.state = state;
-	}
-
-	public boolean isSubmitted() {
-		return submitted;
-	}
-
-	public void setSubmitted(boolean submitted) {
-		this.submitted = submitted;
-	}
-
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java

@@ -1,7 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.util.ArrayList;
-
-public class ZenodoModelList extends ArrayList<ZenodoModel> {
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java

@@ -12,9 +12,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.math.NumberUtils;
-import org.apache.commons.lang3.time.DateUtils;
 import org.apache.http.HttpHeaders;
-import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -214,11 +212,11 @@ public class HttpConnector2 {
 					.format(
 						"Unexpected status code: %s errors: %s", urlConn.getResponseCode(),
 						MAPPER.writeValueAsString(report)));
-		} catch (MalformedURLException | UnknownHostException e) {
+		} catch (MalformedURLException e) {
 			log.error(e.getMessage(), e);
 			report.put(e.getClass().getName(), e.getMessage());
 			throw new CollectorException(e.getMessage(), e);
-		} catch (SocketTimeoutException | SocketException e) {
+		} catch (SocketTimeoutException | SocketException | UnknownHostException e) {
 			log.error(e.getMessage(), e);
 			report.put(e.getClass().getName(), e.getMessage());
 			backoffAndSleep(getClientParams().getRetryDelay() * retryNumber * 1000);

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/HashableStructuredProperty.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2024.
+ * SPDX-FileCopyrightText: © 2023 Consiglio Nazionale delle Ricerche
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+public class HashableStructuredProperty extends StructuredProperty {
+
+	private static final long serialVersionUID = 8371670185221126045L;
+
+	public static HashableStructuredProperty newInstance(String value, Qualifier qualifier, DataInfo dataInfo) {
+		if (value == null) {
+			return null;
+		}
+		final HashableStructuredProperty sp = new HashableStructuredProperty();
+		sp.setValue(value);
+		sp.setQualifier(qualifier);
+		sp.setDataInfo(dataInfo);
+		return sp;
+	}
+
+	public static HashableStructuredProperty newInstance(StructuredProperty sp) {
+		HashableStructuredProperty hsp = new HashableStructuredProperty();
+		hsp.setQualifier(sp.getQualifier());
+		hsp.setValue(sp.getValue());
+		hsp.setQualifier(sp.getQualifier());
+		return hsp;
+	}
+
+	public static StructuredProperty toStructuredProperty(HashableStructuredProperty hsp) {
+		StructuredProperty sp = new StructuredProperty();
+		sp.setQualifier(hsp.getQualifier());
+		sp.setValue(hsp.getValue());
+		sp.setQualifier(hsp.getQualifier());
+		return sp;
+	}
+
+	@Override
+	public int hashCode() {
+		return new HashCodeBuilder(11, 91)
+			.append(getQualifier().getClassid())
+			.append(getQualifier().getSchemeid())
+			.append(getValue())
+			.hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj == null) {
+			return false;
+		}
+		if (obj == this) {
+			return true;
+		}
+		if (obj.getClass() != getClass()) {
+			return false;
+		}
+		final HashableStructuredProperty rhs = (HashableStructuredProperty) obj;
+		return new EqualsBuilder()
+			.append(getQualifier().getClassid(), rhs.getQualifier().getClassid())
+			.append(getQualifier().getSchemeid(), rhs.getQualifier().getSchemeid())
+			.append(getValue(), rhs.getValue())
+			.isEquals();
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/CleaningFunctions.java

@@ -43,34 +43,4 @@ public class CleaningFunctions {
 		return !PidBlacklistProvider.getBlacklist(s.getQualifier().getClassid()).contains(pidValue);
 	}
 
-	/**
-	 * Utility method that normalises PID values on a per-type basis.
-	 * @param pid the PID whose value will be normalised.
-	 * @return the PID containing the normalised value.
-	 */
-	public static StructuredProperty normalizePidValue(StructuredProperty pid) {
-		pid
-			.setValue(
-				normalizePidValue(
-					pid.getQualifier().getClassid(),
-					pid.getValue()));
-
-		return pid;
-	}
-
-	public static String normalizePidValue(String pidType, String pidValue) {
-		String value = Optional
-			.ofNullable(pidValue)
-			.map(String::trim)
-			.orElseThrow(() -> new IllegalArgumentException("PID value cannot be empty"));
-
-		switch (pidType) {
-
-			// TODO add cleaning for more PID types as needed
-			case "doi":
-				return value.toLowerCase().replaceFirst(DOI_PREFIX_REGEX, DOI_PREFIX);
-		}
-		return value;
-	}
-
 }

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/DoiCleaningRule.java

@@ -6,18 +6,11 @@ import org.apache.commons.lang3.StringUtils;
 public class DoiCleaningRule {
 
 	public static String clean(final String doi) {
-		return doi
+		if (doi == null)
-			.toLowerCase()
-			.replaceAll("\\s", "")
-			.replaceAll("^doi:", "")
-			.replaceFirst(CleaningFunctions.DOI_PREFIX_REGEX, CleaningFunctions.DOI_PREFIX);
-	}
-
-	public static String normalizeDoi(final String input) {
-		if (input == null)
 			return null;
-		final String replaced = input
+		final String replaced = doi
 			.replaceAll("\\n|\\r|\\t|\\s", "")
+			.replaceAll("^doi:", "")
 			.toLowerCase()
 			.replaceFirst(CleaningFunctions.DOI_PREFIX_REGEX, CleaningFunctions.DOI_PREFIX);
 		if (StringUtils.isEmpty(replaced))
@@ -32,7 +25,6 @@ public class DoiCleaningRule {
 			return null;
 
 		return ret;
-
 	}
 
 }

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java

@@ -119,7 +119,7 @@ public class GraphCleaningFunctions extends CleaningFunctions {
 							.getContext()
 							.stream()
 							.filter(c -> !StringUtils.startsWith(c.getId().toLowerCase(), contextId))
-							.collect(Collectors.toList()));
+							.collect(Collectors.toCollection(ArrayList::new)));
 			}
 			return (T) res;
 		} else {
@@ -563,12 +563,24 @@ public class GraphCleaningFunctions extends CleaningFunctions {
 						Optional
 							.ofNullable(i.getPid())
 							.ifPresent(pid -> {
-								final Set<StructuredProperty> pids = Sets.newHashSet(pid);
+								final Set<HashableStructuredProperty> pids = pid
+									.stream()
+									.map(HashableStructuredProperty::newInstance)
+									.collect(Collectors.toCollection(HashSet::new));
 								Optional
 									.ofNullable(i.getAlternateIdentifier())
 									.ifPresent(altId -> {
-										final Set<StructuredProperty> altIds = Sets.newHashSet(altId);
+										final Set<HashableStructuredProperty> altIds = altId
-										i.setAlternateIdentifier(Lists.newArrayList(Sets.difference(altIds, pids)));
+											.stream()
+											.map(HashableStructuredProperty::newInstance)
+											.collect(Collectors.toCollection(HashSet::new));
+										i
+											.setAlternateIdentifier(
+												Sets
+													.difference(altIds, pids)
+													.stream()
+													.map(HashableStructuredProperty::toStructuredProperty)
+													.collect(Collectors.toList()));
 									});
 							});
 
@@ -1003,4 +1015,41 @@ public class GraphCleaningFunctions extends CleaningFunctions {
 			.orElse(null);
 	}
 
+	/**
+	 * Implements bad and ugly things that we should get rid of ASAP.
+	 *
+	 * @param value
+	 * @return
+	 * @param <T>
+	 */
+	public static <T extends Oaf> T dedicatedUglyHacks(T value) {
+		if (value instanceof OafEntity) {
+			if (value instanceof Result) {
+				final Result r = (Result) value;
+
+				// Fix for AMS Acta
+				Optional
+					.ofNullable(r.getInstance())
+					.map(
+						instance -> instance
+							.stream()
+							.filter(
+								i -> Optional
+									.ofNullable(i.getHostedby())
+									.map(KeyValue::getKey)
+									.map(dsId -> dsId.equals("10|re3data_____::4cc76bed7ce2fb95fd8e7a2dfde16016"))
+									.orElse(false)))
+					.ifPresent(instance -> instance.forEach(i -> {
+						if (Optional
+							.ofNullable(i.getPid())
+							.map(pid -> pid.stream().noneMatch(p -> p.getValue().startsWith("10.6092/unibo/amsacta")))
+							.orElse(false)) {
+							i.setHostedby(UNKNOWN_REPOSITORY);
+						}
+					}));
+			}
+		}
+		return value;
+	}
+
 }

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/IdentifierFactory.java

@@ -175,7 +175,7 @@ public class IdentifierFactory implements Serializable {
 			return entity
 				.getPid()
 				.stream()
-				.map(CleaningFunctions::normalizePidValue)
+				.map(PidCleaner::normalizePidValue)
 				.filter(CleaningFunctions::pidFilter)
 				.collect(
 					Collectors
@@ -207,7 +207,7 @@ public class IdentifierFactory implements Serializable {
 					// filter away PIDs provided by a DS that is not considered an authority for the
 					// given PID Type
 					.filter(p -> shouldFilterPidByCriteria(collectedFrom, p, mapHandles))
-					.map(CleaningFunctions::normalizePidValue)
+					.map(PidCleaner::normalizePidValue)
 					.filter(p -> isNotFromDelegatedAuthority(collectedFrom, p))
 					.filter(CleaningFunctions::pidFilter))
 			.orElse(Stream.empty());

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java

@@ -0,0 +1,106 @@
+
+package eu.dnetlib.dhp.schema.oaf.utils;
+
+import java.util.*;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.oaf.Oaf;
+import eu.dnetlib.dhp.schema.oaf.OafEntity;
+import eu.dnetlib.dhp.schema.oaf.Result;
+
+public class MergeEntitiesComparator implements Comparator<Oaf> {
+	static final List<String> PID_AUTHORITIES = Arrays
+		.asList(
+			ModelConstants.ARXIV_ID,
+			ModelConstants.PUBMED_CENTRAL_ID,
+			ModelConstants.EUROPE_PUBMED_CENTRAL_ID,
+			ModelConstants.DATACITE_ID,
+			ModelConstants.CROSSREF_ID);
+
+	static final List<String> RESULT_TYPES = Arrays
+		.asList(
+			ModelConstants.ORP_RESULTTYPE_CLASSID,
+			ModelConstants.SOFTWARE_RESULTTYPE_CLASSID,
+			ModelConstants.DATASET_RESULTTYPE_CLASSID,
+			ModelConstants.PUBLICATION_RESULTTYPE_CLASSID);
+
+	public static final Comparator<Oaf> INSTANCE = new MergeEntitiesComparator();
+
+	@Override
+	public int compare(Oaf left, Oaf right) {
+		if (left == null && right == null)
+			return 0;
+		if (left == null)
+			return -1;
+		if (right == null)
+			return 1;
+
+		int res = 0;
+
+		// pid authority
+		int cfp1 = Optional
+			.ofNullable(left.getCollectedfrom())
+			.map(
+				cf -> cf
+					.stream()
+					.map(kv -> PID_AUTHORITIES.indexOf(kv.getKey()))
+					.max(Integer::compare)
+					.orElse(-1))
+			.orElse(-1);
+		int cfp2 = Optional
+			.ofNullable(right.getCollectedfrom())
+			.map(
+				cf -> cf
+					.stream()
+					.map(kv -> PID_AUTHORITIES.indexOf(kv.getKey()))
+					.max(Integer::compare)
+					.orElse(-1))
+			.orElse(-1);
+
+		if (cfp1 >= 0 && cfp1 > cfp2) {
+			return 1;
+		} else if (cfp2 >= 0 && cfp2 > cfp1) {
+			return -1;
+		}
+
+		// trust
+		if (left.getDataInfo() != null && right.getDataInfo() != null) {
+			res = left.getDataInfo().getTrust().compareTo(right.getDataInfo().getTrust());
+		}
+
+		// result type
+		if (res == 0) {
+			if (left instanceof Result && right instanceof Result) {
+				Result r1 = (Result) left;
+				Result r2 = (Result) right;
+
+				if (r1.getResulttype() == null || r1.getResulttype().getClassid() == null) {
+					if (r2.getResulttype() != null && r2.getResulttype().getClassid() != null) {
+						return -1;
+					}
+				} else if (r2.getResulttype() == null || r2.getResulttype().getClassid() == null) {
+					return 1;
+				}
+
+				int rt1 = RESULT_TYPES.indexOf(r1.getResulttype().getClassid());
+				int rt2 = RESULT_TYPES.indexOf(r2.getResulttype().getClassid());
+
+				if (rt1 >= 0 && rt1 > rt2) {
+					return 1;
+				} else if (rt2 >= 0 && rt2 > rt1) {
+					return -1;
+				}
+			}
+		}
+
+		// id
+		if (res == 0) {
+			if (left instanceof OafEntity && right instanceof OafEntity) {
+				res = ((OafEntity) right).getId().compareTo(((OafEntity) left).getId());
+			}
+		}
+
+		return res;
+	}
+
+}

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java

@@ -30,6 +30,7 @@ import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.schema.oaf.*;
 
 public class MergeUtils {
+
 	public static <T extends Oaf> T mergeById(String s, Iterator<T> oafEntityIterator) {
 		return mergeGroup(s, oafEntityIterator, true);
 	}
@@ -40,27 +41,12 @@ public class MergeUtils {
 
 	public static <T extends Oaf> T mergeGroup(String s, Iterator<T> oafEntityIterator,
 		boolean checkDelegateAuthority) {
-		TreeSet<T> sortedEntities = new TreeSet<>((o1, o2) -> {
-			int res = 0;
 
-			if (o1.getDataInfo() != null && o2.getDataInfo() != null) {
+		ArrayList<T> sortedEntities = new ArrayList<>();
-				res = o1.getDataInfo().getTrust().compareTo(o2.getDataInfo().getTrust());
+		oafEntityIterator.forEachRemaining(sortedEntities::add);
-			}
+		sortedEntities.sort(MergeEntitiesComparator.INSTANCE.reversed());
 
-			if (res == 0) {
+		Iterator<T> it = sortedEntities.iterator();
-				if (o1 instanceof Result && o2 instanceof Result) {
-					return ResultTypeComparator.INSTANCE.compare((Result) o1, (Result) o2);
-				}
-			}
-
-			return res;
-		});
-
-		while (oafEntityIterator.hasNext()) {
-			sortedEntities.add(oafEntityIterator.next());
-		}
-
-		Iterator<T> it = sortedEntities.descendingIterator();
 		T merged = it.next();
 
 		while (it.hasNext()) {
@@ -103,7 +89,7 @@ public class MergeUtils {
 	private static Oaf mergeEntities(Oaf left, Oaf right, boolean checkDelegatedAuthority) {
 
 		if (sameClass(left, right, Result.class)) {
-			if (!left.getClass().equals(right.getClass()) || checkDelegatedAuthority) {
+			if (checkDelegatedAuthority) {
 				return mergeResultsOfDifferentTypes((Result) left, (Result) right);
 			}
 
@@ -143,7 +129,7 @@ public class MergeUtils {
 	 * https://graph.openaire.eu/docs/data-model/pids-and-identifiers#delegated-authorities and in that case it prefers
 	 * such version.
 	 * <p>
-	 * Otherwise, it considers a resulttype priority order implemented in {@link ResultTypeComparator}
+	 * Otherwise, it considers a resulttype priority order implemented in {@link MergeEntitiesComparator}
 	 * and proceeds with the canonical property merging.
 	 *
 	 * @param left
@@ -161,8 +147,9 @@ public class MergeUtils {
 		if (!leftFromDelegatedAuthority && rightFromDelegatedAuthority) {
 			return right;
 		}
+
 		// TODO: raise trust to have preferred fields from one or the other??
-		if (new ResultTypeComparator().compare(left, right) < 0) {
+		if (MergeEntitiesComparator.INSTANCE.compare(left, right) > 0) {
 			return mergeResultFields(left, right);
 		} else {
 			return mergeResultFields(right, left);
@@ -225,9 +212,9 @@ public class MergeUtils {
 
 	private static <T, K> List<T> mergeLists(final List<T> left, final List<T> right, int trust,
 		Function<T, K> keyExtractor, BinaryOperator<T> merger) {
-		if (left == null) {
+		if (left == null || left.isEmpty()) {
-			return right;
+			return right != null ? right : new ArrayList<>();
-		} else if (right == null) {
+		} else if (right == null || right.isEmpty()) {
 			return left;
 		}
 
@@ -342,7 +329,7 @@ public class MergeUtils {
 		final T merged = mergeOafFields(original, enrich, trust);
 
 		merged.setOriginalId(unionDistinctListOfString(merged.getOriginalId(), enrich.getOriginalId()));
-		merged.setPid(unionDistinctLists(merged.getPid(), enrich.getPid(), trust));
+		merged.setPid(mergeLists(merged.getPid(), enrich.getPid(), trust, MergeUtils::spKeyExtractor, (p1, p2) -> p1));
 		merged.setDateofcollection(LocalDateTime.now().toString());
 		merged
 			.setDateoftransformation(
@@ -405,7 +392,7 @@ public class MergeUtils {
 		}
 
 		// should be an instance attribute, get the first non-null value
-		merge.setLanguage(coalesce(merge.getLanguage(), enrich.getLanguage()));
+		merge.setLanguage(coalesceQualifier(merge.getLanguage(), enrich.getLanguage()));
 
 		// distinct countries, do not manage datainfo
 		merge.setCountry(mergeQualifiers(merge.getCountry(), enrich.getCountry(), trust));
@@ -446,7 +433,10 @@ public class MergeUtils {
 
 		// merge datainfo for same context id
 		merge.setContext(mergeLists(merge.getContext(), enrich.getContext(), trust, Context::getId, (r, l) -> {
-			r.getDataInfo().addAll(l.getDataInfo());
+			ArrayList<DataInfo> di = new ArrayList<>();
+			di.addAll(r.getDataInfo());
+			di.addAll(l.getDataInfo());
+			r.setDataInfo(di);
 			return r;
 		}));
 
@@ -478,6 +468,10 @@ public class MergeUtils {
 		merge.setIsInDiamondJournal(booleanOR(merge.getIsInDiamondJournal(), enrich.getIsInDiamondJournal()));
 		merge.setPubliclyFunded(booleanOR(merge.getPubliclyFunded(), enrich.getPubliclyFunded()));
 
+		if (StringUtils.isBlank(merge.getTransformativeAgreement())) {
+			merge.setTransformativeAgreement(enrich.getTransformativeAgreement());
+		}
+
 		return merge;
 	}
 
@@ -575,6 +569,13 @@ public class MergeUtils {
 		return m != null ? m : e;
 	}
 
+	private static Qualifier coalesceQualifier(Qualifier m, Qualifier e) {
+		if (m == null || m.getClassid() == null || StringUtils.isBlank(m.getClassid())) {
+			return e;
+		}
+		return m;
+	}
+
 	private static List<Author> mergeAuthors(List<Author> author, List<Author> author1, int trust) {
 		List<List<Author>> authors = new ArrayList<>();
 		if (author != null) {
@@ -587,6 +588,10 @@ public class MergeUtils {
 	}
 
 	private static String instanceKeyExtractor(Instance i) {
+		// three levels of concatenating:
+		// 1. ::
+		// 2. @@
+		// 3. ||
 		return String
 			.join(
 				"::",
@@ -594,10 +599,10 @@ public class MergeUtils {
 				kvKeyExtractor(i.getCollectedfrom()),
 				qualifierKeyExtractor(i.getAccessright()),
 				qualifierKeyExtractor(i.getInstancetype()),
-				Optional.ofNullable(i.getUrl()).map(u -> String.join("::", u)).orElse(null),
+				Optional.ofNullable(i.getUrl()).map(u -> String.join("@@", u)).orElse(null),
 				Optional
 					.ofNullable(i.getPid())
-					.map(pp -> pp.stream().map(MergeUtils::spKeyExtractor).collect(Collectors.joining("::")))
+					.map(pp -> pp.stream().map(MergeUtils::spKeyExtractor).collect(Collectors.joining("@@")))
 					.orElse(null));
 	}
 
@@ -658,6 +663,13 @@ public class MergeUtils {
 			return d1;
 		}
 
+		if (StringUtils.contains(d1.getValue(), "null")) {
+			return d2;
+		}
+		if (StringUtils.contains(d2.getValue(), "null")) {
+			return d1;
+		}
+
 		return Stream
 			.of(d1, d2)
 			.min(
@@ -706,7 +718,7 @@ public class MergeUtils {
 	private static String spKeyExtractor(StructuredProperty sp) {
 		return Optional
 			.ofNullable(sp)
-			.map(s -> Joiner.on("::").join(s, qualifierKeyExtractor(s.getQualifier())))
+			.map(s -> Joiner.on("||").join(qualifierKeyExtractor(s.getQualifier()), s.getValue()))
 			.orElse(null);
 	}
 
@@ -963,7 +975,7 @@ public class MergeUtils {
 	private static String extractKeyFromPid(final StructuredProperty pid) {
 		if (pid == null)
 			return null;
-		final StructuredProperty normalizedPid = CleaningFunctions.normalizePidValue(pid);
+		final StructuredProperty normalizedPid = PidCleaner.normalizePidValue(pid);
 
 		return String.format("%s::%s", normalizedPid.getQualifier().getClassid(), normalizedPid.getValue());
 	}

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PidValueComparator.java

@@ -18,8 +18,8 @@ public class PidValueComparator implements Comparator<StructuredProperty> {
 		if (right == null)
 			return -1;
 
-		StructuredProperty l = CleaningFunctions.normalizePidValue(left);
+		StructuredProperty l = PidCleaner.normalizePidValue(left);
-		StructuredProperty r = CleaningFunctions.normalizePidValue(right);
+		StructuredProperty r = PidCleaner.normalizePidValue(right);
 
 		return Optional
 			.ofNullable(l.getValue())

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java

@@ -1,87 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-import static eu.dnetlib.dhp.schema.common.ModelConstants.CROSSREF_ID;
-
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.Result;
-
-public class ResultTypeComparator implements Comparator<Result> {
-
-	public static final ResultTypeComparator INSTANCE = new ResultTypeComparator();
-
-	@Override
-	public int compare(Result left, Result right) {
-
-		if (left == null && right == null)
-			return 0;
-		if (left == null)
-			return 1;
-		if (right == null)
-			return -1;
-
-		HashSet<String> lCf = getCollectedFromIds(left);
-		HashSet<String> rCf = getCollectedFromIds(right);
-
-		if (lCf.contains(CROSSREF_ID) && !rCf.contains(CROSSREF_ID)) {
-			return -1;
-		}
-		if (!lCf.contains(CROSSREF_ID) && rCf.contains(CROSSREF_ID)) {
-			return 1;
-		}
-
-		if (left.getResulttype() == null || left.getResulttype().getClassid() == null) {
-			if (right.getResulttype() == null || right.getResulttype().getClassid() == null) {
-				return 0;
-			}
-			return 1;
-		} else if (right.getResulttype() == null || right.getResulttype().getClassid() == null) {
-			return -1;
-		}
-
-		String lClass = left.getResulttype().getClassid();
-		String rClass = right.getResulttype().getClassid();
-
-		if (!lClass.equals(rClass)) {
-			if (lClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
-				return -1;
-			if (rClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
-				return 1;
-
-			if (lClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
-				return -1;
-			if (rClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
-				return 1;
-
-			if (lClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
-				return -1;
-			if (rClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
-				return 1;
-
-			if (lClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
-				return -1;
-			if (rClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
-				return 1;
-		}
-
-		// Else (but unlikely), lexicographical ordering will do.
-		return lClass.compareTo(rClass);
-	}
-
-	protected HashSet<String> getCollectedFromIds(Result left) {
-		return Optional
-			.ofNullable(left.getCollectedfrom())
-			.map(
-				cf -> cf
-					.stream()
-					.map(KeyValue::getKey)
-					.collect(Collectors.toCollection(HashSet::new)))
-			.orElse(new HashSet<>());
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java

@@ -28,6 +28,7 @@ import com.jayway.jsonpath.JsonPath;
 
 import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
 import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
+import eu.dnetlib.dhp.schema.oaf.utils.PidCleaner;
 import net.minidev.json.JSONArray;
 import scala.collection.JavaConverters;
 import scala.collection.Seq;
@@ -104,7 +105,7 @@ public class DHPUtils {
 
 	public static String generateUnresolvedIdentifier(final String pid, final String pidType) {
 
-		final String cleanedPid = CleaningFunctions.normalizePidValue(pidType, pid);
+		final String cleanedPid = PidCleaner.normalizePidValue(pidType, pid);
 
 		return String.format("unresolved::%s::%s", cleanedPid, pidType.toLowerCase().trim());
 	}

dhp-common/src/main/resources/eu/dnetlib/scholexplorer/relation/relations.json

@@ -154,5 +154,13 @@
   "unknown":{
     "original":"Unknown",
     "inverse":"Unknown"
+  },
+  "isamongtopnsimilardocuments": {
+    "original": "IsAmongTopNSimilarDocuments",
+    "inverse": "HasAmongTopNSimilarDocuments"
+  },
+  "hasamongtopnsimilardocuments": {
+    "original": "HasAmongTopNSimilarDocuments",
+    "inverse": "IsAmongTopNSimilarDocuments"
   }
 }

dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala

@@ -65,12 +65,13 @@ abstract class AbstractScalaApplication(
     val conf: SparkConf = new SparkConf()
     val master = parser.get("master")
     log.info(s"Creating Spark session: Master: $master")
-    SparkSession
+    val b = SparkSession
       .builder()
       .config(conf)
       .appName(getClass.getSimpleName)
-      .master(master)
+    if (master != null)
-      .getOrCreate()
+      b.master(master)
+    b.getOrCreate()
   }
 
   def reportTotalSize(targetPath: String, outputBasePath: String): Unit = {

dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala

@@ -65,7 +65,11 @@ object ScholixUtils extends Serializable {
   }
 
   def generateScholixResourceFromResult(r: Result): ScholixResource = {
-    generateScholixResourceFromSummary(ScholixUtils.resultToSummary(r))
+    val sum = ScholixUtils.resultToSummary(r)
+    if (sum != null)
+      generateScholixResourceFromSummary(ScholixUtils.resultToSummary(r))
+    else
+      null
   }
 
   val statsAggregator: Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] =
@@ -153,6 +157,14 @@ object ScholixUtils extends Serializable {
 
   }
 
+  def invRel(rel: String): String = {
+    val semanticRelation = relations.getOrElse(rel.toLowerCase, null)
+    if (semanticRelation != null)
+      semanticRelation.inverse
+    else
+      null
+  }
+
   def extractCollectedFrom(summary: ScholixResource): List[ScholixEntityId] = {
     if (summary.getCollectedFrom != null && !summary.getCollectedFrom.isEmpty) {
       val l: List[ScholixEntityId] = summary.getCollectedFrom.asScala.map { d =>
@@ -377,10 +389,7 @@ object ScholixUtils extends Serializable {
     if (persistentIdentifiers.isEmpty)
       return null
     s.setLocalIdentifier(persistentIdentifiers.asJava)
-    if (r.isInstanceOf[Publication])
+//    s.setTypology(r.getResulttype.getClassid)
-      s.setTypology(Typology.publication)
-    else
-      s.setTypology(Typology.dataset)
 
     s.setSubType(r.getInstance().get(0).getInstancetype.getClassname)
 

dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java

@@ -1,109 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-@Disabled
-class ZenodoAPIClientTest {
-
-	private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
-	private final String ACCESS_TOKEN = "";
-
-	private final String CONCEPT_REC_ID = "657113";
-
-	private final String depositionId = "674915";
-
-	@Test
-	void testUploadOldDeposition() throws IOException, MissingConceptDoiException {
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-		Assertions.assertEquals(200, client.uploadOpenDeposition(depositionId));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz"));
-
-		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
-		Assertions.assertEquals(200, client.sendMretadata(metadata));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	void testNewDeposition() throws IOException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-		Assertions.assertEquals(201, client.newDeposition());
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz"));
-
-		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
-		Assertions.assertEquals(200, client.sendMretadata(metadata));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	void testNewVersionNewName() throws IOException, MissingConceptDoiException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-
-		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/newVersion")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition"));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	void testNewVersionOldName() throws IOException, MissingConceptDoiException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-
-		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/newVersion2")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition"));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-}

dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/IdentifierFactoryTest.java

@@ -29,7 +29,7 @@ class IdentifierFactoryTest {
 			"publication_doi2.json", "50|doi_________::79dbc7a2a56dc1532659f9038843256e", true);
 
 		verifyIdentifier(
-			"publication_doi3.json", "50|pmc_________::94e4cb08c93f8733b48e2445d04002ac", true);
+			"publication_doi3.json", "50|pmc_________::e2a339e0e11bfbf55462e14a07f1b304", true);
 
 		verifyIdentifier(
 			"publication_doi4.json", "50|od______2852::38861c44e6052a8d49f59a4c39ba5e66", true);
@@ -41,7 +41,7 @@ class IdentifierFactoryTest {
 			"publication_pmc1.json", "50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f", true);
 
 		verifyIdentifier(
-			"publication_pmc2.json", "50|pmc_________::94e4cb08c93f8733b48e2445d04002ac", true);
+			"publication_pmc2.json", "50|pmc_________::e2a339e0e11bfbf55462e14a07f1b304", true);
 
 		verifyIdentifier(
 			"publication_openapc.json", "50|doi_________::79dbc7a2a56dc1532659f9038843256e", true);

dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java

@@ -177,7 +177,7 @@ class OafMapperUtilsTest {
 		assertTrue(cfId(d1.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
 
 		assertEquals(
-			ModelConstants.DATASET_RESULTTYPE_CLASSID,
+			ModelConstants.PUBLICATION_RESULTTYPE_CLASSID,
 			((Result) MergeUtils
 				.merge(p2, d1))
 					.getResulttype()

dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/publication_doi3.json

@@ -29,7 +29,7 @@
         },
         {
           "qualifier": {"classid": "pmc"},
-          "value": "21459329"
+          "value": "PMC21459329"
         }
       ]
     }

dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/publication_pmc2.json

@@ -13,7 +13,7 @@
         },
         {
           "qualifier":{"classid":"pmc"},
-          "value":"21459329"
+          "value":"PMC21459329"
         }
       ]
     }

dhp-pace-core/pom.xml

@@ -24,7 +24,7 @@
 				<executions>
 					<execution>
 						<id>scala-compile-first</id>
-						<phase>initialize</phase>
+						<phase>process-resources</phase>
 						<goals>
 							<goal>add-source</goal>
 							<goal>compile</goal>
@@ -59,14 +59,6 @@
 			<groupId>edu.cmu</groupId>
 			<artifactId>secondstring</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>com.google.guava</groupId>
-			<artifactId>guava</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>com.google.code.gson</groupId>
-			<artifactId>gson</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-lang3</artifactId>
@@ -91,10 +83,6 @@
 			<groupId>com.fasterxml.jackson.core</groupId>
 			<artifactId>jackson-databind</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>org.apache.commons</groupId>
-			<artifactId>commons-math3</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>com.jayway.jsonpath</groupId>
 			<artifactId>json-path</artifactId>
@@ -113,4 +101,90 @@
 		</dependency>
 	</dependencies>
 
+	<profiles>
+		<profile>
+			<id>spark-24</id>
+			<activation>
+				<activeByDefault>true</activeByDefault>
+			</activation>
+
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.codehaus.mojo</groupId>
+						<artifactId>build-helper-maven-plugin</artifactId>
+						<version>3.4.0</version>
+						<executions>
+							<execution>
+								<phase>generate-sources</phase>
+								<goals>
+									<goal>add-source</goal>
+								</goals>
+								<configuration>
+									<sources>
+										<source>src/main/spark-2</source>
+									</sources>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+
+		<profile>
+			<id>spark-34</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.codehaus.mojo</groupId>
+						<artifactId>build-helper-maven-plugin</artifactId>
+						<version>3.4.0</version>
+						<executions>
+							<execution>
+								<phase>generate-sources</phase>
+								<goals>
+									<goal>add-source</goal>
+								</goals>
+								<configuration>
+									<sources>
+										<source>src/main/spark-2</source>
+									</sources>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+
+		<profile>
+			<id>spark-35</id>
+
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.codehaus.mojo</groupId>
+						<artifactId>build-helper-maven-plugin</artifactId>
+						<version>3.4.0</version>
+						<executions>
+							<execution>
+								<phase>generate-sources</phase>
+								<goals>
+									<goal>add-source</goal>
+								</goals>
+								<configuration>
+									<sources>
+										<source>src/main/spark-35</source>
+									</sources>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+
 </project>

dhp-pace-core/src/main/java/eu/dnetlib/pace/clustering/LegalnameClustering.java

@@ -2,31 +2,41 @@
 package eu.dnetlib.pace.clustering;
 
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
 import eu.dnetlib.pace.config.Config;
 
-@ClusteringClass("keywordsclustering")
+@ClusteringClass("legalnameclustering")
-public class KeywordsClustering extends AbstractClusteringFunction {
+public class LegalnameClustering extends AbstractClusteringFunction {
 
-	public KeywordsClustering(Map<String, Object> params) {
+	private static final Pattern CITY_CODE_PATTERN = Pattern.compile("city::\\d+");
+	private static final Pattern KEYWORD_CODE_PATTERN = Pattern.compile("key::\\d+");
+
+	public LegalnameClustering(Map<String, Object> params) {
 		super(params);
 	}
 
+	public Set<String> getRegexList(String input, Pattern codeRegex) {
+		Matcher matcher = codeRegex.matcher(input);
+		Set<String> cities = new HashSet<>();
+		while (matcher.find()) {
+			cities.add(matcher.group());
+		}
+		return cities;
+	}
+
 	@Override
 	protected Collection<String> doApply(final Config conf, String s) {
 
-		// takes city codes and keywords codes without duplicates
-		Set<String> keywords = getKeywords(s, conf.translationMap(), paramOrDefault("windowSize", 4));
-		Set<String> cities = getCities(s, paramOrDefault("windowSize", 4));
-
 		// list of combination to return as result
 		final Collection<String> combinations = new LinkedHashSet<String>();
 
-		for (String keyword : keywordsToCodes(keywords, conf.translationMap())) {
+		for (String keyword : getRegexList(s, KEYWORD_CODE_PATTERN)) {
-			for (String city : citiesToCodes(cities)) {
+			for (String city : getRegexList(s, CITY_CODE_PATTERN)) {
 				combinations.add(keyword + "-" + city);
 				if (combinations.size() >= paramOrDefault("max", 2)) {
 					return combinations;
@@ -42,9 +52,6 @@ public class KeywordsClustering extends AbstractClusteringFunction {
 		return fields
 			.stream()
 			.filter(f -> !f.isEmpty())
-			.map(KeywordsClustering::cleanup)
-			.map(KeywordsClustering::normalize)
-			.map(s -> filterAllStopWords(s))
 			.map(s -> doApply(conf, s))
 			.map(c -> filterBlacklisted(c, ngramBlacklist))
 			.flatMap(c -> c.stream())

dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java

@@ -1,12 +1,6 @@
 
 package eu.dnetlib.pace.common;
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.Sets;
-import com.ibm.icu.text.Transliterator;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-
 import java.io.IOException;
 import java.io.StringWriter;
 import java.nio.charset.StandardCharsets;
@@ -15,6 +9,13 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Sets;
+import com.ibm.icu.text.Transliterator;
+
 /**
  * Set of common functions for the framework
  *
@@ -26,6 +27,14 @@ public class AbstractPaceFunctions extends PaceCommonUtils {
 	private static Map<String, String> cityMap = AbstractPaceFunctions
 		.loadMapFromClasspath("/eu/dnetlib/pace/config/city_map.csv");
 
+	// keywords map to be used when translating the keyword names into codes
+	private static Map<String, String> keywordMap = AbstractPaceFunctions
+		.loadMapFromClasspath("/eu/dnetlib/pace/config/translation_map.csv");
+
+	// country map to be used when inferring the country from the city name
+	private static Map<String, String> countryMap = AbstractPaceFunctions
+		.loadCountryMapFromClasspath("/eu/dnetlib/pace/config/country_map.csv");
+
 	// list of stopwords in different languages
 	protected static Set<String> stopwords_gr = loadFromClasspath("/eu/dnetlib/pace/config/stopwords_gr.txt");
 	protected static Set<String> stopwords_en = loadFromClasspath("/eu/dnetlib/pace/config/stopwords_en.txt");
@@ -73,6 +82,64 @@ public class AbstractPaceFunctions extends PaceCommonUtils {
 		return s12;
 	}
 
+	public static String countryInference(final String original, String inferFrom) {
+		if (!original.equalsIgnoreCase("unknown"))
+			return original;
+
+		inferFrom = cleanup(inferFrom);
+		inferFrom = normalize(inferFrom);
+		inferFrom = filterAllStopWords(inferFrom);
+		Set<String> cities = getCities(inferFrom, 4);
+		return citiesToCountry(cities).stream().findFirst().orElse("UNKNOWN");
+	}
+
+	public static String cityInference(String original) {
+		original = cleanup(original);
+		original = normalize(original);
+		original = filterAllStopWords(original);
+
+		Set<String> cities = getCities(original, 4);
+
+		for (String city : cities) {
+			original = original.replaceAll(city, cityMap.get(city));
+		}
+
+		return original;
+	}
+
+	public static String keywordInference(String original) {
+		original = cleanup(original);
+		original = normalize(original);
+		original = filterAllStopWords(original);
+
+		Set<String> keywords = getKeywords(original, keywordMap, 4);
+
+		for (String keyword : keywords) {
+			original = original.replaceAll(keyword, keywordMap.get(keyword));
+		}
+
+		return original;
+	}
+
+	public static String cityKeywordInference(String original) {
+		original = cleanup(original);
+		original = normalize(original);
+		original = filterAllStopWords(original);
+
+		Set<String> keywords = getKeywords(original, keywordMap, 4);
+		Set<String> cities = getCities(original, 4);
+
+		for (String keyword : keywords) {
+			original = original.replaceAll(keyword, keywordMap.get(keyword));
+		}
+
+		for (String city : cities) {
+			original = original.replaceAll(city, cityMap.get(city));
+		}
+
+		return original;
+	}
+
 	protected static String fixXML(final String a) {
 
 		return a
@@ -207,6 +274,30 @@ public class AbstractPaceFunctions extends PaceCommonUtils {
 		return m;
 	}
 
+	public static Map<String, String> loadCountryMapFromClasspath(final String classpath) {
+
+		Transliterator transliterator = Transliterator.getInstance("Any-Eng");
+
+		final Map<String, String> m = new HashMap<>();
+		try {
+			for (final String s : IOUtils
+				.readLines(AbstractPaceFunctions.class.getResourceAsStream(classpath), StandardCharsets.UTF_8)) {
+				// string is like this: country_code;city1;city2;city3
+				String[] line = s.split(";");
+				String value = line[0];
+				for (int i = 1; i < line.length; i++) {
+					String city = fixAliases(transliterator.transliterate(line[i].toLowerCase()));
+					String code = cityMap.get(city);
+					m.put(code, value);
+				}
+			}
+		} catch (final Throwable e) {
+			return new HashMap<>();
+		}
+		return m;
+
+	}
+
 	public static String removeKeywords(String s, Set<String> keywords) {
 
 		s = " " + s + " ";
@@ -236,6 +327,10 @@ public class AbstractPaceFunctions extends PaceCommonUtils {
 		return toCodes(keywords, cityMap);
 	}
 
+	public static Set<String> citiesToCountry(Set<String> cities) {
+		return toCodes(toCodes(cities, cityMap), countryMap);
+	}
+
 	protected static String firstLC(final String s) {
 		return StringUtils.substring(s, 0, 1).toLowerCase();
 	}

dhp-pace-core/src/main/java/eu/dnetlib/pace/model/FieldDef.java

@@ -47,9 +47,21 @@ public class FieldDef implements Serializable {
 
 	private String clean;
 
+	private String infer;
+
+	private String inferenceFrom;
+
 	public FieldDef() {
 	}
 
+	public String getInferenceFrom() {
+		return inferenceFrom;
+	}
+
+	public void setInferenceFrom(final String inferenceFrom) {
+		this.inferenceFrom = inferenceFrom;
+	}
+
 	public String getName() {
 		return name;
 	}
@@ -126,6 +138,14 @@ public class FieldDef implements Serializable {
 		this.clean = clean;
 	}
 
+	public String getInfer() {
+		return infer;
+	}
+
+	public void setInfer(String infer) {
+		this.infer = infer;
+	}
+
 	@Override
 	public String toString() {
 		try {

dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala

@@ -3,7 +3,7 @@ package eu.dnetlib.pace.model
 import com.jayway.jsonpath.{Configuration, JsonPath}
 import eu.dnetlib.pace.common.AbstractPaceFunctions
 import eu.dnetlib.pace.config.{DedupConfig, Type}
-import eu.dnetlib.pace.util.MapDocumentUtil
+import eu.dnetlib.pace.util.{MapDocumentUtil, SparkCompatUtils}
 import org.apache.commons.lang3.StringUtils
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
@@ -52,7 +52,7 @@ case class SparkModel(conf: DedupConfig) {
   val orderingFieldPosition: Int = schema.fieldIndex(orderingFieldName)
 
   val parseJsonDataset: (Dataset[String] => Dataset[Row]) = df => {
-    df.map(r => rowFromJson(r))(RowEncoder(schema))
+    df.map(r => rowFromJson(r))(SparkCompatUtils.encoderFor(schema))
   }
 
   def rowFromJson(json: String): Row = {
@@ -123,9 +123,19 @@ case class SparkModel(conf: DedupConfig) {
               case _ => res(index)
             }
           }
+
+          if (StringUtils.isNotBlank(fdef.getInfer)) {
+            val inferFrom : String = if (StringUtils.isNotBlank(fdef.getInferenceFrom)) fdef.getInferenceFrom else fdef.getPath
+            res(index) = res(index) match {
+              case x: Seq[String] => x.map(inference(_, MapDocumentUtil.getJPathString(inferFrom, documentContext), fdef.getInfer))
+              case _ => inference(res(index).toString, MapDocumentUtil.getJPathString(inferFrom, documentContext), fdef.getInfer)
+            }
+          }
+
         }
 
         res
+
     }
 
     new GenericRowWithSchema(values, schema)
@@ -146,5 +156,17 @@ case class SparkModel(conf: DedupConfig) {
     res
   }
 
+  def inference(value: String, inferfrom: String, infertype: String) : String = {
+    val res = infertype match {
+      case "country" => AbstractPaceFunctions.countryInference(value, inferfrom)
+      case "city" => AbstractPaceFunctions.cityInference(value)
+      case "keyword" => AbstractPaceFunctions.keywordInference(value)
+      case "city_keyword" => AbstractPaceFunctions.cityKeywordInference(value)
+      case _ => value
+    }
+
+    res
+  }
+
 }
 

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/CityMatch.java

@@ -1,48 +0,0 @@
-
-package eu.dnetlib.pace.tree;
-
-import java.util.Map;
-import java.util.Set;
-
-import eu.dnetlib.pace.config.Config;
-import eu.dnetlib.pace.tree.support.AbstractStringComparator;
-import eu.dnetlib.pace.tree.support.ComparatorClass;
-
-@ComparatorClass("cityMatch")
-public class CityMatch extends AbstractStringComparator {
-
-	private Map<String, String> params;
-
-	public CityMatch(Map<String, String> params) {
-		super(params);
-		this.params = params;
-	}
-
-	@Override
-	public double distance(final String a, final String b, final Config conf) {
-
-		String ca = cleanup(a);
-		String cb = cleanup(b);
-
-		ca = normalize(ca);
-		cb = normalize(cb);
-
-		ca = filterAllStopWords(ca);
-		cb = filterAllStopWords(cb);
-
-		Set<String> cities1 = getCities(ca, Integer.parseInt(params.getOrDefault("windowSize", "4")));
-		Set<String> cities2 = getCities(cb, Integer.parseInt(params.getOrDefault("windowSize", "4")));
-
-		Set<String> codes1 = citiesToCodes(cities1);
-		Set<String> codes2 = citiesToCodes(cities2);
-
-		// if no cities are detected, the comparator gives 1.0
-		if (codes1.isEmpty() && codes2.isEmpty())
-			return 1.0;
-		else {
-			if (codes1.isEmpty() ^ codes2.isEmpty())
-				return -1; // undefined if one of the two has no cities
-			return commonElementsPercentage(codes1, codes2);
-		}
-	}
-}

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/CodeMatch.java

@@ -0,0 +1,51 @@
+
+package eu.dnetlib.pace.tree;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import eu.dnetlib.pace.config.Config;
+import eu.dnetlib.pace.tree.support.AbstractStringComparator;
+import eu.dnetlib.pace.tree.support.ComparatorClass;
+
+@ComparatorClass("codeMatch")
+public class CodeMatch extends AbstractStringComparator {
+
+	private Map<String, String> params;
+
+	private Pattern CODE_REGEX;
+
+	public CodeMatch(Map<String, String> params) {
+		super(params);
+		this.params = params;
+		this.CODE_REGEX = Pattern.compile(params.getOrDefault("codeRegex", "[a-zA-Z]::\\d+"));
+	}
+
+	public Set<String> getRegexList(String input) {
+		Matcher matcher = this.CODE_REGEX.matcher(input);
+		Set<String> cities = new HashSet<>();
+		while (matcher.find()) {
+			cities.add(matcher.group());
+		}
+		return cities;
+	}
+
+	@Override
+	public double distance(final String a, final String b, final Config conf) {
+
+		Set<String> codes1 = getRegexList(a);
+		Set<String> codes2 = getRegexList(b);
+
+		// if no codes are detected, the comparator gives 1.0
+		if (codes1.isEmpty() && codes2.isEmpty())
+			return 1.0;
+		else {
+			if (codes1.isEmpty() ^ codes2.isEmpty())
+				return -1; // undefined if one of the two has no codes
+			return commonElementsPercentage(codes1, codes2);
+		}
+	}
+}

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/CountryMatch.java

@@ -0,0 +1,54 @@
+
+package eu.dnetlib.pace.tree;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.wcohen.ss.AbstractStringDistance;
+
+import eu.dnetlib.pace.config.Config;
+import eu.dnetlib.pace.tree.support.AbstractStringComparator;
+import eu.dnetlib.pace.tree.support.ComparatorClass;
+
+@ComparatorClass("countryMatch")
+public class CountryMatch extends AbstractStringComparator {
+
+	private Map<String, String> params;
+
+	public CountryMatch(Map<String, String> params) {
+		super(params, new com.wcohen.ss.JaroWinkler());
+		this.params = params;
+	}
+
+	public CountryMatch(final double weight) {
+		super(weight, new com.wcohen.ss.JaroWinkler());
+	}
+
+	protected CountryMatch(final double weight, final AbstractStringDistance ssalgo) {
+		super(weight, ssalgo);
+	}
+
+	@Override
+	public double distance(final String a, final String b, final Config conf) {
+
+		if (a.isEmpty() || b.isEmpty()) {
+			return -1.0; // return -1 if a field is missing
+		}
+		if (a.equalsIgnoreCase("unknown") || b.equalsIgnoreCase("unknown")) {
+			return -1.0; // return -1 if a country is UNKNOWN
+		}
+
+		return a.equals(b) ? 1.0 : 0;
+	}
+
+	@Override
+	public double getWeight() {
+		return super.weight;
+	}
+
+	@Override
+	protected double normalize(final double d) {
+		return d;
+	}
+
+}

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/JaroWinklerLegalname.java

@@ -0,0 +1,59 @@
+
+package eu.dnetlib.pace.tree;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.wcohen.ss.AbstractStringDistance;
+
+import eu.dnetlib.pace.config.Config;
+import eu.dnetlib.pace.tree.support.AbstractStringComparator;
+import eu.dnetlib.pace.tree.support.ComparatorClass;
+
+@ComparatorClass("jaroWinklerLegalname")
+public class JaroWinklerLegalname extends AbstractStringComparator {
+
+	private Map<String, String> params;
+
+	private final String CITY_CODE_REGEX = "city::\\d+";
+	private final String KEYWORD_CODE_REGEX = "key::\\d+";
+
+	public JaroWinklerLegalname(Map<String, String> params) {
+		super(params, new com.wcohen.ss.JaroWinkler());
+		this.params = params;
+	}
+
+	public JaroWinklerLegalname(double weight) {
+		super(weight, new com.wcohen.ss.JaroWinkler());
+	}
+
+	protected JaroWinklerLegalname(double weight, AbstractStringDistance ssalgo) {
+		super(weight, ssalgo);
+	}
+
+	@Override
+	public double distance(String a, String b, final Config conf) {
+
+		String ca = a.replaceAll(CITY_CODE_REGEX, "").replaceAll(KEYWORD_CODE_REGEX, " ");
+		String cb = b.replaceAll(CITY_CODE_REGEX, "").replaceAll(KEYWORD_CODE_REGEX, " ");
+
+		ca = ca.replaceAll("[ ]{2,}", " ");
+		cb = cb.replaceAll("[ ]{2,}", " ");
+
+		if (ca.isEmpty() && cb.isEmpty())
+			return 1.0;
+		else
+			return normalize(ssalgo.score(ca, cb));
+	}
+
+	@Override
+	public double getWeight() {
+		return super.weight;
+	}
+
+	@Override
+	protected double normalize(double d) {
+		return d;
+	}
+
+}

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/JaroWinklerNormalizedName.java

@@ -1,74 +0,0 @@
-
-package eu.dnetlib.pace.tree;
-
-import java.util.Map;
-import java.util.Set;
-
-import com.wcohen.ss.AbstractStringDistance;
-
-import eu.dnetlib.pace.config.Config;
-import eu.dnetlib.pace.tree.support.AbstractStringComparator;
-import eu.dnetlib.pace.tree.support.ComparatorClass;
-
-@ComparatorClass("jaroWinklerNormalizedName")
-public class JaroWinklerNormalizedName extends AbstractStringComparator {
-
-	private Map<String, String> params;
-
-	public JaroWinklerNormalizedName(Map<String, String> params) {
-		super(params, new com.wcohen.ss.JaroWinkler());
-		this.params = params;
-	}
-
-	public JaroWinklerNormalizedName(double weight) {
-		super(weight, new com.wcohen.ss.JaroWinkler());
-	}
-
-	protected JaroWinklerNormalizedName(double weight, AbstractStringDistance ssalgo) {
-		super(weight, ssalgo);
-	}
-
-	@Override
-	public double distance(String a, String b, final Config conf) {
-		String ca = cleanup(a);
-		String cb = cleanup(b);
-
-		ca = normalize(ca);
-		cb = normalize(cb);
-
-		ca = filterAllStopWords(ca);
-		cb = filterAllStopWords(cb);
-
-		Set<String> keywords1 = getKeywords(
-			ca, conf.translationMap(), Integer.parseInt(params.getOrDefault("windowSize", "4")));
-		Set<String> keywords2 = getKeywords(
-			cb, conf.translationMap(), Integer.parseInt(params.getOrDefault("windowSize", "4")));
-
-		Set<String> cities1 = getCities(ca, Integer.parseInt(params.getOrDefault("windowSize", "4")));
-		Set<String> cities2 = getCities(cb, Integer.parseInt(params.getOrDefault("windowSize", "4")));
-
-		ca = removeKeywords(ca, keywords1);
-		ca = removeKeywords(ca, cities1);
-		cb = removeKeywords(cb, keywords2);
-		cb = removeKeywords(cb, cities2);
-
-		ca = ca.replaceAll("[ ]{2,}", " ");
-		cb = cb.replaceAll("[ ]{2,}", " ");
-
-		if (ca.isEmpty() && cb.isEmpty())
-			return 1.0;
-		else
-			return normalize(ssalgo.score(ca, cb));
-	}
-
-	@Override
-	public double getWeight() {
-		return super.weight;
-	}
-
-	@Override
-	protected double normalize(double d) {
-		return d;
-	}
-
-}

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/KeywordMatch.java

@@ -1,50 +0,0 @@
-
-package eu.dnetlib.pace.tree;
-
-import java.util.Map;
-import java.util.Set;
-
-import eu.dnetlib.pace.config.Config;
-import eu.dnetlib.pace.tree.support.AbstractStringComparator;
-import eu.dnetlib.pace.tree.support.ComparatorClass;
-
-@ComparatorClass("keywordMatch")
-public class KeywordMatch extends AbstractStringComparator {
-
-	Map<String, String> params;
-
-	public KeywordMatch(Map<String, String> params) {
-		super(params);
-		this.params = params;
-	}
-
-	@Override
-	public double distance(final String a, final String b, final Config conf) {
-
-		String ca = cleanup(a);
-		String cb = cleanup(b);
-
-		ca = normalize(ca);
-		cb = normalize(cb);
-
-		ca = filterAllStopWords(ca);
-		cb = filterAllStopWords(cb);
-
-		Set<String> keywords1 = getKeywords(
-			ca, conf.translationMap(), Integer.parseInt(params.getOrDefault("windowSize", "4")));
-		Set<String> keywords2 = getKeywords(
-			cb, conf.translationMap(), Integer.parseInt(params.getOrDefault("windowSize", "4")));
-
-		Set<String> codes1 = toCodes(keywords1, conf.translationMap());
-		Set<String> codes2 = toCodes(keywords2, conf.translationMap());
-
-		// if no cities are detected, the comparator gives 1.0
-		if (codes1.isEmpty() && codes2.isEmpty())
-			return 1.0;
-		else {
-			if (codes1.isEmpty() ^ codes2.isEmpty())
-				return -1.0; // undefined if one of the two has no keywords
-			return commonElementsPercentage(codes1, codes2);
-		}
-	}
-}

dhp-pace-core/src/main/spark-2/eu/dnetlib/pace/util/SparkCompatUtils.scala

@@ -0,0 +1,12 @@
+package eu.dnetlib.pace.util
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
+import org.apache.spark.sql.types.StructType
+
+object SparkCompatUtils {
+
+  def encoderFor(schema: StructType): ExpressionEncoder[Row] = {
+    RowEncoder(schema)
+  }
+}

dhp-pace-core/src/main/spark-35/eu/dnetlib/pace/util/SparkCompatUtils.scala

@@ -0,0 +1,12 @@
+package eu.dnetlib.pace.util
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.types.StructType
+
+object SparkCompatUtils {
+
+  def encoderFor(schema: StructType): ExpressionEncoder[Row] = {
+    ExpressionEncoder(schema)
+  }
+}

dhp-pace-core/src/test/java/eu/dnetlib/pace/clustering/ClusteringFunctionTest.java

@@ -8,6 +8,7 @@ import org.junit.jupiter.api.Test;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.mongodb.connection.Cluster;
 
 import eu.dnetlib.pace.AbstractPaceTest;
 import eu.dnetlib.pace.common.AbstractPaceFunctions;
@@ -177,41 +178,16 @@ public class ClusteringFunctionTest extends AbstractPaceTest {
 	}
 
 	@Test
-	public void testKeywordsClustering() {
+	public void legalnameClustering() {
 
-		final ClusteringFunction cf = new KeywordsClustering(params);
+		final ClusteringFunction cf = new LegalnameClustering(params);
-		final String s = "Polytechnic University of Turin";
+		String s = "key::1 key::2 city::1";
 		System.out.println(s);
 		System.out.println(cf.apply(conf, Lists.newArrayList(s)));
 
-		final String s1 = "POLITECNICO DI TORINO";
+		s = "key::1 key::2 city::1 city::2";
-		System.out.println(s1);
+		System.out.println(s);
-		System.out.println(cf.apply(conf, Lists.newArrayList(s1)));
+		System.out.println(cf.apply(conf, Lists.newArrayList(s)));
-
-		final String s2 = "Universita farmaceutica culturale di milano bergamo";
-		System.out.println("s2 = " + s2);
-		System.out.println(cf.apply(conf, Lists.newArrayList(s2)));
-
-		final String s3 = "universita universita milano milano";
-		System.out.println("s3 = " + s3);
-		System.out.println(cf.apply(conf, Lists.newArrayList(s3)));
-
-		final String s4 = "Politechniki Warszawskiej (Warsaw University of Technology)";
-		System.out.println("s4 = " + s4);
-		System.out.println(cf.apply(conf, Lists.newArrayList(s4)));
-
-		final String s5 = "İstanbul Ticarət Universiteti";
-		System.out.println("s5 = " + s5);
-		System.out.println(cf.apply(conf, Lists.newArrayList(s5)));
-
-		final String s6 = "National and Kapodistrian University of Athens";
-		System.out.println("s6 = " + s6);
-		System.out.println(cf.apply(conf, Lists.newArrayList(s6)));
-
-		final String s7 = "Εθνικό και Καποδιστριακό Πανεπιστήμιο Αθηνών";
-		System.out.println("s7 = " + s7);
-		System.out.println(cf.apply(conf, Lists.newArrayList(s7)));
-
 	}
 
 	@Test

dhp-pace-core/src/test/java/eu/dnetlib/pace/common/PaceFunctionTest.java

@@ -54,4 +54,47 @@ public class PaceFunctionTest extends AbstractPaceFunctions {
 		System.out.println("Fixed aliases  : " + fixAliases(TEST_STRING));
 	}
 
+	@Test
+	public void countryInferenceTest() {
+		assertEquals("IT", countryInference("UNKNOWN", "Università di Bologna"));
+		assertEquals("UK", countryInference("UK", "Università di Bologna"));
+		assertEquals("IT", countryInference("UNKNOWN", "Universiteé de Naples"));
+		assertEquals("UNKNOWN", countryInference("UNKNOWN", "Università del Lavoro"));
+	}
+
+	@Test
+	public void cityInferenceTest() {
+		assertEquals("universita city::3181928", cityInference("Università di Bologna"));
+		assertEquals("university city::3170647", cityInference("University of Pisa"));
+		assertEquals("universita", cityInference("Università del lavoro"));
+		assertEquals("universita city::3173331 city::3169522", cityInference("Università di Modena e Reggio Emilia"));
+	}
+
+	@Test
+	public void keywordInferenceTest() {
+		assertEquals("key::41 turin", keywordInference("Polytechnic University of Turin"));
+		assertEquals("key::41 torino", keywordInference("POLITECNICO DI TORINO"));
+		assertEquals(
+			"key::1 key::60 key::81 milano bergamo",
+			keywordInference("Universita farmaceutica culturale di milano bergamo"));
+		assertEquals("key::1 key::1 milano milano", keywordInference("universita universita milano milano"));
+		assertEquals(
+			"key::10 kapodistriako panepistemio athenon",
+			keywordInference("Εθνικό και Καποδιστριακό Πανεπιστήμιο Αθηνών"));
+	}
+
+	@Test
+	public void cityKeywordInferenceTest() {
+		assertEquals("key::41 city::3165524", cityKeywordInference("Polytechnic University of Turin"));
+		assertEquals("key::41 city::3165524", cityKeywordInference("POLITECNICO DI TORINO"));
+		assertEquals(
+			"key::1 key::60 key::81 city::3173435 city::3182164",
+			cityKeywordInference("Universita farmaceutica culturale di milano bergamo"));
+		assertEquals(
+			"key::1 key::1 city::3173435 city::3173435", cityKeywordInference("universita universita milano milano"));
+		assertEquals(
+			"key::10 kapodistriako panepistemio city::264371",
+			cityKeywordInference("Εθνικό και Καποδιστριακό Πανεπιστήμιο Αθηνών"));
+	}
+
 }

dhp-pace-core/src/test/java/eu/dnetlib/pace/comparators/ComparatorTest.java

@@ -35,6 +35,7 @@ public class ComparatorTest extends AbstractPaceTest {
 		params.put("name_th", "0.95");
 		params.put("jpath_value", "$.value");
 		params.put("jpath_classid", "$.qualifier.classid");
+		params.put("codeRegex", "key::\\d+");
 	}
 
 	@Test
@@ -44,52 +45,23 @@ public class ComparatorTest extends AbstractPaceTest {
 	}
 
 	@Test
-	public void cityMatchTest() {
+	public void codeMatchTest() {
-		final CityMatch cityMatch = new CityMatch(params);
+		CodeMatch codeMatch = new CodeMatch(params);
 
-		// both names with no cities
+		// both names with no codes
-		assertEquals(1.0, cityMatch.distance("Università", "Centro di ricerca", conf));
+		assertEquals(1.0, codeMatch.distance("testing1", "testing2", conf));
 
-		// one of the two names with no cities
+		// one of the two names with no codes
-		assertEquals(-1.0, cityMatch.distance("Università di Bologna", "Centro di ricerca", conf));
+		assertEquals(-1.0, codeMatch.distance("testing1 key::1", "testing", conf));
 
-		// both names with cities (same)
+		// both names with codes (same)
-		assertEquals(1.0, cityMatch.distance("Universita di Bologna", "Biblioteca di Bologna", conf));
+		assertEquals(1.0, codeMatch.distance("testing1 key::1", "testing2 key::1", conf));
 
-		// both names with cities (different)
+		// both names with codes (different)
-		assertEquals(0.0, cityMatch.distance("Universita di Bologna", "Universita di Torino", conf));
+		assertEquals(0.0, codeMatch.distance("testing1 key::1", "testing2 key::2", conf));
-		assertEquals(0.0, cityMatch.distance("Franklin College", "Concordia College", conf));
 
-		// particular cases
+		// both names with codes (1 same, 1 different)
-		assertEquals(1.0, cityMatch.distance("Free University of Bozen-Bolzano", "Università di Bolzano", conf));
+		assertEquals(0.5, codeMatch.distance("key::1 key::2 testing1", "key::1 testing", conf));
-		assertEquals(
-			1.0,
-			cityMatch
-				.distance(
-					"Politechniki Warszawskiej (Warsaw University of Technology)", "Warsaw University of Technology",
-					conf));
-
-		// failing becasuse 'Allen' is a transliterrated greek stopword
-		// assertEquals(-1.0, cityMatch.distance("Allen (United States)", "United States Military Academy", conf));
-		assertEquals(-1.0, cityMatch.distance("Washington (United States)", "United States Military Academy", conf));
-	}
-
-	@Test
-	public void keywordMatchTest() {
-		params.put("threshold", "0.5");
-
-		final KeywordMatch keywordMatch = new KeywordMatch(params);
-
-		assertEquals(
-			0.5, keywordMatch.distance("Biblioteca dell'Universita di Bologna", "Università di Bologna", conf));
-		assertEquals(1.0, keywordMatch.distance("Universita degli studi di Pisa", "Universita di Pisa", conf));
-		assertEquals(1.0, keywordMatch.distance("Polytechnic University of Turin", "POLITECNICO DI TORINO", conf));
-		assertEquals(1.0, keywordMatch.distance("Istanbul Commerce University", "İstanbul Ticarət Universiteti", conf));
-		assertEquals(1.0, keywordMatch.distance("Franklin College", "Concordia College", conf));
-		assertEquals(2.0 / 3.0, keywordMatch.distance("University of Georgia", "Georgia State University", conf));
-		assertEquals(0.5, keywordMatch.distance("University College London", "University of London", conf));
-		assertEquals(0.5, keywordMatch.distance("Washington State University", "University of Washington", conf));
-		assertEquals(-1.0, keywordMatch.distance("Allen (United States)", "United States Military Academy", conf));
 
 	}
 
@@ -155,15 +127,15 @@ public class ComparatorTest extends AbstractPaceTest {
 	}
 
 	@Test
-	public void jaroWinklerNormalizedNameTest() {
+	public void jaroWinklerLegalnameTest() {
 
-		final JaroWinklerNormalizedName jaroWinklerNormalizedName = new JaroWinklerNormalizedName(params);
+		final JaroWinklerLegalname jaroWinklerLegalname = new JaroWinklerLegalname(params);
 
-		double result = jaroWinklerNormalizedName
+		double result = jaroWinklerLegalname
-			.distance("AT&T (United States)", "United States Military Academy", conf);
+			.distance("AT&T (United States)", "United States key::2 key::1", conf);
 		System.out.println("result = " + result);
 
-		result = jaroWinklerNormalizedName.distance("NOAA - Servicio Meteorol\\u00f3gico Nacional", "NOAA - NWS", conf);
+		result = jaroWinklerLegalname.distance("NOAA - Servicio Meteorol\\u00f3gico Nacional", "NOAA - NWS", conf);
 		System.out.println("result = " + result);
 
 	}
@@ -336,4 +308,23 @@ public class ComparatorTest extends AbstractPaceTest {
 		System.out.println("compare = " + compare);
 	}
 
+	@Test
+	public void countryMatch() {
+
+		CountryMatch countryMatch = new CountryMatch(params);
+
+		double result = countryMatch.distance("UNKNOWN", "UNKNOWN", conf);
+		assertEquals(-1.0, result);
+
+		result = countryMatch.distance("CL", "UNKNOWN", conf);
+		assertEquals(-1.0, result);
+
+		result = countryMatch.distance("CL", "IT", conf);
+		assertEquals(0.0, result);
+
+		result = countryMatch.distance("CL", "CL", conf);
+		assertEquals(1.0, result);
+
+	}
+
 }

dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java

@@ -11,6 +11,7 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import eu.dnetlib.pace.model.Person;
+import jdk.nashorn.internal.ir.annotations.Ignore;
 
 public class UtilTest {
 

dhp-shade-package/dependency-reduced-pom.xml

@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <parent>
+    <artifactId>dhp</artifactId>
+    <groupId>eu.dnetlib.dhp</groupId>
+    <version>1.2.5-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>dhp-shade-package</artifactId>
+  <description>This module create a jar of all module dependencies</description>
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <transformers>
+                <transformer>
+                  <mainClass>eu.dnetlib.dhp.oa.dedup.SparkCreateSimRels</mainClass>
+                </transformer>
+                <transformer />
+                <transformer>
+                  <resource>META-INF/cxf/bus-extensions.txt</resource>
+                </transformer>
+              </transformers>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/maven/**</exclude>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <relocations>
+                <relocation>
+                  <pattern>com</pattern>
+                  <shadedPattern>repackaged.com.google.common</shadedPattern>
+                  <includes>
+                    <include>com.google.common.**</include>
+                  </includes>
+                </relocation>
+              </relocations>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+  <dependencies>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>1.18.28</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>5.6.1</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>junit-jupiter-api</artifactId>
+          <groupId>org.junit.jupiter</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>junit-jupiter-params</artifactId>
+          <groupId>org.junit.jupiter</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>junit-jupiter-engine</artifactId>
+          <groupId>org.junit.jupiter</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <version>3.3.3</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>byte-buddy</artifactId>
+          <groupId>net.bytebuddy</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>byte-buddy-agent</artifactId>
+          <groupId>net.bytebuddy</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-junit-jupiter</artifactId>
+      <version>3.3.3</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <distributionManagement>
+    <site>
+      <id>DHPSite</id>
+      <url>${dhp.site.stage.path}/dhp-common</url>
+    </site>
+  </distributionManagement>
+</project>

dhp-shade-package/pom.xml

@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>eu.dnetlib.dhp</groupId>
+        <artifactId>dhp</artifactId>
+        <version>1.2.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+
+    </parent>
+
+    <artifactId>dhp-shade-package</artifactId>
+    <packaging>jar</packaging>
+
+    <distributionManagement>
+        <site>
+            <id>DHPSite</id>
+            <url>${dhp.site.stage.path}/dhp-common</url>
+        </site>
+    </distributionManagement>
+
+    <description>This module create a jar of all module dependencies</description>
+
+
+    <dependencies>
+
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-actionmanager</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>eu.dnetlib.dhp</groupId>-->
+<!--            <artifactId>dhp-aggregation</artifactId>-->
+<!--            <version>${project.version}</version>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>eu.dnetlib.dhp</groupId>-->
+<!--            <artifactId>dhp-blacklist</artifactId>-->
+<!--            <version>${project.version}</version>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>eu.dnetlib.dhp</groupId>-->
+<!--            <artifactId>dhp-broker-events</artifactId>-->
+<!--            <version>${project.version}</version>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>eu.dnetlib.dhp</groupId>-->
+<!--            <artifactId>dhp-dedup-openaire</artifactId>-->
+<!--            <version>${project.version}</version>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>eu.dnetlib.dhp</groupId>-->
+<!--            <artifactId>dhp-enrichment</artifactId>-->
+<!--            <version>${project.version}</version>-->
+<!--        </dependency>-->
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-graph-mapper</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-graph-provision</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-impact-indicators</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-stats-actionsets</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-stats-hist-snaps</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-stats-monitor-irish</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-stats-promote</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-stats-update</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-swh</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-usage-raw-data-update</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-usage-stats-build</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>eu.dnetlib.dhp.oa.dedup.SparkCreateSimRels</mainClass>
+                                </transformer>
+                                <!-- This is needed if you have dependencies that use Service Loader. Most Google Cloud client libraries do. -->
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                    <resource>META-INF/cxf/bus-extensions.txt</resource>
+                                </transformer>
+                            </transformers>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/maven/**</exclude>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com</pattern>
+                                    <shadedPattern>repackaged.com.google.common</shadedPattern>
+                                    <includes>
+                                        <include>com.google.common.**</include>
+                                    </includes>
+                                </relocation>
+                            </relocations>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

dhp-workflows/dhp-actionmanager/pom.xml

@@ -51,48 +51,5 @@
             <artifactId>hadoop-distcp</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>eu.dnetlib</groupId>
-            <artifactId>dnet-actionmanager-api</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>eu.dnetlib</groupId>
-            <artifactId>dnet-actionmanager-common</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>eu.dnetlib</groupId>
-                    <artifactId>dnet-openaireplus-mapping-utils</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>saxonica</groupId>
-                    <artifactId>saxon</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>saxonica</groupId>
-                    <artifactId>saxon-dom</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>jgrapht</groupId>
-                    <artifactId>jgrapht</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>net.sf.ehcache</groupId>
-                    <artifactId>ehcache</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.springframework</groupId>
-                    <artifactId>spring-test</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.*</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>apache</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
     </dependencies>
 </project>

dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/ISClient.java

@@ -4,7 +4,6 @@ package eu.dnetlib.dhp.actionmanager;
 import java.io.Serializable;
 import java.io.StringReader;
 import java.util.List;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -22,7 +21,6 @@ import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 
-import eu.dnetlib.actionmanager.rmi.ActionManagerException;
 import eu.dnetlib.dhp.utils.ISLookupClientFactory;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
@@ -65,7 +63,7 @@ public class ISClient implements Serializable {
 						.map(t -> buildDirectory(basePath, t))
 						.collect(Collectors.toList()))
 				.orElseThrow(() -> new IllegalStateException("empty set list"));
-		} catch (ActionManagerException | ISLookUpException e) {
+		} catch (ISLookUpException e) {
 			throw new IllegalStateException("unable to query ActionSets info from the IS");
 		}
 	}
@@ -89,31 +87,18 @@ public class ISClient implements Serializable {
 		return Joiner.on("/").join(basePath, t.getMiddle(), t.getRight());
 	}
 
-	private String getBasePathHDFS(ISLookUpService isLookup) throws ActionManagerException {
+	private String getBasePathHDFS(ISLookUpService isLookup) throws ISLookUpException {
 		return queryServiceProperty(isLookup, "basePath");
 	}
 
 	private String queryServiceProperty(ISLookUpService isLookup, final String propertyName)
-		throws ActionManagerException {
+		throws ISLookUpException {
 		final String q = "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ActionManagerServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='"
 			+ propertyName
 			+ "']/@value/string()";
 		log.debug("quering for service property: {}", q);
-		try {
+
-			final List<String> value = isLookup.quickSearchProfile(q);
+		final List<String> value = isLookup.quickSearchProfile(q);
-			return Iterables.getOnlyElement(value);
+		return Iterables.getOnlyElement(value);
-		} catch (ISLookUpException e) {
-			String msg = "Error accessing service profile, using query: " + q;
-			log.error(msg, e);
-			throw new ActionManagerException(msg, e);
-		} catch (NoSuchElementException e) {
-			String msg = "missing service property: " + propertyName;
-			log.error(msg, e);
-			throw new ActionManagerException(msg, e);
-		} catch (IllegalArgumentException e) {
-			String msg = "found more than one service property: " + propertyName;
-			log.error(msg, e);
-			throw new ActionManagerException(msg, e);
-		}
 	}
 }

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml

@@ -103,6 +103,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -156,6 +157,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml

@@ -95,6 +95,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml

@@ -125,6 +125,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -134,21 +135,10 @@
             <arg>--outputPath</arg><arg>${workingDir}/action_payload_by_type</arg>
             <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
         </spark>
-        <ok to="ForkPromote"/>
+        <ok to="PromoteActionPayloadForDatasetTable"/>
         <error to="Kill"/>
     </action>
 
-    <fork name="ForkPromote">
-        <path start="PromoteActionPayloadForDatasetTable"/>
-        <path start="PromoteActionPayloadForDatasourceTable"/>
-        <path start="PromoteActionPayloadForOrganizationTable"/>
-        <path start="PromoteActionPayloadForOtherResearchProductTable"/>
-        <path start="PromoteActionPayloadForProjectTable"/>
-        <path start="PromoteActionPayloadForPublicationTable"/>
-        <path start="PromoteActionPayloadForRelationTable"/>
-        <path start="PromoteActionPayloadForSoftwareTable"/>
-    </fork>
-
     <action name="PromoteActionPayloadForDatasetTable">
         <sub-workflow>
             <app-path>${wf:appPath()}/promote_action_payload_for_dataset_table</app-path>
@@ -160,7 +150,7 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="PromoteActionPayloadForDatasourceTable"/>
         <error to="Kill"/>
     </action>
 
@@ -175,7 +165,7 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="PromoteActionPayloadForOrganizationTable"/>
         <error to="Kill"/>
     </action>
 
@@ -190,7 +180,7 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="PromoteActionPayloadForOtherResearchProductTable"/>
         <error to="Kill"/>
     </action>
 
@@ -205,7 +195,7 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="PromoteActionPayloadForProjectTable"/>
         <error to="Kill"/>
     </action>
 
@@ -220,7 +210,7 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="PromoteActionPayloadForPublicationTable"/>
         <error to="Kill"/>
     </action>
 
@@ -235,7 +225,7 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="PromoteActionPayloadForRelationTable"/>
         <error to="Kill"/>
     </action>
 
@@ -250,7 +240,7 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="PromoteActionPayloadForSoftwareTable"/>
         <error to="Kill"/>
     </action>
 
@@ -265,11 +255,9 @@
                 </property>
             </configuration>
         </sub-workflow>
-        <ok to="JoinPromote"/>
+        <ok to="End"/>
         <error to="Kill"/>
     </action>
 
-    <join name="JoinPromote" to="End"/>
-
     <end name="End"/>
 </workflow-app>

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml

@@ -95,6 +95,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml

@@ -103,6 +103,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -155,11 +156,12 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-                --conf spark.sql.shuffle.partitions=2560
+                --conf spark.sql.shuffle.partitions=8000
             </spark-opts>
             <arg>--inputGraphTablePath</arg><arg>${workingDir}/otherresearchproduct</arg>
             <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml

@@ -95,6 +95,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml

@@ -103,11 +103,12 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-                --conf spark.sql.shuffle.partitions=7000
+                --conf spark.sql.shuffle.partitions=15000
             </spark-opts>
             <arg>--inputGraphTablePath</arg><arg>${inputGraphRootPath}/publication</arg>
             <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
@@ -156,11 +157,12 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-                --conf spark.sql.shuffle.partitions=7000
+                --conf spark.sql.shuffle.partitions=15000
             </spark-opts>
             <arg>--inputGraphTablePath</arg><arg>${workingDir}/publication</arg>
             <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml

@@ -95,11 +95,12 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-                --conf spark.sql.shuffle.partitions=10000
+                --conf spark.sql.shuffle.partitions=15000
             </spark-opts>
             <arg>--inputGraphTablePath</arg><arg>${inputGraphRootPath}/relation</arg>
             <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Relation</arg>

dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml

@@ -103,6 +103,7 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
@@ -155,11 +156,12 @@
                 --executor-memory=${sparkExecutorMemory}
                 --executor-cores=${sparkExecutorCores}
                 --driver-memory=${sparkDriverMemory}
+                --conf spark.executor.memoryOverhead=${sparkExecutorMemory}
                 --conf spark.extraListeners=${spark2ExtraListeners}
                 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
                 --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-                --conf spark.sql.shuffle.partitions=2560
+                --conf spark.sql.shuffle.partitions=4000
             </spark-opts>
             <arg>--inputGraphTablePath</arg><arg>${workingDir}/software</arg>
             <arg>--graphTableClassName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>

dhp-workflows/dhp-actionmanager/src/test/java/eu/dnetlib/dhp/actionmanager/promote/PromoteResultWithMeasuresTest.java

@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2024.
+ * SPDX-FileCopyrightText: © 2023 Consiglio Nazionale delle Ricerche
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package eu.dnetlib.dhp.actionmanager.promote;
+
+import static eu.dnetlib.dhp.common.FunctionalInterfaceSupport.*;
+import static eu.dnetlib.dhp.schema.common.ModelSupport.isSubClass;
+import static org.apache.spark.sql.functions.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.List;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.*;
+import org.apache.spark.sql.Dataset;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.*;
+
+public class PromoteResultWithMeasuresTest {
+
+	private static final Logger log = LoggerFactory.getLogger(PromoteResultWithMeasuresTest.class);
+
+	private static SparkSession spark;
+
+	private static Path tempDir;
+
+	public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	@BeforeAll
+	public static void beforeAll() throws IOException {
+		tempDir = Files.createTempDirectory(PromoteResultWithMeasuresTest.class.getSimpleName());
+		log.info("using work dir {}", tempDir);
+
+		SparkConf conf = new SparkConf();
+		conf.setMaster("local[*]");
+		conf.setAppName(PromoteResultWithMeasuresTest.class.getSimpleName());
+		conf.set("spark.driver.host", "localhost");
+
+		conf.set("hive.metastore.local", "true");
+		conf.set("spark.ui.enabled", "false");
+
+		conf.set("spark.sql.warehouse.dir", tempDir.toString());
+		conf.set("hive.metastore.warehouse.dir", tempDir.resolve("warehouse").toString());
+
+		spark = SparkSession.builder().config(conf).getOrCreate();
+	}
+
+	@AfterAll
+	public static void afterAll() throws IOException {
+		spark.stop();
+		FileUtils.deleteDirectory(tempDir.toFile());
+	}
+
+	@Test
+	void testPromoteResultWithMeasures_job() throws Exception {
+
+		final String inputGraphTablePath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/promote/measures/graph")
+			.getPath();
+
+		final String inputActionPayloadPath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/promote/measures/actionPayloads")
+			.getPath();
+
+		final String actionPayloadsPath = tempDir.resolve("actionPayloads").toString();
+
+		spark
+			.read()
+			.text(inputActionPayloadPath)
+			.withColumn("payload", col("value"))
+			.select("payload")
+			.write()
+			.parquet(actionPayloadsPath);
+
+		final Path outputGraphTablePath = tempDir.resolve("outputGraphTablePath");
+
+		PromoteActionPayloadForGraphTableJob
+			.main(new String[] {
+				"--isSparkSessionManaged", Boolean.FALSE.toString(),
+				"--graphTableClassName", Publication.class.getCanonicalName(),
+				"--inputGraphTablePath", inputGraphTablePath,
+				"--inputActionPayloadPath", actionPayloadsPath,
+				"--actionPayloadClassName", Result.class.getCanonicalName(),
+				"--outputGraphTablePath", outputGraphTablePath.toString(),
+				"--mergeAndGetStrategy", MergeAndGet.Strategy.MERGE_FROM_AND_GET.toString(),
+				"--promoteActionStrategy", PromoteAction.Strategy.ENRICH.toString(),
+				"--shouldGroupById", "true"
+			});
+
+		assertFalse(isDirEmpty(outputGraphTablePath));
+
+		final Encoder<Publication> pubEncoder = Encoders.bean(Publication.class);
+		List<Publication> results = spark
+			.read()
+			.schema(pubEncoder.schema())
+			.json(outputGraphTablePath.toString())
+			.as(pubEncoder)
+			.collectAsList();
+
+		verify(results);
+	}
+
+	@Test
+	void testPromoteResultWithMeasures_internal() throws JsonProcessingException {
+
+		Dataset<Publication> rowDS = spark
+			.read()
+			.schema(Encoders.bean(Publication.class).schema())
+			.json("src/test/resources/eu/dnetlib/dhp/actionmanager/promote/measures/graph")
+			.as(Encoders.bean(Publication.class));
+
+		Dataset<Result> actionPayloadDS = spark
+			.read()
+			.schema(Encoders.bean(Result.class).schema())
+			.json("src/test/resources/eu/dnetlib/dhp/actionmanager/promote/measures/actionPayloads")
+			.as(Encoders.bean(Result.class));
+
+		final MergeAndGet.Strategy mergeFromAndGet = MergeAndGet.Strategy.MERGE_FROM_AND_GET;
+
+		final SerializableSupplier<Function<Publication, String>> rowIdFn = ModelSupport::idFn;
+		final SerializableSupplier<BiFunction<Publication, Result, Publication>> mergeAndGetFn = MergeAndGet
+			.functionFor(mergeFromAndGet);
+		final SerializableSupplier<Publication> zeroFn = () -> Publication.class
+			.cast(new eu.dnetlib.dhp.schema.oaf.Publication());
+		final SerializableSupplier<Function<Publication, Boolean>> isNotZeroFn = PromoteResultWithMeasuresTest::isNotZeroFnUsingIdOrSourceAndTarget;
+
+		Dataset<Publication> joinedResults = PromoteActionPayloadFunctions
+			.joinGraphTableWithActionPayloadAndMerge(
+				rowDS,
+				actionPayloadDS,
+				rowIdFn,
+				ModelSupport::idFn,
+				mergeAndGetFn,
+				PromoteAction.Strategy.ENRICH,
+				Publication.class,
+				Result.class);
+
+		SerializableSupplier<BiFunction<Publication, Publication, Publication>> mergeRowsAndGetFn = MergeAndGet
+			.functionFor(mergeFromAndGet);
+
+		Dataset<Publication> mergedResults = PromoteActionPayloadFunctions
+			.groupGraphTableByIdAndMerge(
+				joinedResults, rowIdFn, mergeRowsAndGetFn, zeroFn, isNotZeroFn, Publication.class);
+
+		verify(mergedResults.collectAsList());
+	}
+
+	private static void verify(List<Publication> results) throws JsonProcessingException {
+		assertNotNull(results);
+		assertEquals(1, results.size());
+
+		Result r = results.get(0);
+
+		log.info(OBJECT_MAPPER.writeValueAsString(r));
+
+		assertNotNull(r.getMeasures());
+		assertFalse(r.getMeasures().isEmpty());
+		assertTrue(
+			r
+				.getMeasures()
+				.stream()
+				.map(Measure::getId)
+				.collect(Collectors.toCollection(HashSet::new))
+				.containsAll(
+					Lists
+						.newArrayList(
+							"downloads", "views", "influence", "popularity", "influence_alt", "popularity_alt",
+							"impulse")));
+	}
+
+	private static <T extends Oaf> Function<T, Boolean> isNotZeroFnUsingIdOrSourceAndTarget() {
+		return t -> {
+			if (isSubClass(t, Relation.class)) {
+				final Relation rel = (Relation) t;
+				return StringUtils.isNotBlank(rel.getSource()) && StringUtils.isNotBlank(rel.getTarget());
+			}
+			return StringUtils.isNotBlank(((OafEntity) t).getId());
+		};
+	}
+
+	private static boolean isDirEmpty(final Path directory) throws IOException {
+		try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(directory)) {
+			return !dirStream.iterator().hasNext();
+		}
+	}
+
+}

dhp-workflows/dhp-actionmanager/src/test/resources/eu/dnetlib/dhp/actionmanager/promote/measures/actionPayloads/part0000.json

@@ -0,0 +1,3 @@
+{"collectedfrom":null,"dataInfo":null,"lastupdatetimestamp":null,"id":"50|doi_dedup___::02317b7093277ec8aa0311d5c6a25b9b","originalId":null,"pid":null,"dateofcollection":null,"dateoftransformation":null,"extraInfo":null,"oaiprovenance":null,"measures":[{"id":"downloads","unit":[{"key":"opendoar____::358aee4cc897452c00244351e4d91f69||ZENODO","value":"125","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"views","unit":[{"key":"opendoar____::358aee4cc897452c00244351e4d91f69||ZENODO","value":"35","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:usage_counts","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"context":null,"processingchargeamount":null,"processingchargecurrency":null,"author":null,"resulttype":null,"metaResourceType":null,"language":null,"country":null,"subject":null,"title":null,"relevantdate":null,"description":null,"dateofacceptance":null,"publisher":null,"embargoenddate":null,"source":null,"fulltext":null,"format":null,"contributor":null,"resourcetype":null,"coverage":null,"bestaccessright":null,"externalReference":null,"instance":null,"eoscifguidelines":null,"openAccessColor":null,"publiclyFunded":null,"transformativeAgreement":null,"isGreen":null,"isInDiamondJournal":null}
+{"collectedfrom":null,"dataInfo":null,"lastupdatetimestamp":null,"id":"50|doi_dedup___::02317b7093277ec8aa0311d5c6a25b9b","originalId":null,"pid":null,"dateofcollection":null,"dateoftransformation":null,"extraInfo":null,"oaiprovenance":null,"measures":[{"id":"influence","unit":[{"key":"score","value":"3.1167566E-9","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"popularity","unit":[{"key":"score","value":"7.335433E-9","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C4","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"influence_alt","unit":[{"key":"score","value":"4","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"popularity_alt","unit":[{"key":"score","value":"2.96","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C4","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"impulse","unit":[{"key":"score","value":"4","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"context":null,"processingchargeamount":null,"processingchargecurrency":null,"author":null,"resulttype":null,"metaResourceType":null,"language":null,"country":null,"subject":null,"title":null,"relevantdate":null,"description":null,"dateofacceptance":null,"publisher":null,"embargoenddate":null,"source":null,"fulltext":null,"format":null,"contributor":null,"resourcetype":null,"coverage":null,"bestaccessright":null,"externalReference":null,"instance":null,"eoscifguidelines":null,"openAccessColor":null,"publiclyFunded":null,"transformativeAgreement":null,"isGreen":null,"isInDiamondJournal":null}
+{"collectedfrom":null,"dataInfo":null,"lastupdatetimestamp":null,"id":"50|doi_dedup___::02317b7093277ec8aa0311d5c6a25b9b","originalId":null,"pid":null,"dateofcollection":null,"dateoftransformation":null,"extraInfo":null,"oaiprovenance":null,"measures":null,"context":null,"processingchargeamount":null,"processingchargecurrency":null,"author":null,"resulttype":null,"metaResourceType":null,"language":null,"country":null,"subject":null,"title":null,"relevantdate":null,"description":null,"dateofacceptance":null,"publisher":null,"embargoenddate":null,"source":null,"fulltext":null,"format":null,"contributor":null,"resourcetype":null,"coverage":null,"bestaccessright":null,"externalReference":null,"instance":null,"eoscifguidelines":null,"openAccessColor":"hybrid","publiclyFunded":false,"transformativeAgreement":null,"isGreen":true,"isInDiamondJournal":false}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java

@@ -42,6 +42,9 @@ public class Constants {
 	public static final String NULL = "NULL";
 	public static final String NA = "N/A";
 
+	public static final String WEB_CRAWL_ID = "10|openaire____::fb98a192f6a055ba495ef414c330834b";
+	public static final String WEB_CRAWL_NAME = "Web Crawl";
+
 	public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
 	private Constants() {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java

@@ -9,7 +9,7 @@ import java.util.List;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.BZip2Codec;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaPairRDD;
@@ -28,6 +28,7 @@ import eu.dnetlib.dhp.schema.action.AtomicAction;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.oaf.*;
 import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
+import eu.dnetlib.dhp.schema.oaf.utils.DoiCleaningRule;
 import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
 import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
 import scala.Tuple2;
@@ -40,9 +41,13 @@ public class PrepareAffiliationRelations implements Serializable {
 	private static final Logger log = LoggerFactory.getLogger(PrepareAffiliationRelations.class);
 	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 	private static final String ID_PREFIX = "50|doi_________::";
-	public static final String BIP_AFFILIATIONS_CLASSID = "result:organization:bipinference";
+	public static final String BIP_AFFILIATIONS_CLASSID = "result:organization:openaireinference";
-	public static final String BIP_AFFILIATIONS_CLASSNAME = "Affiliation relation inferred by BIP!";
+	public static final String BIP_AFFILIATIONS_CLASSNAME = "Affiliation relation inferred by OpenAIRE";
-	public static final String BIP_INFERENCE_PROVENANCE = "bip:affiliation:crossref";
+	public static final String BIP_INFERENCE_PROVENANCE = "openaire:affiliation";
+	public static final String OPENAIRE_DATASOURCE_ID = "10|infrastruct_::f66f1bd369679b5b077dcdf006089556";
+	public static final String OPENAIRE_DATASOURCE_NAME = "OpenAIRE";
+	public static final String DOI_URL_PREFIX = "https://doi.org/";
+	public static final int DOI_URL_PREFIX_LENGTH = 16;
 
 	public static <I extends Result> void main(String[] args) throws Exception {
 
@@ -70,6 +75,12 @@ public class PrepareAffiliationRelations implements Serializable {
 		final String dataciteInputPath = parser.get("dataciteInputPath");
 		log.info("dataciteInputPath: {}", dataciteInputPath);
 
+		final String webcrawlInputPath = parser.get("webCrawlInputPath");
+		log.info("webcrawlInputPath: {}", webcrawlInputPath);
+
+		final String publisherInputPath = parser.get("publisherInputPath");
+		log.info("publisherInputPath: {}", publisherInputPath);
+
 		final String outputPath = parser.get("outputPath");
 		log.info("outputPath: {}", outputPath);
 
@@ -80,37 +91,74 @@ public class PrepareAffiliationRelations implements Serializable {
 			isSparkSessionManaged,
 			spark -> {
 				Constants.removeOutputDir(spark, outputPath);
-
+				createActionSet(
-				List<KeyValue> collectedFromCrossref = OafMapperUtils
+					spark, crossrefInputPath, pubmedInputPath, openapcInputPath, dataciteInputPath, webcrawlInputPath,
-					.listKeyValues(ModelConstants.CROSSREF_ID, "Crossref");
+					publisherInputPath, outputPath);
-				JavaPairRDD<Text, Text> crossrefRelations = prepareAffiliationRelations(
-					spark, crossrefInputPath, collectedFromCrossref);
-
-				List<KeyValue> collectedFromPubmed = OafMapperUtils
-					.listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed");
-				JavaPairRDD<Text, Text> pubmedRelations = prepareAffiliationRelations(
-					spark, pubmedInputPath, collectedFromPubmed);
-
-				List<KeyValue> collectedFromOpenAPC = OafMapperUtils
-					.listKeyValues(ModelConstants.OPEN_APC_ID, "OpenAPC");
-				JavaPairRDD<Text, Text> openAPCRelations = prepareAffiliationRelations(
-					spark, openapcInputPath, collectedFromOpenAPC);
-
-				List<KeyValue> collectedFromDatacite = OafMapperUtils
-					.listKeyValues(ModelConstants.DATACITE_ID, "Datacite");
-				JavaPairRDD<Text, Text> dataciteRelations = prepareAffiliationRelations(
-					spark, dataciteInputPath, collectedFromDatacite);
-
-				crossrefRelations
-					.union(pubmedRelations)
-					.union(openAPCRelations)
-					.union(dataciteRelations)
-					.saveAsHadoopFile(
-						outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class);
-
 			});
 	}
 
+	private static void createActionSet(SparkSession spark, String crossrefInputPath, String pubmedInputPath,
+		String openapcInputPath, String dataciteInputPath, String webcrawlInputPath, String publisherlInputPath,
+		String outputPath) {
+		List<KeyValue> collectedfromOpenAIRE = OafMapperUtils
+			.listKeyValues(OPENAIRE_DATASOURCE_ID, OPENAIRE_DATASOURCE_NAME);
+
+		JavaPairRDD<Text, Text> crossrefRelations = prepareAffiliationRelationsNewModel(
+			spark, crossrefInputPath, collectedfromOpenAIRE);
+
+		JavaPairRDD<Text, Text> pubmedRelations = prepareAffiliationRelations(
+			spark, pubmedInputPath, collectedfromOpenAIRE);
+
+		JavaPairRDD<Text, Text> openAPCRelations = prepareAffiliationRelationsNewModel(
+			spark, openapcInputPath, collectedfromOpenAIRE);
+
+		JavaPairRDD<Text, Text> dataciteRelations = prepareAffiliationRelations(
+			spark, dataciteInputPath, collectedfromOpenAIRE);
+
+		JavaPairRDD<Text, Text> webCrawlRelations = prepareAffiliationRelations(
+			spark, webcrawlInputPath, collectedfromOpenAIRE);
+
+		JavaPairRDD<Text, Text> publisherRelations = prepareAffiliationRelationFromPublisher(
+			spark, publisherlInputPath, collectedfromOpenAIRE);
+
+		crossrefRelations
+			.union(pubmedRelations)
+			.union(openAPCRelations)
+			.union(dataciteRelations)
+			.union(webCrawlRelations)
+			.union(publisherRelations)
+			.saveAsHadoopFile(
+				outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, BZip2Codec.class);
+	}
+
+	private static JavaPairRDD<Text, Text> prepareAffiliationRelationFromPublisherNewModel(SparkSession spark,
+		String inputPath,
+		List<KeyValue> collectedfrom) {
+
+		Dataset<Row> df = spark
+			.read()
+			.schema(
+				"`DOI` STRING, `Organizations` ARRAY<STRUCT<`PID`:STRING, `Value`:STRING,`Confidence`:DOUBLE, `Status`:STRING>>")
+			.json(inputPath)
+			.where("DOI is not null");
+
+		return getTextTextJavaPairRDD(collectedfrom, df.selectExpr("DOI", "Organizations as Matchings"));
+
+	}
+
+	private static JavaPairRDD<Text, Text> prepareAffiliationRelationFromPublisher(SparkSession spark, String inputPath,
+		List<KeyValue> collectedfrom) {
+
+		Dataset<Row> df = spark
+			.read()
+			.schema("`DOI` STRING, `Organizations` ARRAY<STRUCT<`RORid`:STRING,`Confidence`:DOUBLE>>")
+			.json(inputPath)
+			.where("DOI is not null");
+
+		return getTextTextJavaPairRDD(collectedfrom, df.selectExpr("DOI", "Organizations as Matchings"));
+
+	}
+
 	private static <I extends Result> JavaPairRDD<Text, Text> prepareAffiliationRelations(SparkSession spark,
 		String inputPath,
 		List<KeyValue> collectedfrom) {
@@ -119,8 +167,27 @@ public class PrepareAffiliationRelations implements Serializable {
 		Dataset<Row> df = spark
 			.read()
 			.schema("`DOI` STRING, `Matchings` ARRAY<STRUCT<`RORid`:STRING,`Confidence`:DOUBLE>>")
-			.json(inputPath);
+			.json(inputPath)
+			.where("DOI is not null");
 
+		return getTextTextJavaPairRDD(collectedfrom, df);
+	}
+
+	private static <I extends Result> JavaPairRDD<Text, Text> prepareAffiliationRelationsNewModel(SparkSession spark,
+		String inputPath,
+		List<KeyValue> collectedfrom) {
+		// load and parse affiliation relations from HDFS
+		Dataset<Row> df = spark
+			.read()
+			.schema(
+				"`DOI` STRING, `Matchings` ARRAY<STRUCT<`PID`:STRING, `Value`:STRING,`Confidence`:DOUBLE, `Status`:STRING>>")
+			.json(inputPath)
+			.where("DOI is not null");
+
+		return getTextTextJavaPairRDDNew(collectedfrom, df);
+	}
+
+	private static JavaPairRDD<Text, Text> getTextTextJavaPairRDD(List<KeyValue> collectedfrom, Dataset<Row> df) {
 		// unroll nested arrays
 		df = df
 			.withColumn("matching", functions.explode(new Column("Matchings")))
@@ -136,7 +203,7 @@ public class PrepareAffiliationRelations implements Serializable {
 
 				// DOI to OpenAIRE id
 				final String paperId = ID_PREFIX
-					+ IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", row.getAs("doi")));
+					+ IdentifierFactory.md5(DoiCleaningRule.clean(removePrefix(row.getAs("doi"))));
 
 				// ROR id to OpenAIRE id
 				final String affId = GenerateRorActionSetJob.calculateOpenaireId(row.getAs("rorid"));
@@ -168,6 +235,69 @@ public class PrepareAffiliationRelations implements Serializable {
 					new Text(OBJECT_MAPPER.writeValueAsString(aa))));
 	}
 
+	private static JavaPairRDD<Text, Text> getTextTextJavaPairRDDNew(List<KeyValue> collectedfrom, Dataset<Row> df) {
+		// unroll nested arrays
+		df = df
+			.withColumn("matching", functions.explode(new Column("Matchings")))
+			.select(
+				new Column("DOI").as("doi"),
+				new Column("matching.PID").as("pidtype"),
+				new Column("matching.Value").as("pidvalue"),
+				new Column("matching.Confidence").as("confidence"),
+				new Column("matching.Status").as("status"))
+			.where("status = 'active'");
+
+		// prepare action sets for affiliation relations
+		return df
+			.toJavaRDD()
+			.flatMap((FlatMapFunction<Row, Relation>) row -> {
+
+				// DOI to OpenAIRE id
+				final String paperId = ID_PREFIX
+					+ IdentifierFactory.md5(DoiCleaningRule.clean(removePrefix(row.getAs("doi"))));
+
+				// Organization to OpenAIRE identifier
+				String affId = null;
+				if (row.getAs("pidtype").equals("ROR"))
+					// ROR id to OpenIARE id
+					affId = GenerateRorActionSetJob.calculateOpenaireId(row.getAs("pidvalue"));
+				else
+					// getting the OpenOrgs identifier for the organization
+					affId = row.getAs("pidvalue");
+
+				Qualifier qualifier = OafMapperUtils
+					.qualifier(
+						BIP_AFFILIATIONS_CLASSID,
+						BIP_AFFILIATIONS_CLASSNAME,
+						ModelConstants.DNET_PROVENANCE_ACTIONS,
+						ModelConstants.DNET_PROVENANCE_ACTIONS);
+
+				// format data info; setting `confidence` into relation's `trust`
+				DataInfo dataInfo = OafMapperUtils
+					.dataInfo(
+						false,
+						BIP_INFERENCE_PROVENANCE,
+						true,
+						false,
+						qualifier,
+						Double.toString(row.getAs("confidence")));
+
+				// return bi-directional relations
+				return getAffiliationRelationPair(paperId, affId, collectedfrom, dataInfo).iterator();
+
+			})
+			.map(p -> new AtomicAction(Relation.class, p))
+			.mapToPair(
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+					new Text(OBJECT_MAPPER.writeValueAsString(aa))));
+	}
+
+	private static String removePrefix(String doi) {
+		if (doi.startsWith(DOI_URL_PREFIX))
+			return doi.substring(DOI_URL_PREFIX_LENGTH);
+		return doi;
+	}
+
 	private static List<Relation> getAffiliationRelationPair(String paperId, String affId, List<KeyValue> collectedfrom,
 		DataInfo dataInfo) {
 		return Arrays

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java

@@ -10,6 +10,7 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
 import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.spark.SparkConf;
@@ -83,7 +84,7 @@ public class SparkAtomicActionScoreJob implements Serializable {
 				resultsRDD
 					.union(projectsRDD)
 					.saveAsHadoopFile(
-						outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class);
+						outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, BZip2Codec.class);
 			});
 	}
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java

@@ -115,19 +115,7 @@ public class PrepareFOSSparkJob implements Serializable {
 			.forEach(
 				l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID, true)));
 		r.setSubject(sbjs);
-		r
+
-			.setDataInfo(
-				OafMapperUtils
-					.dataInfo(
-						false, null, true,
-						false,
-						OafMapperUtils
-							.qualifier(
-								ModelConstants.PROVENANCE_ENRICH,
-								null,
-								ModelConstants.DNET_PROVENANCE_ACTIONS,
-								ModelConstants.DNET_PROVENANCE_ACTIONS),
-						null));
 		return r;
 	}
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareSDGSparkJob.java

@@ -6,26 +6,23 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.api.java.function.MapGroupsFunction;
-import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.*;
-import org.apache.spark.sql.Encoders;
+import org.jetbrains.annotations.NotNull;
-import org.apache.spark.sql.SaveMode;
-import org.apache.spark.sql.SparkSession;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.SDGDataModel;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.schema.oaf.Result;
-import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
 import eu.dnetlib.dhp.schema.oaf.Subject;
-import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
 import eu.dnetlib.dhp.utils.DHPUtils;
 
 public class PrepareSDGSparkJob implements Serializable {
@@ -52,54 +49,91 @@ public class PrepareSDGSparkJob implements Serializable {
 		final String outputPath = parser.get("outputPath");
 		log.info("outputPath: {}", outputPath);
 
+		final Boolean distributeDOI = Optional
+			.ofNullable(parser.get("distributeDoi"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("distribute doi {}", distributeDOI);
+
 		SparkConf conf = new SparkConf();
 		runWithSparkSession(
 			conf,
 			isSparkSessionManaged,
 			spark -> {
-				doPrepare(
+				if (distributeDOI)
-					spark,
+					doPrepare(
-					sourcePath,
+						spark,
+						sourcePath,
+
+						outputPath);
+				else
+					doPrepareoaid(spark, sourcePath, outputPath);
 
-					outputPath);
 			});
 	}
 
 	private static void doPrepare(SparkSession spark, String sourcePath, String outputPath) {
-		Dataset<SDGDataModel> sdgDataset = readPath(spark, sourcePath, SDGDataModel.class);
+		Dataset<Row> sdgDataset = spark
+			.read()
+			.format("csv")
+			.option("sep", DEFAULT_DELIMITER)
+			.option("inferSchema", "true")
+			.option("header", "true")
+			.option("quotes", "\"")
+			.load(sourcePath);
 
 		sdgDataset
-			.groupByKey((MapFunction<SDGDataModel, String>) r -> r.getDoi().toLowerCase(), Encoders.STRING())
+			.groupByKey((MapFunction<Row, String>) v -> ((String) v.getAs("doi")).toLowerCase(), Encoders.STRING())
-			.mapGroups((MapGroupsFunction<String, SDGDataModel, Result>) (k, it) -> {
+			.mapGroups(
-				Result r = new Result();
+				(MapGroupsFunction<String, Row, Result>) (k,
-				r.setId(DHPUtils.generateUnresolvedIdentifier(k, DOI));
+					it) -> getResult(
-				SDGDataModel first = it.next();
+						DHPUtils
-				List<Subject> sbjs = new ArrayList<>();
+							.generateUnresolvedIdentifier(
-				sbjs.add(getSubject(first.getSbj(), SDG_CLASS_ID, SDG_CLASS_NAME, UPDATE_SUBJECT_SDG_CLASS_ID));
+								ModelSupport.entityIdPrefix.get(Result.class.getSimpleName().toLowerCase()) + "|" + k,
-				it
+								DOI),
-					.forEachRemaining(
+						it),
-						s -> sbjs
+				Encoders.bean(Result.class))
-							.add(getSubject(s.getSbj(), SDG_CLASS_ID, SDG_CLASS_NAME, UPDATE_SUBJECT_SDG_CLASS_ID)));
-				r.setSubject(sbjs);
-				r
-					.setDataInfo(
-						OafMapperUtils
-							.dataInfo(
-								false, null, true,
-								false,
-								OafMapperUtils
-									.qualifier(
-										ModelConstants.PROVENANCE_ENRICH,
-										null,
-										ModelConstants.DNET_PROVENANCE_ACTIONS,
-										ModelConstants.DNET_PROVENANCE_ACTIONS),
-								null));
-				return r;
-			}, Encoders.bean(Result.class))
 			.write()
 			.mode(SaveMode.Overwrite)
 			.option("compression", "gzip")
 			.json(outputPath + "/sdg");
 	}
 
+	private static void doPrepareoaid(SparkSession spark, String sourcePath, String outputPath) {
+		Dataset<Row> sdgDataset = spark
+			.read()
+			.format("csv")
+			.option("sep", DEFAULT_DELIMITER)
+			.option("inferSchema", "true")
+			.option("header", "true")
+			.option("quotes", "\"")
+			.load(sourcePath);
+		;
+
+		sdgDataset
+			.groupByKey((MapFunction<Row, String>) r -> "50|" + ((String) r.getAs("oaid")), Encoders.STRING())
+			.mapGroups(
+				(MapGroupsFunction<String, Row, Result>) PrepareSDGSparkJob::getResult, Encoders.bean(Result.class))
+			.write()
+			.mode(SaveMode.Overwrite)
+			.option("compression", "gzip")
+			.json(outputPath + "/sdg");
+	}
+
+	private static @NotNull Result getResult(String id, Iterator<Row> it) {
+		Result r = new Result();
+		r.setId(id);
+		Row first = it.next();
+		List<Subject> sbjs = new ArrayList<>();
+		sbjs.add(getSubject(first.getAs("sdg"), SDG_CLASS_ID, SDG_CLASS_NAME, UPDATE_SUBJECT_SDG_CLASS_ID));
+		it
+			.forEachRemaining(
+				s -> sbjs
+					.add(getSubject(s.getAs("sdg"), SDG_CLASS_ID, SDG_CLASS_NAME, UPDATE_SUBJECT_SDG_CLASS_ID)));
+		r.setSubject(sbjs);
+
+		return r;
+	}
+
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/fosnodoi/CreateActionSetSparkJob.java

@@ -13,9 +13,6 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.function.FilterFunction;
-import org.apache.spark.api.java.function.FlatMapFunction;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SparkSession;
@@ -24,13 +21,9 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.opencitations.model.COCI;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.schema.action.AtomicAction;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.Relation;
 import eu.dnetlib.dhp.schema.oaf.Result;
-import eu.dnetlib.dhp.schema.oaf.utils.*;
 import scala.Tuple2;
 
 public class CreateActionSetSparkJob implements Serializable {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java

@@ -49,6 +49,9 @@ public class ReadCOCI implements Serializable {
 		final String workingPath = parser.get("inputPath");
 		log.info("workingPath {}", workingPath);
 
+		final String backupPath = parser.get("backupPath");
+		log.info("backupPath {}", backupPath);
+
 		SparkConf sconf = new SparkConf();
 
 		Configuration conf = new Configuration();
@@ -68,12 +71,14 @@ public class ReadCOCI implements Serializable {
 					workingPath,
 					fileSystem,
 					outputPath,
+					backupPath,
 					delimiter);
 			});
 	}
 
 	private static void doRead(SparkSession spark, String workingPath, FileSystem fileSystem,
 		String outputPath,
+		String backupPath,
 		String delimiter) throws IOException {
 		RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
 			.listFiles(
@@ -107,7 +112,8 @@ public class ReadCOCI implements Serializable {
 				.mode(SaveMode.Append)
 				.option("compression", "gzip")
 				.json(outputPath);
-			fileSystem.rename(fileStatus.getPath(), new Path("/tmp/miriam/OC/DONE"));
+
+			fileSystem.rename(fileStatus.getPath(), new Path(backupPath));
 		}
 
 	}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/personentity/CoAuthorshipIterator.java

@@ -0,0 +1,80 @@
+
+package eu.dnetlib.dhp.actionmanager.personentity;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.Person;
+import eu.dnetlib.dhp.schema.oaf.Relation;
+import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
+import eu.dnetlib.dhp.utils.DHPUtils;
+
+public class CoAuthorshipIterator implements Iterator<Relation> {
+	private int firstIndex;
+	private int secondIndex;
+	private boolean firstRelation;
+	private List<String> authors;
+	private static final String PERSON_PREFIX = ModelSupport.getIdPrefix(Person.class) + "|orcid_______::";
+	private static final String OPENAIRE_PREFIX = "openaire____";
+	private static final String SEPARATOR = "::";
+	private static final String ORCID_KEY = "10|" + OPENAIRE_PREFIX + SEPARATOR
+		+ DHPUtils.md5(ModelConstants.ORCID.toLowerCase());
+	public static final String ORCID_AUTHORS_CLASSID = "sysimport:crosswalk:orcid";
+	public static final String ORCID_AUTHORS_CLASSNAME = "Imported from ORCID";
+
+	@Override
+	public boolean hasNext() {
+		return firstIndex < authors.size() - 1;
+	}
+
+	@Override
+	public Relation next() {
+		Relation rel = null;
+		if (firstRelation) {
+			rel = getRelation(authors.get(firstIndex), authors.get(secondIndex));
+			firstRelation = Boolean.FALSE;
+		} else {
+			rel = getRelation(authors.get(secondIndex), authors.get(firstIndex));
+			firstRelation = Boolean.TRUE;
+			secondIndex += 1;
+			if (secondIndex >= authors.size()) {
+				firstIndex += 1;
+				secondIndex = firstIndex + 1;
+			}
+		}
+
+		return rel;
+	}
+
+	public CoAuthorshipIterator(List<String> authors) {
+		this.authors = authors;
+		this.firstIndex = 0;
+		this.secondIndex = 1;
+		this.firstRelation = Boolean.TRUE;
+
+	}
+
+	private Relation getRelation(String orcid1, String orcid2) {
+		String source = PERSON_PREFIX + IdentifierFactory.md5(orcid1);
+		String target = PERSON_PREFIX + IdentifierFactory.md5(orcid2);
+		return OafMapperUtils
+			.getRelation(
+				source, target, ModelConstants.PERSON_PERSON_RELTYPE,
+				ModelConstants.PERSON_PERSON_SUBRELTYPE,
+				ModelConstants.PERSON_PERSON_HASCOAUTHORED,
+				Arrays.asList(OafMapperUtils.keyValue(ORCID_KEY, ModelConstants.ORCID_DS)),
+				OafMapperUtils
+					.dataInfo(
+						false, null, false, false,
+						OafMapperUtils
+							.qualifier(
+								ORCID_AUTHORS_CLASSID, ORCID_AUTHORS_CLASSNAME,
+								ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS),
+						"0.91"),
+				null);
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/personentity/Coauthors.java

@@ -0,0 +1,20 @@
+
+package eu.dnetlib.dhp.actionmanager.personentity;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import eu.dnetlib.dhp.schema.oaf.Relation;
+
+public class Coauthors implements Serializable {
+	private List<String> coauthors;
+
+	public List<String> getCoauthors() {
+		return coauthors;
+	}
+
+	public void setCoauthors(List<String> coauthors) {
+		this.coauthors = coauthors;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/personentity/Couples.java

@@ -0,0 +1,40 @@
+
+package eu.dnetlib.dhp.actionmanager.personentity;
+
+import java.io.Serializable;
+
+import eu.dnetlib.dhp.schema.oaf.Person;
+import eu.dnetlib.dhp.schema.oaf.Relation;
+import scala.Tuple2;
+
+public class Couples implements Serializable {
+	Person p;
+	Relation r;
+
+	public Couples() {
+
+	}
+
+	public Person getP() {
+		return p;
+	}
+
+	public void setP(Person p) {
+		this.p = p;
+	}
+
+	public Relation getR() {
+		return r;
+	}
+
+	public void setR(Relation r) {
+		this.r = r;
+	}
+
+	public static <Tuples> Couples newInstance(Tuple2<Person, Relation> couple) {
+		Couples c = new Couples();
+		c.p = couple._1();
+		c.r = couple._2();
+		return c;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/personentity/ExtractPerson.java

@@ -0,0 +1,437 @@
+
+package eu.dnetlib.dhp.actionmanager.personentity;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+import static org.apache.spark.sql.functions.*;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.*;
+import org.apache.spark.sql.*;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.spark_project.jetty.util.StringUtil;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.collection.orcid.model.Author;
+import eu.dnetlib.dhp.collection.orcid.model.Employment;
+import eu.dnetlib.dhp.collection.orcid.model.Work;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Person;
+import eu.dnetlib.dhp.schema.oaf.Relation;
+import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
+import eu.dnetlib.dhp.schema.oaf.utils.PidCleaner;
+import eu.dnetlib.dhp.schema.oaf.utils.PidType;
+import eu.dnetlib.dhp.utils.DHPUtils;
+import scala.Tuple2;
+
+public class ExtractPerson implements Serializable {
+	private static final Logger log = LoggerFactory.getLogger(ExtractPerson.class);
+
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+	private static final String OPENAIRE_PREFIX = "openaire____";
+	private static final String SEPARATOR = "::";
+	private static final String orcidKey = "10|" + OPENAIRE_PREFIX + SEPARATOR
+		+ DHPUtils.md5(ModelConstants.ORCID.toLowerCase());
+
+	private static final String DOI_PREFIX = "50|doi_________::";
+
+	private static final String PMID_PREFIX = "50|pmid________::";
+	private static final String ARXIV_PREFIX = "50|arXiv_______::";
+
+	private static final String PMCID_PREFIX = "50|pmcid_______::";
+	private static final String ROR_PREFIX = "20|ror_________::";
+	private static final String PERSON_PREFIX = ModelSupport.getIdPrefix(Person.class) + "|orcid_______";
+	public static final String ORCID_AUTHORS_CLASSID = "sysimport:crosswalk:orcid";
+	public static final String ORCID_AUTHORS_CLASSNAME = "Imported from ORCID";
+
+	public static void main(final String[] args) throws IOException, ParseException {
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					Objects
+						.requireNonNull(
+							ExtractPerson.class
+								.getResourceAsStream(
+									"/eu/dnetlib/dhp/actionmanager/personentity/as_parameters.json"))));
+
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		final String inputPath = parser.get("inputPath");
+		log.info("inputPath {}", inputPath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath {}", outputPath);
+
+		final String workingDir = parser.get("workingDir");
+		log.info("workingDir {}", workingDir);
+
+		SparkConf conf = new SparkConf();
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration());
+				createActionSet(spark, inputPath, outputPath, workingDir);
+			});
+
+	}
+
+	private static void createActionSet(SparkSession spark, String inputPath, String outputPath, String workingDir) {
+
+		Dataset<Author> authors = spark
+			.read()
+			.parquet(inputPath + "Authors")
+			.as(Encoders.bean(Author.class));
+
+		Dataset<Work> works = spark
+			.read()
+			.parquet(inputPath + "Works")
+			.as(Encoders.bean(Work.class))
+			.filter(
+				(FilterFunction<Work>) w -> Optional.ofNullable(w.getPids()).isPresent() &&
+					w
+						.getPids()
+						.stream()
+						.anyMatch(
+							p -> p.getSchema().equalsIgnoreCase("doi") ||
+								p.getSchema().equalsIgnoreCase("pmc") ||
+								p.getSchema().equalsIgnoreCase("pmid") ||
+								p.getSchema().equalsIgnoreCase("arxiv")));
+
+		Dataset<Employment> employmentDataset = spark
+			.read()
+			.parquet(inputPath + "Employments")
+			.as(Encoders.bean(Employment.class));
+
+		Dataset<Author> peopleToMap = authors
+			.joinWith(works, authors.col("orcid").equalTo(works.col("orcid")))
+			.map((MapFunction<Tuple2<Author, Work>, Author>) t2 -> t2._1(), Encoders.bean(Author.class))
+			.groupByKey((MapFunction<Author, String>) a -> a.getOrcid(), Encoders.STRING())
+			.mapGroups((MapGroupsFunction<String, Author, Author>) (k, it) -> it.next(), Encoders.bean(Author.class));
+
+		Dataset<Employment> employment = employmentDataset
+			.joinWith(peopleToMap, employmentDataset.col("orcid").equalTo(peopleToMap.col("orcid")))
+			.map((MapFunction<Tuple2<Employment, Author>, Employment>) t2 -> t2._1(), Encoders.bean(Employment.class));
+
+		Dataset<Person> people;
+		peopleToMap.map((MapFunction<Author, Person>) op -> {
+			Person person = new Person();
+			person.setId(DHPUtils.generateIdentifier(op.getOrcid(), PERSON_PREFIX));
+			person
+				.setBiography(
+					Optional
+						.ofNullable(op.getBiography())
+
+						.orElse(""));
+			KeyValue kv = OafMapperUtils.keyValue(orcidKey, ModelConstants.ORCID_DS);
+			kv.setDataInfo(null);
+			person.setCollectedfrom(Arrays.asList(kv));
+			person
+				.setAlternativeNames(
+					Optional
+						.ofNullable(op.getOtherNames())
+
+						.orElse(new ArrayList<>()));
+			person
+				.setFamilyName(
+					Optional
+						.ofNullable(op.getFamilyName())
+
+						.orElse(""));
+			person
+				.setGivenName(
+					Optional
+						.ofNullable(op.getGivenName())
+
+						.orElse(""));
+			person
+				.setPid(
+					Optional
+						.ofNullable(op.getOtherPids())
+						.map(
+							v -> v
+								.stream()
+								.map(
+									p -> OafMapperUtils
+										.structuredProperty(
+											p.getValue(), p.getSchema(), p.getSchema(), ModelConstants.DNET_PID_TYPES,
+											ModelConstants.DNET_PID_TYPES, null))
+								.collect(Collectors.toList()))
+						.orElse(new ArrayList<>()));
+			person
+				.getPid()
+				.add(
+					OafMapperUtils
+						.structuredProperty(
+							op.getOrcid(), ModelConstants.ORCID, ModelConstants.ORCID_CLASSNAME,
+							ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, null));
+			person.setDateofcollection(op.getLastModifiedDate());
+			person.setOriginalId(Arrays.asList(op.getOrcid()));
+			return person;
+		}, Encoders.bean(Person.class))
+			.write()
+			.option("compression", "gzip")
+			.mode(SaveMode.Overwrite)
+			.json(workingDir + "/people");
+
+		works
+			.flatMap(
+				(FlatMapFunction<Work, Relation>) ExtractPerson::getAuthorshipRelationIterator,
+				Encoders.bean(Relation.class))
+			.write()
+			.option("compression", "gzip")
+			.mode(SaveMode.Overwrite)
+			.json(workingDir + "/authorship");
+
+		Dataset<Relation> coauthorship = works
+			.flatMap((FlatMapFunction<Work, Tuple2<String, String>>) w -> {
+				List<Tuple2<String, String>> lista = new ArrayList<>();
+				w.getPids().stream().forEach(p -> {
+					if (p.getSchema().equalsIgnoreCase("doi") || p.getSchema().equalsIgnoreCase("pmc")
+						|| p.getSchema().equalsIgnoreCase("pmid") || p.getSchema().equalsIgnoreCase("arxiv"))
+						lista.add(new Tuple2<>(p.getValue(), w.getOrcid()));
+				});
+				return lista.iterator();
+			}, Encoders.tuple(Encoders.STRING(), Encoders.STRING()))
+			.groupByKey((MapFunction<Tuple2<String, String>, String>) Tuple2::_1, Encoders.STRING())
+			.mapGroups(
+				(MapGroupsFunction<String, Tuple2<String, String>, Coauthors>) (k, it) -> extractCoAuthors(it),
+				Encoders.bean(Coauthors.class))
+			.flatMap(
+				(FlatMapFunction<Coauthors, Relation>) c -> new CoAuthorshipIterator(c.getCoauthors()),
+				Encoders.bean(Relation.class))
+			.groupByKey((MapFunction<Relation, String>) r -> r.getSource() + r.getTarget(), Encoders.STRING())
+			.mapGroups(
+				(MapGroupsFunction<String, Relation, Relation>) (k, it) -> it.next(), Encoders.bean(Relation.class));
+
+		coauthorship
+			.write()
+			.option("compression", "gzip")
+			.mode(SaveMode.Overwrite)
+			.json(workingDir + "/coauthorship");
+
+		employment
+			.filter((FilterFunction<Employment>) e -> Optional.ofNullable(e.getAffiliationId()).isPresent())
+			.filter((FilterFunction<Employment>) e -> e.getAffiliationId().getSchema().equalsIgnoreCase("ror"))
+			.map(
+				(MapFunction<Employment, Relation>) ExtractPerson::getAffiliationRelation,
+				Encoders.bean(Relation.class))
+			.write()
+			.option("compression", "gzip")
+			.mode(SaveMode.Overwrite)
+			.json(workingDir + "/affiliation");
+
+		people = spark
+			.read()
+			.textFile(workingDir + "/people")
+			.map(
+				(MapFunction<String, Person>) value -> OBJECT_MAPPER
+					.readValue(value, Person.class),
+				Encoders.bean(Person.class));
+
+		people.show(false);
+		people
+			.toJavaRDD()
+			.map(p -> new AtomicAction(p.getClass(), p))
+			.union(
+				getRelations(spark, workingDir + "/authorship").toJavaRDD().map(r -> new AtomicAction(r.getClass(), r)))
+			.union(
+				getRelations(spark, workingDir + "/coauthorship")
+					.toJavaRDD()
+					.map(r -> new AtomicAction(r.getClass(), r)))
+			.union(
+				getRelations(spark, workingDir + "/affiliation")
+					.toJavaRDD()
+					.map(r -> new AtomicAction(r.getClass(), r)))
+			.mapToPair(
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+			.saveAsHadoopFile(
+				outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, BZip2Codec.class);
+	}
+
+	private static Dataset<Relation> getRelations(SparkSession spark, String path) {
+		return spark
+			.read()
+			.textFile(path)
+			.map(
+				(MapFunction<String, Relation>) value -> OBJECT_MAPPER
+					.readValue(value, Relation.class),
+				Encoders.bean(Relation.class));// spark.read().json(path).as(Encoders.bean(Relation.class));
+	}
+
+	private static Coauthors extractCoAuthors(Iterator<Tuple2<String, String>> it) {
+		Coauthors coauth = new Coauthors();
+		List<String> coauthors = new ArrayList<>();
+		while (it.hasNext())
+			coauthors.add(it.next()._2());
+		coauth.setCoauthors(coauthors);
+
+		return coauth;
+	}
+
+	private static Relation getAffiliationRelation(Employment row) {
+		String source = PERSON_PREFIX + IdentifierFactory.md5(row.getOrcid());
+		String target = ROR_PREFIX
+			+ IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAffiliationId().getValue()));
+		List<KeyValue> properties = new ArrayList<>();
+
+		Relation relation = OafMapperUtils
+			.getRelation(
+				source, target, ModelConstants.ORG_PERSON_RELTYPE, ModelConstants.ORG_PERSON_SUBRELTYPE,
+				ModelConstants.ORG_PERSON_PARTICIPATES,
+				Arrays.asList(OafMapperUtils.keyValue(orcidKey, ModelConstants.ORCID_DS)),
+				OafMapperUtils
+					.dataInfo(
+						false, null, false, false,
+						OafMapperUtils
+							.qualifier(
+								ORCID_AUTHORS_CLASSID, ORCID_AUTHORS_CLASSNAME, ModelConstants.DNET_PROVENANCE_ACTIONS,
+								ModelConstants.DNET_PROVENANCE_ACTIONS),
+						"0.91"),
+				null);
+
+		if (Optional.ofNullable(row.getStartDate()).isPresent() && StringUtil.isNotBlank(row.getStartDate())) {
+			KeyValue kv = new KeyValue();
+			kv.setKey("startDate");
+			kv.setValue(row.getStartDate());
+			properties.add(kv);
+		}
+		if (Optional.ofNullable(row.getEndDate()).isPresent() && StringUtil.isNotBlank(row.getEndDate())) {
+			KeyValue kv = new KeyValue();
+			kv.setKey("endDate");
+			kv.setValue(row.getEndDate());
+			properties.add(kv);
+		}
+
+		if (properties.size() > 0)
+			relation.setProperties(properties);
+		return relation;
+
+	}
+
+	private static Collection<? extends Relation> getCoAuthorshipRelations(String orcid1, String orcid2) {
+		String source = PERSON_PREFIX + "::" + IdentifierFactory.md5(orcid1);
+		String target = PERSON_PREFIX + "::" + IdentifierFactory.md5(orcid2);
+
+		return Arrays
+			.asList(
+				OafMapperUtils
+					.getRelation(
+						source, target, ModelConstants.PERSON_PERSON_RELTYPE,
+						ModelConstants.PERSON_PERSON_SUBRELTYPE,
+						ModelConstants.PERSON_PERSON_HASCOAUTHORED,
+						Arrays.asList(OafMapperUtils.keyValue(orcidKey, ModelConstants.ORCID_DS)),
+						OafMapperUtils
+							.dataInfo(
+								false, null, false, false,
+								OafMapperUtils
+									.qualifier(
+										ORCID_AUTHORS_CLASSID, ORCID_AUTHORS_CLASSNAME,
+										ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS),
+								"0.91"),
+						null),
+				OafMapperUtils
+					.getRelation(
+						target, source, ModelConstants.PERSON_PERSON_RELTYPE,
+						ModelConstants.PERSON_PERSON_SUBRELTYPE,
+						ModelConstants.PERSON_PERSON_HASCOAUTHORED,
+						Arrays.asList(OafMapperUtils.keyValue(orcidKey, ModelConstants.ORCID_DS)),
+						OafMapperUtils
+							.dataInfo(
+								false, null, false, false,
+								OafMapperUtils
+									.qualifier(
+										ORCID_AUTHORS_CLASSID, ORCID_AUTHORS_CLASSNAME,
+										ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS),
+								"0.91"),
+						null));
+
+	}
+
+	private static @NotNull Iterator<Relation> getAuthorshipRelationIterator(Work w) {
+
+		if (Optional.ofNullable(w.getPids()).isPresent())
+			return w
+				.getPids()
+				.stream()
+				.map(pid -> getRelation(w.getOrcid(), pid))
+				.filter(Objects::nonNull)
+				.collect(Collectors.toList())
+				.iterator();
+		List<Relation> ret = new ArrayList<>();
+		return ret.iterator();
+	}
+
+	private static Relation getRelation(String orcid, eu.dnetlib.dhp.collection.orcid.model.Pid pid) {
+		String target;
+		String source = PERSON_PREFIX + "::" + IdentifierFactory.md5(orcid);
+		switch (pid.getSchema()) {
+			case "doi":
+				target = DOI_PREFIX
+					+ IdentifierFactory
+						.md5(PidCleaner.normalizePidValue(PidType.doi.toString(), pid.getValue()));
+				break;
+			case "pmid":
+				target = PMID_PREFIX
+					+ IdentifierFactory
+						.md5(PidCleaner.normalizePidValue(PidType.pmid.toString(), pid.getValue()));
+				break;
+			case "arxiv":
+				target = ARXIV_PREFIX
+					+ IdentifierFactory
+						.md5(PidCleaner.normalizePidValue(PidType.arXiv.toString(), pid.getValue()));
+				break;
+			case "pmcid":
+				target = PMCID_PREFIX
+					+ IdentifierFactory
+						.md5(PidCleaner.normalizePidValue(PidType.pmc.toString(), pid.getValue()));
+				break;
+
+			default:
+				return null;
+		}
+
+		return OafMapperUtils
+			.getRelation(
+				source, target, ModelConstants.RESULT_PERSON_RELTYPE,
+				ModelConstants.RESULT_PERSON_SUBRELTYPE,
+				ModelConstants.RESULT_PERSON_HASAUTHORED,
+				Arrays.asList(OafMapperUtils.keyValue(orcidKey, ModelConstants.ORCID_DS)),
+				OafMapperUtils
+					.dataInfo(
+						false, null, false, false,
+						OafMapperUtils
+							.qualifier(
+								ORCID_AUTHORS_CLASSID, ORCID_AUTHORS_CLASSNAME, ModelConstants.DNET_PROVENANCE_ACTIONS,
+								ModelConstants.DNET_PROVENANCE_ACTIONS),
+						"0.91"),
+				null);
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/personentity/WorkList.java

@@ -0,0 +1,25 @@
+
+package eu.dnetlib.dhp.actionmanager.personentity;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+
+import eu.dnetlib.dhp.collection.orcid.model.Work;
+
+public class WorkList implements Serializable {
+	private ArrayList<Work> workArrayList;
+
+	public ArrayList<Work> getWorkArrayList() {
+		return workArrayList;
+	}
+
+	public void setWorkArrayList(ArrayList<Work> workArrayList) {
+		this.workArrayList = workArrayList;
+	}
+
+	public WorkList() {
+		workArrayList = new ArrayList<>();
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/sdgnodoi/CreateActionSetSparkJob.java

@@ -0,0 +1,91 @@
+
+package eu.dnetlib.dhp.actionmanager.sdgnodoi;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.Hdfs;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.oaf.Result;
+import scala.Tuple2;
+
+public class CreateActionSetSparkJob implements Serializable {
+
+	private static final Logger log = LoggerFactory.getLogger(CreateActionSetSparkJob.class);
+
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	public static void main(final String[] args) throws IOException, ParseException {
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					Objects
+						.requireNonNull(
+							CreateActionSetSparkJob.class
+								.getResourceAsStream(
+									"/eu/dnetlib/dhp/actionmanager/fosnodoi/as_parameters.json"))));
+
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		final String inputPath = parser.get("sourcePath");
+		log.info("inputPath {}", inputPath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath {}", outputPath);
+
+		SparkConf conf = new SparkConf();
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration());
+				createActionSet(spark, inputPath, outputPath);
+			});
+
+	}
+
+	private static void createActionSet(SparkSession spark, String inputPath, String outputPath) {
+		spark
+			.read()
+			.textFile(inputPath)
+			.map(
+				(MapFunction<String, Result>) value -> OBJECT_MAPPER.readValue(value, Result.class),
+				Encoders.bean(Result.class))
+			.toJavaRDD()
+			.map(p -> new AtomicAction(p.getClass(), p))
+			.mapToPair(
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+			.saveAsHadoopFile(
+				outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class);
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java

@@ -5,13 +5,13 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 
 import java.io.Serializable;
 import java.util.*;
-import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.BZip2Codec;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.FilterFunction;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.apache.spark.sql.*;
 import org.apache.spark.sql.types.StructType;
@@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import eu.dnetlib.dhp.actionmanager.Constants;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.schema.action.AtomicAction;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
@@ -43,8 +44,7 @@ public class CreateActionSetFromWebEntries implements Serializable {
 	private static final String PMID_PREFIX = "50|pmid________::";
 
 	private static final String PMCID_PREFIX = "50|pmc_________::";
-	private static final String WEB_CRAWL_ID = "10|openaire____::fb98a192f6a055ba495ef414c330834b";
+
-	private static final String WEB_CRAWL_NAME = "Web Crawl";
 	public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
 	public static void main(String[] args) throws Exception {
@@ -70,6 +70,9 @@ public class CreateActionSetFromWebEntries implements Serializable {
 		final String outputPath = parser.get("outputPath");
 		log.info("outputPath: {}", outputPath);
 
+		final String blackListInputPath = parser.get("blackListPath");
+		log.info("blackListInputPath: {}", blackListInputPath);
+
 		SparkConf conf = new SparkConf();
 
 		runWithSparkSession(
@@ -77,35 +80,39 @@ public class CreateActionSetFromWebEntries implements Serializable {
 			isSparkSessionManaged,
 			spark -> {
 
-				createActionSet(spark, inputPath, outputPath);
+				createActionSet(spark, inputPath, outputPath, blackListInputPath);
 
 			});
 	}
 
 	public static void createActionSet(SparkSession spark, String inputPath,
-		String outputPath) {
+		String outputPath, String blackListInputPath) {
 
 		final Dataset<Row> dataset = readWebCrawl(spark, inputPath)
-			.filter("publication_year <= 2020 or country_code=='IE'")
+			.filter("country_code=='IE'")
 			.drop("publication_year");
 
-		dataset.flatMap((FlatMapFunction<Row, Relation>) row -> {
+		final Dataset<Row> blackList = readBlackList(spark, blackListInputPath);
-			List<Relation> ret = new ArrayList<>();
-			final String ror = ROR_PREFIX
-				+ IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror")));
-			ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror));
-			ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror));
-			ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror));
 
-			return ret
+		dataset
-				.iterator();
+			.join(blackList, dataset.col("id").equalTo(blackList.col("OpenAlexId")), "left")
-		}, Encoders.bean(Relation.class))
+			.filter((FilterFunction<Row>) r -> r.getAs("OpenAlexId") == null)
+			.drop("OpenAlexId")
+			.flatMap((FlatMapFunction<Row, Relation>) row -> {
+				List<Relation> ret = new ArrayList<>();
+				final String ror = ROR_PREFIX
+					+ IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror")));
+				ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror));
+
+				return ret
+					.iterator();
+			}, Encoders.bean(Relation.class))
 			.toJavaRDD()
 			.map(p -> new AtomicAction(p.getClass(), p))
 			.mapToPair(
 				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
 					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
-			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class);
+			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, BZip2Codec.class);
 
 	}
 
@@ -129,13 +136,22 @@ public class CreateActionSetFromWebEntries implements Serializable {
 				"institution", functions
 					.explode(
 						functions.col("institutions")))
+
 			.selectExpr(
-				"id", "doi", "ids.pmcid as pmcid", "ids.pmid as pmid", "institution.ror as ror",
+				"id", "doi", "institution.ror as ror",
 				"institution.country_code as country_code", "publication_year")
 			.distinct();
 
 	}
 
+	private static Dataset<Row> readBlackList(SparkSession spark, String inputPath) {
+
+		return spark
+			.read()
+			.json(inputPath)
+			.select("OpenAlexId");
+	}
+
 	private static List<Relation> createAffiliationRelationPairPMCID(String pmcid, String ror) {
 		if (pmcid == null)
 			return new ArrayList<>();
@@ -195,7 +211,7 @@ public class CreateActionSetFromWebEntries implements Serializable {
 						ModelConstants.IS_AUTHOR_INSTITUTION_OF,
 						Arrays
 							.asList(
-								OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)),
+								OafMapperUtils.keyValue(Constants.WEB_CRAWL_ID, Constants.WEB_CRAWL_NAME)),
 						OafMapperUtils
 							.dataInfo(
 								false, null, false, false,
@@ -214,7 +230,7 @@ public class CreateActionSetFromWebEntries implements Serializable {
 						ModelConstants.HAS_AUTHOR_INSTITUTION,
 						Arrays
 							.asList(
-								OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)),
+								OafMapperUtils.keyValue(Constants.WEB_CRAWL_ID, Constants.WEB_CRAWL_NAME)),
 						OafMapperUtils
 							.dataInfo(
 								false, null, false, false,

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/RemoveRelationFromActionSet.java

@@ -0,0 +1,158 @@
+
+package eu.dnetlib.dhp.actionmanager.webcrawl;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+import static org.apache.spark.sql.functions.*;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Optional;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.filefilter.DirectoryFileFilter;
+import org.apache.commons.io.filefilter.FileFileFilter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.FilterFunction;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.*;
+import org.apache.spark.sql.types.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.oaf.Relation;
+import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
+import scala.Tuple2;
+
+public class RemoveRelationFromActionSet
+	implements Serializable {
+	private static final Logger log = LoggerFactory.getLogger(CreateActionSetFromWebEntries.class);
+
+	private static final ObjectMapper MAPPER = new ObjectMapper();
+	private static final StructType KV_SCHEMA = StructType$.MODULE$
+		.apply(
+			Arrays
+				.asList(
+					StructField$.MODULE$.apply("key", DataTypes.StringType, false, Metadata.empty()),
+					StructField$.MODULE$.apply("value", DataTypes.StringType, false, Metadata.empty())));
+
+	private static final StructType ATOMIC_ACTION_SCHEMA = StructType$.MODULE$
+		.apply(
+			Arrays
+				.asList(
+					StructField$.MODULE$.apply("clazz", DataTypes.StringType, false, Metadata.empty()),
+					StructField$.MODULE$
+						.apply(
+							"payload", DataTypes.StringType, false, Metadata.empty())));
+
+	public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	public static void main(String[] args) throws Exception {
+		String jsonConfiguration = IOUtils
+			.toString(
+				CreateActionSetFromWebEntries.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		// the actionSet path
+		final String inputPath = parser.get("sourcePath");
+		log.info("inputPath: {}", inputPath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath: {}", outputPath);
+
+		final String blackListInputPath = parser.get("blackListPath");
+		log.info("blackListInputPath: {}", blackListInputPath);
+
+		SparkConf conf = new SparkConf();
+
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+
+				removeFromActionSet(spark, inputPath, outputPath, blackListInputPath);
+
+			});
+	}
+
+	private static void removeFromActionSet(SparkSession spark, String inputPath, String outputPath,
+		String blackListInputPath) {
+		// read the blacklist
+		Dataset<String> blackList = readBlackList(spark, blackListInputPath)
+			.map(
+				(MapFunction<Row, String>) r -> IdentifierFactory
+					.idFromPid("50", "doi", ((String) r.getAs("doi")).substring(16), true),
+				Encoders.STRING());
+
+		// read the old actionset and get the relations in the payload
+		JavaPairRDD<Text, Text> seq = JavaSparkContext
+			.fromSparkContext(spark.sparkContext())
+			.sequenceFile(inputPath, Text.class, Text.class);
+
+		JavaRDD<Row> rdd = seq
+			.map(x -> RowFactory.create(x._1().toString(), x._2().toString()));
+
+		Dataset<Row> actionSet = spark
+			.createDataFrame(rdd, KV_SCHEMA)
+			.withColumn("atomic_action", from_json(col("value"), ATOMIC_ACTION_SCHEMA))
+			.select(expr("atomic_action.*"));
+
+		Dataset<Relation> relation = actionSet
+			.map(
+				(MapFunction<Row, Relation>) r -> MAPPER.readValue((String) r.getAs("payload"), Relation.class),
+				Encoders.bean(Relation.class));
+
+		// select only the relation not matching any pid in the blacklist as source for the relation
+		Dataset<Relation> relNoSource = relation
+			.joinWith(blackList, relation.col("source").equalTo(blackList.col("value")), "left")
+			.filter((FilterFunction<Tuple2<Relation, String>>) t2 -> t2._2() == null)
+			.map((MapFunction<Tuple2<Relation, String>, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class));
+
+		// select only the relation not matching any pid in the blacklist as target of the relation
+		relNoSource
+			.joinWith(blackList, relNoSource.col("target").equalTo(blackList.col("value")), "left")
+			.filter((FilterFunction<Tuple2<Relation, String>>) t2 -> t2._2() == null)
+			.map((MapFunction<Tuple2<Relation, String>, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class))
+			.toJavaRDD()
+			.map(p -> new AtomicAction(p.getClass(), p))
+			.mapToPair(
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, BZip2Codec.class);
+		;
+
+	}
+
+	private static Dataset<Row> readBlackList(SparkSession spark, String inputPath) {
+
+		return spark
+			.read()
+			.json(inputPath)
+			.select("doi");
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectorWorker.java

@@ -22,9 +22,11 @@ import eu.dnetlib.dhp.collection.plugin.CollectorPlugin;
 import eu.dnetlib.dhp.collection.plugin.base.BaseCollectorPlugin;
 import eu.dnetlib.dhp.collection.plugin.file.FileCollectorPlugin;
 import eu.dnetlib.dhp.collection.plugin.file.FileGZipCollectorPlugin;
+import eu.dnetlib.dhp.collection.plugin.gtr2.Gtr2PublicationsCollectorPlugin;
 import eu.dnetlib.dhp.collection.plugin.mongodb.MDStoreCollectorPlugin;
 import eu.dnetlib.dhp.collection.plugin.mongodb.MongoDbDumpCollectorPlugin;
 import eu.dnetlib.dhp.collection.plugin.oai.OaiCollectorPlugin;
+import eu.dnetlib.dhp.collection.plugin.osf.OsfPreprintsCollectorPlugin;
 import eu.dnetlib.dhp.collection.plugin.rest.RestCollectorPlugin;
 import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
 import eu.dnetlib.dhp.common.collection.CollectorException;
@@ -58,7 +60,7 @@ public class CollectorWorker extends ReportingJob {
 
 	public void collect() throws UnknownCollectorPluginException, CollectorException, IOException {
 
-		final String outputPath = mdStoreVersion.getHdfsPath() + SEQUENCE_FILE_NAME;
+		final String outputPath = this.mdStoreVersion.getHdfsPath() + SEQUENCE_FILE_NAME;
 		log.info("outputPath path is {}", outputPath);
 
 		final CollectorPlugin plugin = getCollectorPlugin();
@@ -68,36 +70,36 @@ public class CollectorWorker extends ReportingJob {
 
 		try (SequenceFile.Writer writer = SequenceFile
 			.createWriter(
-				fileSystem.getConf(),
+				this.fileSystem.getConf(), SequenceFile.Writer.file(new Path(outputPath)), SequenceFile.Writer
-				SequenceFile.Writer.file(new Path(outputPath)),
+					.keyClass(IntWritable.class),
-				SequenceFile.Writer.keyClass(IntWritable.class),
+				SequenceFile.Writer
-				SequenceFile.Writer.valueClass(Text.class),
+					.valueClass(Text.class),
 				SequenceFile.Writer.compression(SequenceFile.CompressionType.BLOCK, new DeflateCodec()))) {
 			final IntWritable key = new IntWritable(counter.get());
 			final Text value = new Text();
 			plugin
-				.collect(api, report)
+				.collect(this.api, this.report)
-				.forEach(
+				.forEach(content -> {
-					content -> {
+					key.set(counter.getAndIncrement());
-						key.set(counter.getAndIncrement());
+					value.set(content);
-						value.set(content);
+					try {
-						try {
+						writer.append(key, value);
-							writer.append(key, value);
+					} catch (final Throwable e) {
-						} catch (Throwable e) {
+						throw new RuntimeException(e);
-							throw new RuntimeException(e);
+					}
-						}
+				});
-					});
+		} catch (final Throwable e) {
-		} catch (Throwable e) {
+			this.report.put(e.getClass().getName(), e.getMessage());
-			report.put(e.getClass().getName(), e.getMessage());
 			throw new CollectorException(e);
 		} finally {
 			shutdown();
-			report.ongoing(counter.longValue(), counter.longValue());
+			this.report.ongoing(counter.longValue(), counter.longValue());
 		}
 	}
 
-	private void scheduleReport(AtomicInteger counter) {
+	private void scheduleReport(final AtomicInteger counter) {
 		schedule(new ReporterCallback() {
+
 			@Override
 			public Long getCurrent() {
 				return counter.longValue();
@@ -112,33 +114,37 @@ public class CollectorWorker extends ReportingJob {
 
 	private CollectorPlugin getCollectorPlugin() throws UnknownCollectorPluginException {
 
-		switch (CollectorPlugin.NAME.valueOf(api.getProtocol())) {
+		switch (CollectorPlugin.NAME.valueOf(this.api.getProtocol())) {
 			case oai:
-				return new OaiCollectorPlugin(clientParams);
+				return new OaiCollectorPlugin(this.clientParams);
 			case rest_json2xml:
-				return new RestCollectorPlugin(clientParams);
+				return new RestCollectorPlugin(this.clientParams);
 			case file:
-				return new FileCollectorPlugin(fileSystem);
+				return new FileCollectorPlugin(this.fileSystem);
 			case fileGzip:
-				return new FileGZipCollectorPlugin(fileSystem);
+				return new FileGZipCollectorPlugin(this.fileSystem);
 			case baseDump:
 				return new BaseCollectorPlugin(this.fileSystem);
+			case gtr2Publications:
+				return new Gtr2PublicationsCollectorPlugin(this.clientParams);
+			case osfPreprints:
+				return new OsfPreprintsCollectorPlugin(this.clientParams);
 			case other:
 				final CollectorPlugin.NAME.OTHER_NAME plugin = Optional
-					.ofNullable(api.getParams().get("other_plugin_type"))
+					.ofNullable(this.api.getParams().get("other_plugin_type"))
 					.map(CollectorPlugin.NAME.OTHER_NAME::valueOf)
 					.orElseThrow(() -> new IllegalArgumentException("invalid other_plugin_type"));
 
 				switch (plugin) {
 					case mdstore_mongodb_dump:
-						return new MongoDbDumpCollectorPlugin(fileSystem);
+						return new MongoDbDumpCollectorPlugin(this.fileSystem);
 					case mdstore_mongodb:
 						return new MDStoreCollectorPlugin();
 					default:
 						throw new UnknownCollectorPluginException("plugin is not managed: " + plugin);
 				}
 			default:
-				throw new UnknownCollectorPluginException("protocol is not managed: " + api.getProtocol());
+				throw new UnknownCollectorPluginException("protocol is not managed: " + this.api.getProtocol());
 		}
 	}
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Author.java

@@ -20,6 +20,9 @@ public class Author extends ORCIDItem {
 
 	private String lastModifiedDate;
 
+	public Author() {
+	}
+
 	public String getBiography() {
 		return biography;
 	}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/ORCIDItem.java

@@ -11,4 +11,7 @@ public class ORCIDItem {
 	public void setOrcid(String orcid) {
 		this.orcid = orcid;
 	}
+
+	public ORCIDItem() {
+	}
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Work.java

@@ -32,4 +32,6 @@ public class Work extends ORCIDItem {
 		pids.add(pid);
 	}
 
+	public Work() {
+	}
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/CollectorPlugin.java

@@ -11,7 +11,7 @@ public interface CollectorPlugin {
 
 	enum NAME {
 
-		oai, other, rest_json2xml, file, fileGzip, baseDump;
+		oai, other, rest_json2xml, file, fileGzip, baseDump, gtr2Publications, osfPreprints;
 
 		public enum OTHER_NAME {
 			mdstore_mongodb_dump, mdstore_mongodb

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsCollectorPlugin.java

@@ -0,0 +1,43 @@
+
+package eu.dnetlib.dhp.collection.plugin.gtr2;
+
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import eu.dnetlib.dhp.collection.ApiDescriptor;
+import eu.dnetlib.dhp.collection.plugin.CollectorPlugin;
+import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
+import eu.dnetlib.dhp.common.collection.CollectorException;
+import eu.dnetlib.dhp.common.collection.HttpClientParams;
+
+public class Gtr2PublicationsCollectorPlugin implements CollectorPlugin {
+
+	private final HttpClientParams clientParams;
+
+	public Gtr2PublicationsCollectorPlugin(final HttpClientParams clientParams) {
+		this.clientParams = clientParams;
+	}
+
+	@Override
+	public Stream<String> collect(final ApiDescriptor api, final AggregatorReport report) throws CollectorException {
+
+		final String baseUrl = api.getBaseUrl();
+		final String startPage = api.getParams().get("startPage");
+		final String endPage = api.getParams().get("endPage");
+		final String fromDate = api.getParams().get("fromDate");
+
+		if ((fromDate != null) && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) {
+			throw new CollectorException("Invalid date (YYYY-MM-DD): " + fromDate);
+		}
+
+		final Iterator<String> iterator = new Gtr2PublicationsIterator(baseUrl, fromDate, startPage, endPage,
+			this.clientParams);
+		final Spliterator<String> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
+
+		return StreamSupport.stream(spliterator, false);
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIterator.java

@@ -0,0 +1,215 @@
+
+package eu.dnetlib.dhp.collection.plugin.gtr2;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.function.Function;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.common.collection.CollectorException;
+import eu.dnetlib.dhp.common.collection.HttpClientParams;
+import eu.dnetlib.dhp.common.collection.HttpConnector2;
+
+public class Gtr2PublicationsIterator implements Iterator<String> {
+
+	public static final int PAGE_SIZE = 20;
+
+	private static final Logger log = LoggerFactory.getLogger(Gtr2PublicationsIterator.class);
+
+	private final HttpConnector2 connector;
+	private static final DateTimeFormatter simpleDateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");
+
+	private static final int MAX_ATTEMPTS = 10;
+
+	private final String baseUrl;
+	private int currPage;
+	private int endPage;
+	private boolean incremental = false;
+	private DateTime fromDate;
+
+	private final Map<String, String> cache = new HashMap<>();
+
+	private final Queue<String> queue = new LinkedList<>();
+
+	private String nextElement;
+
+	public Gtr2PublicationsIterator(final String baseUrl, final String fromDate, final String startPage,
+		final String endPage,
+		final HttpClientParams clientParams)
+		throws CollectorException {
+
+		this.baseUrl = baseUrl;
+		this.currPage = NumberUtils.toInt(startPage, 1);
+		this.endPage = NumberUtils.toInt(endPage, Integer.MAX_VALUE);
+		this.incremental = StringUtils.isNotBlank(fromDate);
+		this.connector = new HttpConnector2(clientParams);
+
+		if (this.incremental) {
+			this.fromDate = parseDate(fromDate);
+		}
+
+		prepareNextElement();
+	}
+
+	@Override
+	public boolean hasNext() {
+		return this.nextElement != null;
+	}
+
+	@Override
+	public String next() {
+		try {
+			return this.nextElement;
+		} finally {
+			prepareNextElement();
+		}
+	}
+
+	@Override
+	public void remove() {
+		throw new UnsupportedOperationException();
+	}
+
+	private void prepareNextElement() {
+		while ((this.currPage <= this.endPage) && this.queue.isEmpty()) {
+			log.debug("FETCHING PAGE + " + this.currPage + "/" + this.endPage);
+			this.queue.addAll(fetchPage(this.currPage++));
+		}
+		this.nextElement = this.queue.poll();
+	}
+
+	private List<String> fetchPage(final int pageNumber) {
+
+		final List<String> res = new ArrayList<>();
+		try {
+			final Document doc = loadURL(cleanURL(this.baseUrl + "/outcomes/publications?p=" + pageNumber), 0);
+
+			if (this.endPage == Integer.MAX_VALUE) {
+				this.endPage = NumberUtils.toInt(doc.valueOf("/*/@*[local-name() = 'totalPages']"));
+			}
+
+			for (final Object po : doc.selectNodes("//*[local-name() = 'publication']")) {
+				final Element mainEntity = (Element) ((Element) po).detach();
+
+				if (filterIncremental(mainEntity)) {
+					res.add(expandMainEntity(mainEntity));
+				} else {
+					log.debug("Skipped entity");
+				}
+
+			}
+		} catch (final Throwable e) {
+			log.error("Exception fetching page " + pageNumber, e);
+			throw new RuntimeException("Exception fetching page " + pageNumber, e);
+		}
+
+		return res;
+	}
+
+	private void addLinkedEntities(final Element master, final String relType, final Element newRoot,
+		final Function<Document, Element> mapper) {
+
+		for (final Object o : master.selectNodes(".//*[local-name()='link']")) {
+			final String rel = ((Element) o).valueOf("@*[local-name()='rel']");
+			final String href = ((Element) o).valueOf("@*[local-name()='href']");
+
+			if (relType.equals(rel) && StringUtils.isNotBlank(href)) {
+				final String cacheKey = relType + "#" + href;
+				if (this.cache.containsKey(cacheKey)) {
+					try {
+						log.debug(" * from cache (" + relType + "): " + href);
+						newRoot.add(DocumentHelper.parseText(this.cache.get(cacheKey)).getRootElement());
+					} catch (final DocumentException e) {
+						log.error("Error retrieving cache element: " + cacheKey, e);
+						throw new RuntimeException("Error retrieving cache element: " + cacheKey, e);
+					}
+				} else {
+					final Document doc = loadURL(cleanURL(href), 0);
+					final Element elem = mapper.apply(doc);
+					newRoot.add(elem);
+					this.cache.put(cacheKey, elem.asXML());
+				}
+
+			}
+		}
+	}
+
+	private boolean filterIncremental(final Element e) {
+		if (!this.incremental || isAfter(e.valueOf("@*[local-name() = 'created']"), this.fromDate)
+			|| isAfter(e.valueOf("@*[local-name() = 'updated']"), this.fromDate)) {
+			return true;
+		}
+		return false;
+	}
+
+	private String expandMainEntity(final Element mainEntity) {
+		final Element newRoot = DocumentHelper.createElement("doc");
+		newRoot.add(mainEntity);
+		addLinkedEntities(mainEntity, "PROJECT", newRoot, this::asProjectElement);
+		return DocumentHelper.createDocument(newRoot).asXML();
+	}
+
+	private Element asProjectElement(final Document doc) {
+		final Element newOrg = DocumentHelper.createElement("project");
+		newOrg.addElement("id").setText(doc.valueOf("/*/@*[local-name()='id']"));
+		newOrg
+			.addElement("code")
+			.setText(doc.valueOf("//*[local-name()='identifier' and @*[local-name()='type'] = 'RCUK']"));
+		newOrg.addElement("title").setText(doc.valueOf("//*[local-name()='title']"));
+		return newOrg;
+	}
+
+	private static String cleanURL(final String url) {
+		String cleaned = url;
+		if (cleaned.contains("gtr.gtr")) {
+			cleaned = cleaned.replace("gtr.gtr", "gtr");
+		}
+		if (cleaned.startsWith("http://")) {
+			cleaned = cleaned.replaceFirst("http://", "https://");
+		}
+		return cleaned;
+	}
+
+	private Document loadURL(final String cleanUrl, final int attempt) {
+		try {
+			log.debug("  * Downloading Url: " + cleanUrl);
+			final byte[] bytes = this.connector.getInputSource(cleanUrl).getBytes("UTF-8");
+			return DocumentHelper.parseText(new String(bytes));
+		} catch (final Throwable e) {
+			log.error("Error dowloading url: " + cleanUrl + ", attempt = " + attempt, e);
+			if (attempt >= MAX_ATTEMPTS) {
+				throw new RuntimeException("Error dowloading url: " + cleanUrl, e);
+			}
+			try {
+				Thread.sleep(60000); // I wait for a minute
+			} catch (final InterruptedException e1) {
+				throw new RuntimeException("Error dowloading url: " + cleanUrl, e);
+			}
+			return loadURL(cleanUrl, attempt + 1);
+		}
+	}
+
+	private DateTime parseDate(final String s) {
+		return DateTime.parse(s.contains("T") ? s.substring(0, s.indexOf("T")) : s, simpleDateTimeFormatter);
+	}
+
+	private boolean isAfter(final String d, final DateTime fromDate) {
+		return StringUtils.isNotBlank(d) && parseDate(d).isAfter(fromDate);
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPlugin.java

@@ -0,0 +1,52 @@
+
+package eu.dnetlib.dhp.collection.plugin.osf;
+
+import java.util.Optional;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+
+import eu.dnetlib.dhp.collection.ApiDescriptor;
+import eu.dnetlib.dhp.collection.plugin.CollectorPlugin;
+import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
+import eu.dnetlib.dhp.common.collection.CollectorException;
+import eu.dnetlib.dhp.common.collection.HttpClientParams;
+
+public class OsfPreprintsCollectorPlugin implements CollectorPlugin {
+
+	public static final int PAGE_SIZE_VALUE_DEFAULT = 100;
+
+	private final HttpClientParams clientParams;
+
+	public OsfPreprintsCollectorPlugin(final HttpClientParams clientParams) {
+		this.clientParams = clientParams;
+	}
+
+	@Override
+	public Stream<String> collect(final ApiDescriptor api, final AggregatorReport report) throws CollectorException {
+		final String baseUrl = api.getBaseUrl();
+
+		final int pageSize = Optional
+			.ofNullable(api.getParams().get("pageSize"))
+			.filter(StringUtils::isNotBlank)
+			.map(s -> NumberUtils.toInt(s, PAGE_SIZE_VALUE_DEFAULT))
+			.orElse(PAGE_SIZE_VALUE_DEFAULT);
+
+		if (StringUtils.isBlank(baseUrl)) {
+			throw new CollectorException("Param 'baseUrl' is null or empty");
+		}
+
+		final OsfPreprintsIterator it = new OsfPreprintsIterator(baseUrl, pageSize, getClientParams());
+
+		return StreamSupport
+			.stream(Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), false);
+	}
+
+	public HttpClientParams getClientParams() {
+		return this.clientParams;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsIterator.java

@@ -0,0 +1,151 @@
+
+package eu.dnetlib.dhp.collection.plugin.osf;
+
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.concurrent.PriorityBlockingQueue;
+
+import org.apache.commons.lang3.StringUtils;
+import org.dom4j.Document;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.collection.plugin.utils.JsonUtils;
+import eu.dnetlib.dhp.common.collection.CollectorException;
+import eu.dnetlib.dhp.common.collection.HttpClientParams;
+import eu.dnetlib.dhp.common.collection.HttpConnector2;
+
+public class OsfPreprintsIterator implements Iterator<String> {
+
+	private static final Logger log = LoggerFactory.getLogger(OsfPreprintsIterator.class);
+
+	private static final int MAX_ATTEMPTS = 5;
+
+	private final HttpClientParams clientParams;
+
+	private final String baseUrl;
+	private final int pageSize;
+
+	private String currentUrl;
+
+	private final Queue<String> recordQueue = new PriorityBlockingQueue<>();
+
+	public OsfPreprintsIterator(
+		final String baseUrl,
+		final int pageSize,
+		final HttpClientParams clientParams) {
+
+		this.clientParams = clientParams;
+		this.baseUrl = baseUrl;
+		this.pageSize = pageSize;
+
+		initQueue();
+	}
+
+	private void initQueue() {
+		this.currentUrl = this.baseUrl + "?filter:is_published:d=true&format=json&page[size]=" + this.pageSize;
+
+		log.info("REST calls starting with {}", this.currentUrl);
+	}
+
+	@Override
+	public boolean hasNext() {
+		synchronized (this.recordQueue) {
+			while (this.recordQueue.isEmpty() && StringUtils.isNotBlank(this.currentUrl)
+				&& this.currentUrl.startsWith("http")) {
+				try {
+					this.currentUrl = downloadPage(this.currentUrl);
+				} catch (final CollectorException e) {
+					log.debug("CollectorPlugin.next()-Exception: {}", e);
+					throw new RuntimeException(e);
+				}
+			}
+
+			if (!this.recordQueue.isEmpty()) {
+				return true;
+			}
+
+			return false;
+		}
+	}
+
+	@Override
+	public String next() {
+		synchronized (this.recordQueue) {
+			return this.recordQueue.poll();
+		}
+	}
+
+	private String downloadPage(final String url) throws CollectorException {
+
+		final Document doc = downloadUrl(url, 0);
+
+		for (final Object o : doc.selectNodes("/*/data")) {
+
+			final Element n = (Element) ((Element) o).detach();
+
+			final Element group = DocumentHelper.createElement("group");
+			group.addAttribute("id", n.valueOf("./id"));
+
+			group.addElement("preprint").add(n);
+
+			for (final Object o1 : n.selectNodes(".//contributors//href")) {
+				final String href = ((Node) o1).getText();
+				if (StringUtils.isNotBlank(href) && href.startsWith("http")) {
+					final Document doc1 = downloadUrl(href, 0);
+					group.addElement("contributors").add(doc1.getRootElement().detach());
+				}
+			}
+			for (final Object o1 : n.selectNodes(".//primary_file//href")) {
+				final String href = ((Node) o1).getText();
+				if (StringUtils.isNotBlank(href) && href.startsWith("http")) {
+					final Document doc1 = downloadUrl(href, 0);
+					group.addElement("primary_file").add(doc1.getRootElement().detach());
+				}
+			}
+
+			this.recordQueue.add(DocumentHelper.createDocument(group).asXML());
+		}
+
+		return doc.valueOf("/*/links/next");
+
+	}
+
+	private Document downloadUrl(final String url, final int attempt) throws CollectorException {
+		if (attempt > MAX_ATTEMPTS) {
+			throw new CollectorException("Max Number of attempts reached, url:" + url);
+		}
+
+		if (attempt > 0) {
+			final int delay = (attempt * 5000);
+			log.debug("Attempt {} with delay {}", attempt, delay);
+			try {
+				Thread.sleep(delay);
+			} catch (final InterruptedException e) {
+				new CollectorException(e);
+			}
+		}
+
+		try {
+			log.info("requesting URL [{}]", url);
+
+			final HttpConnector2 connector = new HttpConnector2(this.clientParams);
+
+			final String json = connector.getInputSource(url);
+			final String xml = JsonUtils.convertToXML(json);
+
+			return DocumentHelper.parseText(xml);
+
+		} catch (final Throwable e) {
+			log.warn(e.getMessage(), e);
+			if ((e instanceof CollectorException) && e.getMessage().contains("401")) {
+				final Element root = DocumentHelper.createElement("error_401_authorization_required");
+				return DocumentHelper.createDocument(root);
+			}
+			return downloadUrl(url, attempt + 1);
+		}
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/researchfi/ResearchFiCollectorPlugin.java

@@ -0,0 +1,76 @@
+
+package eu.dnetlib.dhp.collection.plugin.researchfi;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.dhp.collection.ApiDescriptor;
+import eu.dnetlib.dhp.collection.plugin.CollectorPlugin;
+import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
+import eu.dnetlib.dhp.common.collection.CollectorException;
+
+public class ResearchFiCollectorPlugin implements CollectorPlugin {
+
+	private static final Logger log = LoggerFactory.getLogger(ResearchFiCollectorPlugin.class);
+
+	@Override
+	public Stream<String> collect(final ApiDescriptor api, final AggregatorReport report)
+		throws CollectorException {
+
+		final String authUrl = api.getParams().get("auth_url");
+		final String clientId = api.getParams().get("auth_client_id");
+		final String clientSecret = api.getParams().get("auth_client_secret");
+
+		final String authToken = authenticate(authUrl, clientId, clientSecret);
+
+		final Iterator<String> iter = new ResearchFiIterator(api.getBaseUrl(), authToken);
+
+		return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iter, Spliterator.ORDERED), false);
+	}
+
+	private String authenticate(final String authUrl, final String clientId, final String clientSecret)
+		throws CollectorException {
+		try (final CloseableHttpClient client = HttpClients.createDefault()) {
+			final HttpPost req = new HttpPost(authUrl);
+			final List<NameValuePair> params = new ArrayList<>();
+			params.add(new BasicNameValuePair("grant_type", "client_credentials"));
+			params.add(new BasicNameValuePair("client_id", clientId));
+			params.add(new BasicNameValuePair("client_secret", clientSecret));
+
+			req.setEntity(new UrlEncodedFormEntity(params, "UTF-8"));
+
+			try (final CloseableHttpResponse response = client.execute(req)) {
+				final String content = IOUtils.toString(response.getEntity().getContent());
+				final JSONObject obj = new JSONObject(content);
+				final String token = obj.getString("access_token");
+				if (StringUtils.isNotBlank(token)) {
+					return token;
+				}
+			}
+		} catch (final Throwable e) {
+			log.warn("Error obtaining access token", e);
+			throw new CollectorException("Error obtaining access token", e);
+		}
+		throw new CollectorException("Access token is missing");
+
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/researchfi/ResearchFiIterator.java

@@ -0,0 +1,117 @@
+
+package eu.dnetlib.dhp.collection.plugin.researchfi;
+
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.concurrent.PriorityBlockingQueue;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.Header;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.json.JSONArray;
+
+import eu.dnetlib.dhp.collection.plugin.utils.JsonUtils;
+import eu.dnetlib.dhp.common.collection.CollectorException;
+
+public class ResearchFiIterator implements Iterator<String> {
+
+	private static final Log log = LogFactory.getLog(ResearchFiIterator.class);
+
+	private static final int PAGE_SIZE = 100;
+
+	private final String baseUrl;
+	private final String authToken;
+	private int currPage;
+	private int nPages;
+
+	private final Queue<String> queue = new PriorityBlockingQueue<>();
+
+	public ResearchFiIterator(final String baseUrl, final String authToken) {
+		this.baseUrl = baseUrl;
+		this.authToken = authToken;
+		this.currPage = 0;
+		this.nPages = 0;
+	}
+
+	private void verifyStarted() {
+		if (this.currPage == 0) {
+			try {
+				nextCall();
+			} catch (final CollectorException e) {
+				throw new IllegalStateException(e);
+			}
+		}
+	}
+
+	@Override
+	public boolean hasNext() {
+		synchronized (this.queue) {
+			verifyStarted();
+			return !this.queue.isEmpty();
+		}
+	}
+
+	@Override
+	public String next() {
+		synchronized (this.queue) {
+			verifyStarted();
+			final String res = this.queue.poll();
+			while (this.queue.isEmpty() && (this.currPage < this.nPages)) {
+				try {
+					nextCall();
+				} catch (final CollectorException e) {
+					throw new IllegalStateException(e);
+				}
+			}
+			return res;
+		}
+	}
+
+	private void nextCall() throws CollectorException {
+
+		this.currPage += 1;
+
+		final String url;
+		if (!this.baseUrl.contains("?")) {
+			url = String.format("%s?PageNumber=%d&PageSize=%d", this.baseUrl, this.currPage, PAGE_SIZE);
+		} else if (!this.baseUrl.contains("PageSize=")) {
+			url = String.format("%s&PageNumber=%d&PageSize=%d", this.baseUrl, this.currPage, PAGE_SIZE);
+		} else {
+			url = String.format("%s&PageNumber=%d", this.baseUrl, this.currPage);
+		}
+		log.info("Calling url: " + url);
+
+		try (final CloseableHttpClient client = HttpClients.createDefault()) {
+
+			final HttpGet req = new HttpGet(url);
+			req.addHeader("Authorization", "Bearer " + this.authToken);
+			try (final CloseableHttpResponse response = client.execute(req)) {
+				for (final Header header : response.getAllHeaders()) {
+					log.debug("HEADER: " + header.getName() + " = " + header.getValue());
+					if ("x-page-count".equals(header.getName())) {
+						final int totalPages = NumberUtils.toInt(header.getValue());
+						if (this.nPages != totalPages) {
+							this.nPages = NumberUtils.toInt(header.getValue());
+							log.info("Total pages: " + totalPages);
+						}
+					}
+				}
+
+				final String content = IOUtils.toString(response.getEntity().getContent());
+				final JSONArray jsonArray = new JSONArray(content);
+
+				jsonArray.forEach(obj -> this.queue.add(JsonUtils.convertToXML(obj.toString())));
+			}
+		} catch (final Throwable e) {
+			log.warn("Error calling url: " + url, e);
+			throw new CollectorException("Error calling url: " + url, e);
+		}
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java

@@ -1,6 +1,7 @@
 
 package eu.dnetlib.dhp.collection.plugin.rest;
 
+import java.util.Map;
 import java.util.Optional;
 import java.util.Spliterator;
 import java.util.Spliterators;
@@ -9,6 +10,8 @@ import java.util.stream.StreamSupport;
 
 import org.apache.commons.lang3.StringUtils;
 
+import com.google.gson.Gson;
+
 import eu.dnetlib.dhp.collection.ApiDescriptor;
 import eu.dnetlib.dhp.collection.plugin.CollectorPlugin;
 import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
@@ -47,6 +50,9 @@ public class RestCollectorPlugin implements CollectorPlugin {
 		final String entityXpath = api.getParams().get("entityXpath");
 		final String authMethod = api.getParams().get("authMethod");
 		final String authToken = api.getParams().get("authToken");
+		final String requestHeaderMap = api.getParams().get("requestHeaderMap");
+		Gson gson = new Gson();
+		Map requestHeaders = gson.fromJson(requestHeaderMap, Map.class);
 		final String resultSizeValue = Optional
 			.ofNullable(api.getParams().get("resultSizeValue"))
 			.filter(StringUtils::isNotBlank)
@@ -64,9 +70,6 @@ public class RestCollectorPlugin implements CollectorPlugin {
 		if (StringUtils.isBlank(resultFormatValue)) {
 			throw new CollectorException("Param 'resultFormatValue' is null or empty");
 		}
-		if (StringUtils.isBlank(queryParams)) {
-			throw new CollectorException("Param 'queryParams' is null or empty");
-		}
 		if (StringUtils.isBlank(entityXpath)) {
 			throw new CollectorException("Param 'entityXpath' is null or empty");
 		}
@@ -92,7 +95,8 @@ public class RestCollectorPlugin implements CollectorPlugin {
 			entityXpath,
 			authMethod,
 			authToken,
-			resultOutputFormat);
+			resultOutputFormat,
+			requestHeaders);
 
 		return StreamSupport
 			.stream(

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java

@@ -9,8 +9,11 @@ import java.net.URL;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.PriorityBlockingQueue;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Transformer;
@@ -18,22 +21,18 @@ import javax.xml.transform.TransformerConfigurationException;
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
-import javax.xml.xpath.XPath;
+import javax.xml.xpath.*;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpHeaders;
-import org.apache.http.entity.ContentType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.InputSource;
 
+import com.google.common.collect.Maps;
+
 import eu.dnetlib.dhp.collection.plugin.utils.JsonUtils;
 import eu.dnetlib.dhp.common.collection.CollectorException;
 import eu.dnetlib.dhp.common.collection.HttpClientParams;
@@ -48,20 +47,23 @@ import eu.dnetlib.dhp.common.collection.HttpClientParams;
  *
  */
 public class RestIterator implements Iterator<String> {
-
 	private static final Logger log = LoggerFactory.getLogger(RestIterator.class);
 	public static final String UTF_8 = "UTF-8";
 	private static final int MAX_ATTEMPTS = 5;
 
 	private final HttpClientParams clientParams;
 
-	private final String BASIC = "basic";
+	private final String AUTHBASIC = "basic";
+
+	private static final String XML_HEADER = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+	private static final String EMPTY_XML = XML_HEADER + "<" + JsonUtils.XML_WRAP_TAG + "></" + JsonUtils.XML_WRAP_TAG
+		+ ">";
 
 	private final String baseUrl;
 	private final String resumptionType;
 	private final String resumptionParam;
 	private final String resultFormatValue;
-	private String queryParams;
+	private String queryParams = "";
 	private final int resultSizeValue;
 	private int resumptionInt = 0; // integer resumption token (first record to harvest)
 	private int resultTotal = -1;
@@ -89,6 +91,11 @@ public class RestIterator implements Iterator<String> {
 	 */
 	private final String resultOutputFormat;
 
+	/*
+	 * Can be used to set additional request headers, like for content negotiation
+	 */
+	private Map<String, String> requestHeaders;
+
 	/**
 	 * RestIterator class compatible to version 1.3.33
 	 */
@@ -107,7 +114,8 @@ public class RestIterator implements Iterator<String> {
 		final String entityXpath,
 		final String authMethod,
 		final String authToken,
-		final String resultOutputFormat) {
+		final String resultOutputFormat,
+		final Map<String, String> requestHeaders) {
 
 		this.clientParams = clientParams;
 		this.baseUrl = baseUrl;
@@ -119,6 +127,7 @@ public class RestIterator implements Iterator<String> {
 		this.authMethod = authMethod;
 		this.authToken = authToken;
 		this.resultOutputFormat = resultOutputFormat;
+		this.requestHeaders = requestHeaders != null ? requestHeaders : Maps.newHashMap();
 
 		this.queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue
 			: "";
@@ -148,7 +157,12 @@ public class RestIterator implements Iterator<String> {
 	}
 
 	private void initQueue() {
-		this.query = this.baseUrl + "?" + this.queryParams + this.querySize + this.queryFormat;
+		if (queryParams.equals("") && querySize.equals("") && queryFormat.equals("")) {
+			query = baseUrl;
+		} else {
+			query = baseUrl + "?" + queryParams + querySize + queryFormat;
+		}
+
 		log.info("REST calls starting with {}", this.query);
 	}
 
@@ -162,19 +176,6 @@ public class RestIterator implements Iterator<String> {
 	 */
 	@Override
 	public boolean hasNext() {
-		if (this.recordQueue.isEmpty() && this.query.isEmpty()) {
-			disconnect();
-			return false;
-		}
-		return true;
-	}
-
-	/*
-	 * (non-Javadoc)
-	 * @see java.util.Iterator#next()
-	 */
-	@Override
-	public String next() {
 		synchronized (this.recordQueue) {
 			while (this.recordQueue.isEmpty() && !this.query.isEmpty()) {
 				try {
@@ -184,6 +185,23 @@ public class RestIterator implements Iterator<String> {
 					throw new RuntimeException(e);
 				}
 			}
+
+			if (!this.recordQueue.isEmpty()) {
+				return true;
+			}
+
+			disconnect();
+			return false;
+		}
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see java.util.Iterator#next()
+	 */
+	@Override
+	public String next() {
+		synchronized (this.recordQueue) {
 			return this.recordQueue.poll();
 		}
 	}
@@ -209,9 +227,8 @@ public class RestIterator implements Iterator<String> {
 
 		try {
 			String resultJson;
-			String resultXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+			String resultXml = XML_HEADER;
 			String nextQuery = "";
-			final String emptyXml = resultXml + "<" + JsonUtils.XML_WRAP_TAG + "></" + JsonUtils.XML_WRAP_TAG + ">";
 			Node resultNode = null;
 			NodeList nodeList = null;
 			String qUrlArgument = "";
@@ -226,37 +243,48 @@ public class RestIterator implements Iterator<String> {
 				}
 			}
 
+			// find pagination page start number in queryParam and remove before start the first query
+			if ((resumptionType.toLowerCase().equals("pagination") || resumptionType.toLowerCase().equals("page"))
+				&& (query.contains("paginationStart="))) {
+
+				final Matcher m = Pattern.compile("paginationStart=([0-9]+)").matcher(query);
+				m.find(); // guaranteed to be true for this regex
+
+				String[] pageVal = m.group(0).split("=");
+				pagination = Integer.parseInt(pageVal[1]);
+
+				// remove page start number from query and queryParams
+				queryParams = queryParams.replaceFirst("&?paginationStart=[0-9]+", "");
+				query = query.replaceFirst("&?paginationStart=[0-9]+", "");
+
+			}
+
 			try {
 				log.info("requesting URL [{}]", query);
 
 				final URL qUrl = new URL(query);
 				log.debug("authMethod: {}", this.authMethod);
-				if ("bearer".equalsIgnoreCase(this.authMethod)) {
+				if (this.authMethod == "bearer") {
-					log.trace("authMethod before inputStream: {}", resultXml);
+					log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);
-					final HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection();
+					requestHeaders.put("Authorization", "Bearer " + authToken);
-					conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + this.authToken);
+					// requestHeaders.put("Content-Type", "application/json");
-					conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType());
+				} else if (AUTHBASIC.equalsIgnoreCase(this.authMethod)) {
-					conn.setRequestMethod("GET");
+					log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);
-					theHttpInputStream = conn.getInputStream();
+					requestHeaders.put("Authorization", "Basic " + authToken);
-				} else if (this.BASIC.equalsIgnoreCase(this.authMethod)) {
+					// requestHeaders.put("accept", "application/xml");
-					log.trace("authMethod before inputStream: {}", resultXml);
-					final HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection();
-					conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Basic " + this.authToken);
-					conn.setRequestProperty(HttpHeaders.ACCEPT, ContentType.APPLICATION_XML.getMimeType());
-					conn.setRequestMethod("GET");
-					theHttpInputStream = conn.getInputStream();
-				} else {
-					theHttpInputStream = qUrl.openStream();
 				}
+				HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection();
+				conn.setRequestMethod("GET");
+				this.setRequestHeader(conn);
+				resultStream = conn.getInputStream();
 
-				this.resultStream = theHttpInputStream;
 				if ("json".equals(this.resultOutputFormat)) {
 					resultJson = IOUtils.toString(this.resultStream, StandardCharsets.UTF_8);
 					resultXml = JsonUtils.convertToXML(resultJson);
 					this.resultStream = IOUtils.toInputStream(resultXml, UTF_8);
 				}
 
-				if (!(emptyXml).equalsIgnoreCase(resultXml)) {
+				if (!isEmptyXml(resultXml)) {
 					resultNode = (Node) this.xpath
 						.evaluate("/", new InputSource(this.resultStream), XPathConstants.NODE);
 					nodeList = (NodeList) this.xprEntity.evaluate(resultNode, XPathConstants.NODESET);
@@ -265,8 +293,7 @@ public class RestIterator implements Iterator<String> {
 						final StringWriter sw = new StringWriter();
 						this.transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw));
 						final String toEnqueue = sw.toString();
-						if ((toEnqueue == null) || StringUtils.isBlank(toEnqueue)
+						if ((toEnqueue == null) || StringUtils.isBlank(toEnqueue) || isEmptyXml(toEnqueue)) {
-							|| emptyXml.equalsIgnoreCase(toEnqueue)) {
 							log
 								.warn(
 									"The following record resulted in empty item for the feeding queue: {}", resultXml);
@@ -294,6 +321,7 @@ public class RestIterator implements Iterator<String> {
 							throw new CollectorException("Mode: discover, Param 'resultSizeValue' is less than 2");
 						}
 						qUrlArgument = qUrl.getQuery();
+
 						final String[] arrayQUrlArgument = qUrlArgument.split("&");
 						for (final String arrayUrlArgStr : arrayQUrlArgument) {
 							if (arrayUrlArgStr.startsWith(this.resumptionParam)) {
@@ -307,7 +335,7 @@ public class RestIterator implements Iterator<String> {
 							}
 						}
 
-						if (((emptyXml).equalsIgnoreCase(resultXml))
+						if (isEmptyXml(resultXml)
 							|| ((nodeList != null) && (nodeList.getLength() < this.resultSizeValue))) {
 							// resumptionStr = "";
 							if (nodeList != null) {
@@ -326,13 +354,13 @@ public class RestIterator implements Iterator<String> {
 
 					case "pagination":
 					case "page": // pagination, iterate over page numbers
-						this.pagination += 1;
+						if (nodeList != null && nodeList.getLength() > 0) {
-						if (nodeList != null) {
 							this.discoverResultSize += nodeList.getLength();
 						} else {
 							this.resultTotal = this.discoverResultSize;
 							this.pagination = this.discoverResultSize;
 						}
+						this.pagination += 1;
 						this.resumptionInt = this.pagination;
 						this.resumptionStr = Integer.toString(this.resumptionInt);
 						break;
@@ -380,7 +408,8 @@ public class RestIterator implements Iterator<String> {
 			try {
 				if (this.resultTotal == -1) {
 					this.resultTotal = Integer.parseInt(this.xprResultTotalPath.evaluate(resultNode));
-					if ("page".equalsIgnoreCase(this.resumptionType) && !this.BASIC.equalsIgnoreCase(this.authMethod)) {
+					if ("page".equalsIgnoreCase(this.resumptionType)
+						&& !this.AUTHBASIC.equalsIgnoreCase(this.authMethod)) {
 						this.resultTotal += 1;
 					} // to correct the upper bound
 					log.info("resultTotal was -1 is now: " + this.resultTotal);
@@ -409,6 +438,10 @@ public class RestIterator implements Iterator<String> {
 
 	}
 
+	private boolean isEmptyXml(String s) {
+		return EMPTY_XML.equalsIgnoreCase(s);
+	}
+
 	private boolean isInteger(final String s) {
 		boolean isValidInteger = false;
 		try {
@@ -433,6 +466,22 @@ public class RestIterator implements Iterator<String> {
 		}
 	}
 
+	/**
+	 * setRequestHeader
+	 *
+	 * setRequestProperty: Sets the general request property. If a property with the key already exists, overwrite its value with the new value.
+	 * @param conn
+	 */
+	private void setRequestHeader(HttpURLConnection conn) {
+		if (requestHeaders != null) {
+			for (String key : requestHeaders.keySet()) {
+				conn.setRequestProperty(key, requestHeaders.get(key));
+			}
+			log.debug("Set Request Header with: " + requestHeaders);
+		}
+
+	}
+
 	public String getResultFormatValue() {
 		return this.resultFormatValue;
 	}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java

@@ -8,7 +8,10 @@ import java.io.StringWriter;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CodingErrorAction;
+import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
 
 import javax.xml.stream.XMLEventFactory;
 import javax.xml.stream.XMLEventReader;
@@ -19,6 +22,7 @@ import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.events.StartElement;
 import javax.xml.stream.events.XMLEvent;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -58,13 +62,23 @@ public class XMLIterator implements Iterator<String> {
 
 	private String element;
 
+	private List<String> elements;
+
 	private InputStream inputStream;
 
 	public XMLIterator(final String element, final InputStream inputStream) {
 		super();
 		this.element = element;
+		if (element.contains(",")) {
+			elements = Arrays
+				.stream(element.split(","))
+				.filter(StringUtils::isNoneBlank)
+				.map(String::toLowerCase)
+				.collect(Collectors.toList());
+		}
 		this.inputStream = inputStream;
 		this.parser = getParser();
+
 		try {
 			this.current = findElement(parser);
 		} catch (XMLStreamException e) {
@@ -113,7 +127,7 @@ public class XMLIterator implements Iterator<String> {
 				final XMLEvent event = parser.nextEvent();
 
 				// TODO: replace with depth tracking instead of close tag tracking.
-				if (event.isEndElement() && event.asEndElement().getName().getLocalPart().equals(element)) {
+				if (event.isEndElement() && isCheckTag(event.asEndElement().getName().getLocalPart())) {
 					writer.add(event);
 					break;
 				}
@@ -142,18 +156,16 @@ public class XMLIterator implements Iterator<String> {
 		XMLEvent peek = parser.peek();
 		if (peek != null && peek.isStartElement()) {
 			String name = peek.asStartElement().getName().getLocalPart();
-			if (element.equals(name)) {
+			if (isCheckTag(name))
 				return peek;
-			}
 		}
 
 		while (parser.hasNext()) {
-			final XMLEvent event = parser.nextEvent();
+			XMLEvent event = parser.nextEvent();
 			if (event != null && event.isStartElement()) {
 				String name = event.asStartElement().getName().getLocalPart();
-				if (element.equals(name)) {
+				if (isCheckTag(name))
 					return event;
-				}
 			}
 		}
 		return null;
@@ -161,12 +173,31 @@ public class XMLIterator implements Iterator<String> {
 
 	private XMLEventReader getParser() {
 		try {
-			return inputFactory.get().createXMLEventReader(sanitize(inputStream));
+			XMLInputFactory xif = inputFactory.get();
+			xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+			return xif.createXMLEventReader(sanitize(inputStream));
 		} catch (XMLStreamException e) {
 			throw new RuntimeException(e);
 		}
 	}
 
+	private boolean isCheckTag(final String tagName) {
+		if (elements != null) {
+			final String found = elements
+				.stream()
+				.filter(e -> e.equalsIgnoreCase(tagName))
+				.findFirst()
+				.orElse(null);
+			if (found != null)
+				return true;
+		} else {
+			if (element.equalsIgnoreCase(tagName)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
 	private Reader sanitize(final InputStream in) {
 		final CharsetDecoder charsetDecoder = Charset.forName(UTF_8).newDecoder();
 		charsetDecoder.onMalformedInput(CodingErrorAction.REPLACE);