Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-hadoop

Reviewed-on: #426

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java

@@ -1,53 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import okhttp3.MediaType;
-import okhttp3.RequestBody;
-import okhttp3.internal.Util;
-import okio.BufferedSink;
-import okio.Okio;
-import okio.Source;
-
-public class InputStreamRequestBody extends RequestBody {
-
-	private final InputStream inputStream;
-	private final MediaType mediaType;
-	private final long lenght;
-
-	public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
-
-		return new InputStreamRequestBody(inputStream, mediaType, len);
-	}
-
-	private InputStreamRequestBody(InputStream inputStream, MediaType mediaType, long len) {
-		this.inputStream = inputStream;
-		this.mediaType = mediaType;
-		this.lenght = len;
-	}
-
-	@Override
-	public MediaType contentType() {
-		return mediaType;
-	}
-
-	@Override
-	public long contentLength() {
-
-		return lenght;
-
-	}
-
-	@Override
-	public void writeTo(BufferedSink sink) throws IOException {
-		Source source = null;
-		try {
-			source = Okio.source(inputStream);
-			sink.writeAll(source);
-		} finally {
-			Util.closeQuietly(source);
-		}
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java

@@ -1,8 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-public class MissingConceptDoiException extends Throwable {
-	public MissingConceptDoiException(String message) {
-		super(message);
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java

@@ -1,365 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.*;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.http.HttpHeaders;
-import org.apache.http.entity.ContentType;
-import org.jetbrains.annotations.NotNull;
-
-import com.google.gson.Gson;
-
-import eu.dnetlib.dhp.common.api.zenodo.ZenodoModel;
-import eu.dnetlib.dhp.common.api.zenodo.ZenodoModelList;
-import okhttp3.*;
-
-public class ZenodoAPIClient implements Serializable {
-
-	String urlString;
-	String bucket;
-
-	String deposition_id;
-	String access_token;
-
-	public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
-
-	private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip");
-
-	public String getUrlString() {
-		return urlString;
-	}
-
-	public void setUrlString(String urlString) {
-		this.urlString = urlString;
-	}
-
-	public String getBucket() {
-		return bucket;
-	}
-
-	public void setBucket(String bucket) {
-		this.bucket = bucket;
-	}
-
-	public void setDeposition_id(String deposition_id) {
-		this.deposition_id = deposition_id;
-	}
-
-	public ZenodoAPIClient(String urlString, String access_token) {
-
-		this.urlString = urlString;
-		this.access_token = access_token;
-	}
-
-	/**
-	 * Brand new deposition in Zenodo. It sets the deposition_id and the bucket where to store the files to upload
-	 *
-	 * @return response code
-	 * @throws IOException
-	 */
-	public int newDeposition() throws IOException {
-		String json = "{}";
-
-		URL url = new URL(urlString);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setRequestMethod("POST");
-		conn.setDoOutput(true);
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = json.getBytes("utf-8");
-			os.write(input, 0, input.length);
-		}
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-		conn.disconnect();
-
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel newSubmission = new Gson().fromJson(body, ZenodoModel.class);
-		this.bucket = newSubmission.getLinks().getBucket();
-		this.deposition_id = newSubmission.getId();
-
-		return responseCode;
-	}
-
-	/**
-	 * Upload files in Zenodo.
-	 *
-	 * @param is the inputStream for the file to upload
-	 * @param file_name the name of the file as it will appear on Zenodo
-	 * @return the response code
-	 */
-	public int uploadIS(InputStream is, String file_name) throws IOException {
-
-		URL url = new URL(bucket + "/" + file_name);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, "application/zip");
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("PUT");
-
-		byte[] buf = new byte[8192];
-		int length;
-		try (OutputStream os = conn.getOutputStream()) {
-			while ((length = is.read(buf)) != -1) {
-				os.write(buf, 0, length);
-			}
-
-		}
-		int responseCode = conn.getResponseCode();
-		if (!checkOKStatus(responseCode)) {
-			throw new IOException("Unexpected code " + responseCode + getBody(conn));
-		}
-
-		return responseCode;
-	}
-
-	@NotNull
-	private String getBody(HttpURLConnection conn) throws IOException {
-		String body = "{}";
-		try (BufferedReader br = new BufferedReader(
-			new InputStreamReader(conn.getInputStream(), "utf-8"))) {
-			StringBuilder response = new StringBuilder();
-			String responseLine = null;
-			while ((responseLine = br.readLine()) != null) {
-				response.append(responseLine.trim());
-			}
-
-			body = response.toString();
-
-		}
-		return body;
-	}
-
-	/**
-	 * Associates metadata information to the current deposition
-	 *
-	 * @param metadata the metadata
-	 * @return response code
-	 * @throws IOException
-	 */
-	public int sendMretadata(String metadata) throws IOException {
-
-		URL url = new URL(urlString + "/" + deposition_id);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("PUT");
-
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = metadata.getBytes("utf-8");
-			os.write(input, 0, input.length);
-
-		}
-
-		final int responseCode = conn.getResponseCode();
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + getBody(conn));
-
-		return responseCode;
-
-	}
-
-	private boolean checkOKStatus(int responseCode) {
-
-		if (HttpURLConnection.HTTP_OK != responseCode ||
-			HttpURLConnection.HTTP_CREATED != responseCode)
-			return true;
-		return false;
-	}
-
-	/**
-	 * To publish the current deposition. It works for both new deposition or new version of an old deposition
-	 *
-	 * @return response code
-	 * @throws IOException
-	 */
-	@Deprecated
-	public int publish() throws IOException {
-
-		String json = "{}";
-
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
-
-		Request request = new Request.Builder()
-			.url(urlString + "/" + deposition_id + "/actions/publish")
-			.addHeader("Authorization", "Bearer " + access_token)
-			.post(body)
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			return response.code();
-
-		}
-	}
-
-	/**
-	 * To create a new version of an already published deposition. It sets the deposition_id and the bucket to be used
-	 * for the new version.
-	 *
-	 * @param concept_rec_id the concept record id of the deposition for which to create a new version. It is the last
-	 *            part of the url for the DOI Zenodo suggests to use to cite all versions: DOI: 10.xxx/zenodo.656930
-	 *            concept_rec_id = 656930
-	 * @return response code
-	 * @throws IOException
-	 * @throws MissingConceptDoiException
-	 */
-	public int newVersion(String concept_rec_id) throws IOException, MissingConceptDoiException {
-		setDepositionId(concept_rec_id, 1);
-		String json = "{}";
-
-		URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion");
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("POST");
-
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = json.getBytes("utf-8");
-			os.write(input, 0, input.length);
-
-		}
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
-		String latest_draft = zenodoModel.getLinks().getLatest_draft();
-		deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
-		bucket = getBucket(latest_draft);
-
-		return responseCode;
-
-	}
-
-	/**
-	 * To finish uploading a version or new deposition not published
-	 * It sets the deposition_id and the bucket to be used
-	 *
-	 *
-	 * @param deposition_id the deposition id of the not yet published upload
-	 *            concept_rec_id = 656930
-	 * @return response code
-	 * @throws IOException
-	 * @throws MissingConceptDoiException
-	 */
-	public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
-
-		this.deposition_id = deposition_id;
-
-		String json = "{}";
-
-		URL url = new URL(urlString + "/" + deposition_id);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setRequestMethod("POST");
-		conn.setDoOutput(true);
-		try (OutputStream os = conn.getOutputStream()) {
-			byte[] input = json.getBytes("utf-8");
-			os.write(input, 0, input.length);
-		}
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-		conn.disconnect();
-
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
-		bucket = zenodoModel.getLinks().getBucket();
-
-		return responseCode;
-
-	}
-
-	private void setDepositionId(String concept_rec_id, Integer page) throws IOException, MissingConceptDoiException {
-
-		ZenodoModelList zenodoModelList = new Gson()
-			.fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class);
-
-		for (ZenodoModel zm : zenodoModelList) {
-			if (zm.getConceptrecid().equals(concept_rec_id)) {
-				deposition_id = zm.getId();
-				return;
-			}
-		}
-		if (zenodoModelList.size() == 0)
-			throw new MissingConceptDoiException(
-				"The concept record id specified was missing in the list of depositions");
-		setDepositionId(concept_rec_id, page + 1);
-
-	}
-
-	private String getPrevDepositions(String page) throws IOException {
-
-		HttpUrl.Builder urlBuilder = HttpUrl.parse(urlString).newBuilder();
-		urlBuilder.addQueryParameter("page", page);
-
-		URL url = new URL(urlBuilder.build().toString());
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("GET");
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		return body;
-
-	}
-
-	private String getBucket(String inputUurl) throws IOException {
-
-		URL url = new URL(inputUurl);
-		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-		conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString());
-		conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token);
-		conn.setDoOutput(true);
-		conn.setRequestMethod("GET");
-
-		String body = getBody(conn);
-
-		int responseCode = conn.getResponseCode();
-
-		conn.disconnect();
-		if (!checkOKStatus(responseCode))
-			throw new IOException("Unexpected code " + responseCode + body);
-
-		ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class);
-
-		return zenodoModel.getLinks().getBucket();
-
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java

@@ -1,14 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-public class Community {
-	private String identifier;
-
-	public String getIdentifier() {
-		return identifier;
-	}
-
-	public void setIdentifier(String identifier) {
-		this.identifier = identifier;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java

@@ -1,47 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-public class Creator {
-	private String affiliation;
-	private String name;
-	private String orcid;
-
-	public String getAffiliation() {
-		return affiliation;
-	}
-
-	public void setAffiliation(String affiliation) {
-		this.affiliation = affiliation;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getOrcid() {
-		return orcid;
-	}
-
-	public void setOrcid(String orcid) {
-		this.orcid = orcid;
-	}
-
-	public static Creator newInstance(String name, String affiliation, String orcid) {
-		Creator c = new Creator();
-		if (name != null) {
-			c.name = name;
-		}
-		if (affiliation != null) {
-			c.affiliation = affiliation;
-		}
-		if (orcid != null) {
-			c.orcid = orcid;
-		}
-
-		return c;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java

@@ -1,44 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class File implements Serializable {
-	private String checksum;
-	private String filename;
-	private long filesize;
-	private String id;
-
-	public String getChecksum() {
-		return checksum;
-	}
-
-	public void setChecksum(String checksum) {
-		this.checksum = checksum;
-	}
-
-	public String getFilename() {
-		return filename;
-	}
-
-	public void setFilename(String filename) {
-		this.filename = filename;
-	}
-
-	public long getFilesize() {
-		return filesize;
-	}
-
-	public void setFilesize(long filesize) {
-		this.filesize = filesize;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java

@@ -1,23 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class Grant implements Serializable {
-	private String id;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public static Grant newInstance(String id) {
-		Grant g = new Grant();
-		g.id = id;
-
-		return g;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java

@@ -1,92 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class Links implements Serializable {
-
-	private String bucket;
-
-	private String discard;
-
-	private String edit;
-	private String files;
-	private String html;
-	private String latest_draft;
-	private String latest_draft_html;
-	private String publish;
-
-	private String self;
-
-	public String getBucket() {
-		return bucket;
-	}
-
-	public void setBucket(String bucket) {
-		this.bucket = bucket;
-	}
-
-	public String getDiscard() {
-		return discard;
-	}
-
-	public void setDiscard(String discard) {
-		this.discard = discard;
-	}
-
-	public String getEdit() {
-		return edit;
-	}
-
-	public void setEdit(String edit) {
-		this.edit = edit;
-	}
-
-	public String getFiles() {
-		return files;
-	}
-
-	public void setFiles(String files) {
-		this.files = files;
-	}
-
-	public String getHtml() {
-		return html;
-	}
-
-	public void setHtml(String html) {
-		this.html = html;
-	}
-
-	public String getLatest_draft() {
-		return latest_draft;
-	}
-
-	public void setLatest_draft(String latest_draft) {
-		this.latest_draft = latest_draft;
-	}
-
-	public String getLatest_draft_html() {
-		return latest_draft_html;
-	}
-
-	public void setLatest_draft_html(String latest_draft_html) {
-		this.latest_draft_html = latest_draft_html;
-	}
-
-	public String getPublish() {
-		return publish;
-	}
-
-	public void setPublish(String publish) {
-		this.publish = publish;
-	}
-
-	public String getSelf() {
-		return self;
-	}
-
-	public void setSelf(String self) {
-		this.self = self;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java

@@ -1,153 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Metadata implements Serializable {
-
-	private String access_right;
-	private List<Community> communities;
-	private List<Creator> creators;
-	private String description;
-	private String doi;
-	private List<Grant> grants;
-	private List<String> keywords;
-	private String language;
-	private String license;
-	private PrereserveDoi prereserve_doi;
-	private String publication_date;
-	private List<String> references;
-	private List<RelatedIdentifier> related_identifiers;
-	private String title;
-	private String upload_type;
-	private String version;
-
-	public String getUpload_type() {
-		return upload_type;
-	}
-
-	public void setUpload_type(String upload_type) {
-		this.upload_type = upload_type;
-	}
-
-	public String getVersion() {
-		return version;
-	}
-
-	public void setVersion(String version) {
-		this.version = version;
-	}
-
-	public String getAccess_right() {
-		return access_right;
-	}
-
-	public void setAccess_right(String access_right) {
-		this.access_right = access_right;
-	}
-
-	public List<Community> getCommunities() {
-		return communities;
-	}
-
-	public void setCommunities(List<Community> communities) {
-		this.communities = communities;
-	}
-
-	public List<Creator> getCreators() {
-		return creators;
-	}
-
-	public void setCreators(List<Creator> creators) {
-		this.creators = creators;
-	}
-
-	public String getDescription() {
-		return description;
-	}
-
-	public void setDescription(String description) {
-		this.description = description;
-	}
-
-	public String getDoi() {
-		return doi;
-	}
-
-	public void setDoi(String doi) {
-		this.doi = doi;
-	}
-
-	public List<Grant> getGrants() {
-		return grants;
-	}
-
-	public void setGrants(List<Grant> grants) {
-		this.grants = grants;
-	}
-
-	public List<String> getKeywords() {
-		return keywords;
-	}
-
-	public void setKeywords(List<String> keywords) {
-		this.keywords = keywords;
-	}
-
-	public String getLanguage() {
-		return language;
-	}
-
-	public void setLanguage(String language) {
-		this.language = language;
-	}
-
-	public String getLicense() {
-		return license;
-	}
-
-	public void setLicense(String license) {
-		this.license = license;
-	}
-
-	public PrereserveDoi getPrereserve_doi() {
-		return prereserve_doi;
-	}
-
-	public void setPrereserve_doi(PrereserveDoi prereserve_doi) {
-		this.prereserve_doi = prereserve_doi;
-	}
-
-	public String getPublication_date() {
-		return publication_date;
-	}
-
-	public void setPublication_date(String publication_date) {
-		this.publication_date = publication_date;
-	}
-
-	public List<String> getReferences() {
-		return references;
-	}
-
-	public void setReferences(List<String> references) {
-		this.references = references;
-	}
-
-	public List<RelatedIdentifier> getRelated_identifiers() {
-		return related_identifiers;
-	}
-
-	public void setRelated_identifiers(List<RelatedIdentifier> related_identifiers) {
-		this.related_identifiers = related_identifiers;
-	}
-
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java

@@ -1,25 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class PrereserveDoi implements Serializable {
-	private String doi;
-	private String recid;
-
-	public String getDoi() {
-		return doi;
-	}
-
-	public void setDoi(String doi) {
-		this.doi = doi;
-	}
-
-	public String getRecid() {
-		return recid;
-	}
-
-	public void setRecid(String recid) {
-		this.recid = recid;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java

@@ -1,43 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class RelatedIdentifier implements Serializable {
-	private String identifier;
-	private String relation;
-	private String resource_type;
-	private String scheme;
-
-	public String getIdentifier() {
-		return identifier;
-	}
-
-	public void setIdentifier(String identifier) {
-		this.identifier = identifier;
-	}
-
-	public String getRelation() {
-		return relation;
-	}
-
-	public void setRelation(String relation) {
-		this.relation = relation;
-	}
-
-	public String getResource_type() {
-		return resource_type;
-	}
-
-	public void setResource_type(String resource_type) {
-		this.resource_type = resource_type;
-	}
-
-	public String getScheme() {
-		return scheme;
-	}
-
-	public void setScheme(String scheme) {
-		this.scheme = scheme;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java

@@ -1,118 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class ZenodoModel implements Serializable {
-
-	private String conceptrecid;
-	private String created;
-
-	private List<File> files;
-	private String id;
-	private Links links;
-	private Metadata metadata;
-	private String modified;
-	private String owner;
-	private String record_id;
-	private String state;
-	private boolean submitted;
-	private String title;
-
-	public String getConceptrecid() {
-		return conceptrecid;
-	}
-
-	public void setConceptrecid(String conceptrecid) {
-		this.conceptrecid = conceptrecid;
-	}
-
-	public String getCreated() {
-		return created;
-	}
-
-	public void setCreated(String created) {
-		this.created = created;
-	}
-
-	public List<File> getFiles() {
-		return files;
-	}
-
-	public void setFiles(List<File> files) {
-		this.files = files;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public Links getLinks() {
-		return links;
-	}
-
-	public void setLinks(Links links) {
-		this.links = links;
-	}
-
-	public Metadata getMetadata() {
-		return metadata;
-	}
-
-	public void setMetadata(Metadata metadata) {
-		this.metadata = metadata;
-	}
-
-	public String getModified() {
-		return modified;
-	}
-
-	public void setModified(String modified) {
-		this.modified = modified;
-	}
-
-	public String getOwner() {
-		return owner;
-	}
-
-	public void setOwner(String owner) {
-		this.owner = owner;
-	}
-
-	public String getRecord_id() {
-		return record_id;
-	}
-
-	public void setRecord_id(String record_id) {
-		this.record_id = record_id;
-	}
-
-	public String getState() {
-		return state;
-	}
-
-	public void setState(String state) {
-		this.state = state;
-	}
-
-	public boolean isSubmitted() {
-		return submitted;
-	}
-
-	public void setSubmitted(boolean submitted) {
-		this.submitted = submitted;
-	}
-
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java

@@ -1,7 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.util.ArrayList;
-
-public class ZenodoModelList extends ArrayList<ZenodoModel> {
-}

dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java

@@ -145,6 +145,105 @@ public class AuthorMerger {
 		return null;
 	}
 
+	/**
+	 * This method tries to figure out when two author are the same in the contest
+	 * of ORCID enrichment
+	 *
+	 * @param left  Author in the OAF entity
+	 * @param right Author ORCID
+	 * @return based on a heuristic on the names of the authors if they are the same.
+	 */
+	public static boolean checkORCIDSimilarity(final Author left, final Author right) {
+		final Person pl = parse(left);
+		final Person pr = parse(right);
+
+		// If one of them didn't have a surname we verify if they have the fullName not empty
+		// and verify if the normalized version is equal
+		if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) &&
+			pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) {
+
+			if (pl.getFullname() != null && !pl.getFullname().isEmpty() && pr.getFullname() != null
+				&& !pr.getFullname().isEmpty()) {
+				return pl
+					.getFullname()
+					.stream()
+					.anyMatch(
+						fl -> pr.getFullname().stream().anyMatch(fr -> normalize(fl).equalsIgnoreCase(normalize(fr))));
+			} else {
+				return false;
+			}
+		}
+		// The Authors have one surname in common
+		if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) {
+
+			// If one of them has only a surname and is the same we can say that they are the same author
+			if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) ||
+				(pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank)))
+				return true;
+			// The authors have the same initials of Name in common
+			if (pl
+				.getName()
+				.stream()
+				.anyMatch(
+					nl -> pr
+						.getName()
+						.stream()
+						.anyMatch(nr -> nr.equalsIgnoreCase(nl))))
+				return true;
+		}
+
+		// Sometimes we noticed that publication have author wrote in inverse order Surname, Name
+		// We verify if we have an exact match between name and surname
+		if (pl.getSurname().stream().anyMatch(sl -> pr.getName().stream().anyMatch(nr -> nr.equalsIgnoreCase(sl))) &&
+			pl.getName().stream().anyMatch(nl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(nl))))
+			return true;
+		else
+			return false;
+	}
+	//
+
+	/**
+	 * Method to enrich ORCID information in one list of authors based on another list
+	 *
+	 * @param baseAuthor  the Author List in the OAF Entity
+	 * @param orcidAuthor The list of ORCID Author intersected
+	 * @return The Author List of the OAF Entity enriched with the orcid Author
+	 */
+	public static List<Author> enrichOrcid(List<Author> baseAuthor, List<Author> orcidAuthor) {
+
+		if (baseAuthor == null || baseAuthor.isEmpty())
+			return orcidAuthor;
+
+		if (orcidAuthor == null || orcidAuthor.isEmpty())
+			return baseAuthor;
+
+		if (baseAuthor.size() == 1 && orcidAuthor.size() > 10)
+			return baseAuthor;
+
+		final List<Author> oAuthor = new ArrayList<>();
+		oAuthor.addAll(orcidAuthor);
+
+		baseAuthor.forEach(ba -> {
+			Optional<Author> aMatch = oAuthor.stream().filter(oa -> checkORCIDSimilarity(ba, oa)).findFirst();
+			if (aMatch.isPresent()) {
+				final Author sameAuthor = aMatch.get();
+				addPid(ba, sameAuthor.getPid());
+				oAuthor.remove(sameAuthor);
+			}
+		});
+		return baseAuthor;
+	}
+
+	private static void addPid(final Author a, final List<StructuredProperty> pids) {
+
+		if (a.getPid() == null) {
+			a.setPid(new ArrayList<>());
+		}
+
+		a.getPid().addAll(pids);
+
+	}
+
 	public static String pidToComparableString(StructuredProperty pid) {
 		final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase()
 			: "";

dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java

@@ -1,6 +1,24 @@
 
 package eu.dnetlib.dhp.oa.merge;
 
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+import static org.apache.spark.sql.functions.col;
+import static org.apache.spark.sql.functions.when;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.api.java.function.ReduceFunction;
+import org.apache.spark.sql.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
@@ -8,186 +26,169 @@ import eu.dnetlib.dhp.schema.common.EntityType;
 import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.schema.oaf.OafEntity;
 import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions;
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
+import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
 import eu.dnetlib.dhp.utils.ISLookupClientFactory;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
-import org.apache.commons.io.IOUtils;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.function.MapFunction;
-import org.apache.spark.api.java.function.ReduceFunction;
-import org.apache.spark.sql.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import scala.Tuple2;
 
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ForkJoinPool;
-import java.util.stream.Collectors;
-
-import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
-import static org.apache.spark.sql.functions.col;
-import static org.apache.spark.sql.functions.when;
-
 /**
  * Groups the graph content by entity identifier to ensure ID uniqueness
  */
 public class GroupEntitiesSparkJob {
-    private static final Logger log = LoggerFactory.getLogger(GroupEntitiesSparkJob.class);
+	private static final Logger log = LoggerFactory.getLogger(GroupEntitiesSparkJob.class);
 
-    private static final Encoder<OafEntity> OAFENTITY_KRYO_ENC = Encoders.kryo(OafEntity.class);
+	private static final Encoder<OafEntity> OAFENTITY_KRYO_ENC = Encoders.kryo(OafEntity.class);
 
-    private ArgumentApplicationParser parser;
+	private ArgumentApplicationParser parser;
 
-    public GroupEntitiesSparkJob(ArgumentApplicationParser parser) {
-        this.parser = parser;
-    }
+	public GroupEntitiesSparkJob(ArgumentApplicationParser parser) {
+		this.parser = parser;
+	}
 
-    public static void main(String[] args) throws Exception {
+	public static void main(String[] args) throws Exception {
 
-        String jsonConfiguration = IOUtils
-                .toString(
-                        GroupEntitiesSparkJob.class
-                                .getResourceAsStream(
-                                        "/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json"));
-        final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
-        parser.parseArgument(args);
+		String jsonConfiguration = IOUtils
+			.toString(
+				GroupEntitiesSparkJob.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json"));
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+		parser.parseArgument(args);
 
-        Boolean isSparkSessionManaged = Optional
-                .ofNullable(parser.get("isSparkSessionManaged"))
-                .map(Boolean::valueOf)
-                .orElse(Boolean.TRUE);
-        log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
 
-        final String isLookupUrl = parser.get("isLookupUrl");
-        log.info("isLookupUrl: {}", isLookupUrl);
+		final String isLookupUrl = parser.get("isLookupUrl");
+		log.info("isLookupUrl: {}", isLookupUrl);
 
-        final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl);
+		final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl);
 
-        new GroupEntitiesSparkJob(parser).run(isSparkSessionManaged, isLookupService);
-    }
+		new GroupEntitiesSparkJob(parser).run(isSparkSessionManaged, isLookupService);
+	}
 
-    public void run(Boolean isSparkSessionManaged, ISLookUpService isLookUpService)
-            throws ISLookUpException {
+	public void run(Boolean isSparkSessionManaged, ISLookUpService isLookUpService)
+		throws ISLookUpException {
 
-        String graphInputPath = parser.get("graphInputPath");
-        log.info("graphInputPath: {}", graphInputPath);
+		String graphInputPath = parser.get("graphInputPath");
+		log.info("graphInputPath: {}", graphInputPath);
 
-        String checkpointPath = parser.get("checkpointPath");
-        log.info("checkpointPath: {}", checkpointPath);
+		String checkpointPath = parser.get("checkpointPath");
+		log.info("checkpointPath: {}", checkpointPath);
 
-        String outputPath = parser.get("outputPath");
-        log.info("outputPath: {}", outputPath);
+		String outputPath = parser.get("outputPath");
+		log.info("outputPath: {}", outputPath);
 
-        boolean filterInvisible = Boolean.parseBoolean(parser.get("filterInvisible"));
-        log.info("filterInvisible: {}", filterInvisible);
+		boolean filterInvisible = Boolean.parseBoolean(parser.get("filterInvisible"));
+		log.info("filterInvisible: {}", filterInvisible);
 
-        SparkConf conf = new SparkConf();
-        conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
-        conf.registerKryoClasses(ModelSupport.getOafModelClasses());
+		SparkConf conf = new SparkConf();
+		conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+		conf.registerKryoClasses(ModelSupport.getOafModelClasses());
 
-        final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookUpService);
+		final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookUpService);
 
-        runWithSparkSession(
-                conf,
-                isSparkSessionManaged,
-                spark -> {
-                    HdfsSupport.remove(checkpointPath, spark.sparkContext().hadoopConfiguration());
-                    groupEntities(spark, graphInputPath, checkpointPath, outputPath, filterInvisible, vocs);
-                });
-    }
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				HdfsSupport.remove(checkpointPath, spark.sparkContext().hadoopConfiguration());
+				groupEntities(spark, graphInputPath, checkpointPath, outputPath, filterInvisible, vocs);
+			});
+	}
 
-    private static void groupEntities(
-            SparkSession spark,
-            String inputPath,
-            String checkpointPath,
-            String outputPath,
-            boolean filterInvisible, VocabularyGroup vocs) {
+	private static void groupEntities(
+		SparkSession spark,
+		String inputPath,
+		String checkpointPath,
+		String outputPath,
+		boolean filterInvisible, VocabularyGroup vocs) {
 
-        Dataset<OafEntity> allEntities = spark.emptyDataset(OAFENTITY_KRYO_ENC);
+		Dataset<OafEntity> allEntities = spark.emptyDataset(OAFENTITY_KRYO_ENC);
 
-        for (Map.Entry<EntityType, Class> e : ModelSupport.entityTypes.entrySet()) {
-            String entity = e.getKey().name();
-            Class<? extends OafEntity> entityClass = e.getValue();
-            String entityInputPath = inputPath + "/" + entity;
+		for (Map.Entry<EntityType, Class> e : ModelSupport.entityTypes.entrySet()) {
+			String entity = e.getKey().name();
+			Class<? extends OafEntity> entityClass = e.getValue();
+			String entityInputPath = inputPath + "/" + entity;
 
-            if (!HdfsSupport.exists(entityInputPath, spark.sparkContext().hadoopConfiguration())) {
-                continue;
-            }
+			if (!HdfsSupport.exists(entityInputPath, spark.sparkContext().hadoopConfiguration())) {
+				continue;
+			}
 
-            allEntities = allEntities
-                    .union(
-                            ((Dataset<OafEntity>) spark
-                                    .read()
-                                    .schema(Encoders.bean(entityClass).schema())
-                                    .json(entityInputPath)
-                                    .filter("length(id) > 0")
-                                    .as(Encoders.bean(entityClass)))
-                                    .map((MapFunction<OafEntity, OafEntity>) r -> r, OAFENTITY_KRYO_ENC));
-        }
+			allEntities = allEntities
+				.union(
+					((Dataset<OafEntity>) spark
+						.read()
+						.schema(Encoders.bean(entityClass).schema())
+						.json(entityInputPath)
+						.filter("length(id) > 0")
+						.as(Encoders.bean(entityClass)))
+							.map((MapFunction<OafEntity, OafEntity>) r -> r, OAFENTITY_KRYO_ENC));
+		}
 
-        Dataset<?> groupedEntities = allEntities
-                .map(
-                        (MapFunction<OafEntity, OafEntity>) entity -> GraphCleaningFunctions
-                                .applyCoarVocabularies(entity, vocs),
-                        OAFENTITY_KRYO_ENC)
-                .groupByKey((MapFunction<OafEntity, String>) OafEntity::getId, Encoders.STRING())
-                .reduceGroups((ReduceFunction<OafEntity>) MergeUtils::checkedMerge)
-                .map(
-                        (MapFunction<Tuple2<String, OafEntity>, Tuple2<String, OafEntity>>) t -> new Tuple2<>(
-                                t._2().getClass().getName(), t._2()),
-                        Encoders.tuple(Encoders.STRING(), OAFENTITY_KRYO_ENC));
+		Dataset<?> groupedEntities = allEntities
+			.map(
+				(MapFunction<OafEntity, OafEntity>) entity -> GraphCleaningFunctions
+					.applyCoarVocabularies(entity, vocs),
+				OAFENTITY_KRYO_ENC)
+			.groupByKey((MapFunction<OafEntity, String>) OafEntity::getId, Encoders.STRING())
+			.reduceGroups((ReduceFunction<OafEntity>) OafMapperUtils::mergeEntities)
+			.map(
+				(MapFunction<Tuple2<String, OafEntity>, Tuple2<String, OafEntity>>) t -> new Tuple2<>(
+					t._2().getClass().getName(), t._2()),
+				Encoders.tuple(Encoders.STRING(), OAFENTITY_KRYO_ENC));
 
-        // pivot on "_1" (classname of the entity)
-        // created columns containing only entities of the same class
-        for (Map.Entry<EntityType, Class> e : ModelSupport.entityTypes.entrySet()) {
-            String entity = e.getKey().name();
-            Class<? extends OafEntity> entityClass = e.getValue();
+		// pivot on "_1" (classname of the entity)
+		// created columns containing only entities of the same class
+		for (Map.Entry<EntityType, Class> e : ModelSupport.entityTypes.entrySet()) {
+			String entity = e.getKey().name();
+			Class<? extends OafEntity> entityClass = e.getValue();
 
-            groupedEntities = groupedEntities
-                    .withColumn(
-                            entity,
-                            when(col("_1").equalTo(entityClass.getName()), col("_2")));
-        }
+			groupedEntities = groupedEntities
+				.withColumn(
+					entity,
+					when(col("_1").equalTo(entityClass.getName()), col("_2")));
+		}
 
-        groupedEntities
-                .drop("_1", "_2")
-                .write()
-                .mode(SaveMode.Overwrite)
-                .option("compression", "gzip")
-                .save(checkpointPath);
+		groupedEntities
+			.drop("_1", "_2")
+			.write()
+			.mode(SaveMode.Overwrite)
+			.option("compression", "gzip")
+			.save(checkpointPath);
 
-        ForkJoinPool parPool = new ForkJoinPool(ModelSupport.entityTypes.size());
+		ForkJoinPool parPool = new ForkJoinPool(ModelSupport.entityTypes.size());
 
-        ModelSupport.entityTypes
-                .entrySet()
-                .stream()
-                .map(e -> parPool.submit(() -> {
-                    String entity = e.getKey().name();
-                    Class<? extends OafEntity> entityClass = e.getValue();
+		ModelSupport.entityTypes
+			.entrySet()
+			.stream()
+			.map(e -> parPool.submit(() -> {
+				String entity = e.getKey().name();
+				Class<? extends OafEntity> entityClass = e.getValue();
 
-                    spark
-                            .read()
-                            .load(checkpointPath)
-                            .select(col(entity).as("value"))
-                            .filter("value IS NOT NULL")
-                            .as(OAFENTITY_KRYO_ENC)
-                            .map((MapFunction<OafEntity, OafEntity>) r -> r, (Encoder<OafEntity>) Encoders.bean(entityClass))
-                            .filter(filterInvisible ? "dataInfo.invisible != TRUE" : "TRUE")
-                            .write()
-                            .mode(SaveMode.Overwrite)
-                            .option("compression", "gzip")
-                            .json(outputPath + "/" + entity);
-                }))
-                .collect(Collectors.toList())
-                .forEach(t -> {
-                    try {
-                        t.get();
-                    } catch (InterruptedException | ExecutionException e) {
-                        throw new RuntimeException(e);
-                    }
-                });
-    }
+				spark
+					.read()
+					.load(checkpointPath)
+					.select(col(entity).as("value"))
+					.filter("value IS NOT NULL")
+					.as(OAFENTITY_KRYO_ENC)
+					.map((MapFunction<OafEntity, OafEntity>) r -> r, (Encoder<OafEntity>) Encoders.bean(entityClass))
+					.filter(filterInvisible ? "dataInfo.invisible != TRUE" : "TRUE")
+					.write()
+					.mode(SaveMode.Overwrite)
+					.option("compression", "gzip")
+					.json(outputPath + "/" + entity);
+			}))
+			.collect(Collectors.toList())
+			.forEach(t -> {
+				try {
+					t.get();
+				} catch (InterruptedException | ExecutionException e) {
+					throw new RuntimeException(e);
+				}
+			});
+	}
 }

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeComparator.java

@@ -1,79 +0,0 @@
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-//
-// Source code recreated from a .class file by IntelliJ IDEA
-// (powered by FernFlower decompiler)
-//
-
-
-import eu.dnetlib.dhp.schema.common.EntityType;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.Oaf;
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-import eu.dnetlib.dhp.schema.oaf.Result;
-
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-public class MergeComparator implements Comparator<Oaf> {
-    public MergeComparator() {
-    }
-
-    public int compare(Oaf left, Oaf right) {
-        // nulls at the end
-        if (left == null && right == null) {
-            return 0;
-        } else if (left == null) {
-            return -1;
-        } else if (right == null) {
-            return 1;
-        }
-
-        // invisible
-        if (left.getDataInfo() != null && left.getDataInfo().getInvisible() == true) {
-            if (right.getDataInfo() != null && right.getDataInfo().getInvisible() == false) {
-                return -1;
-            }
-        }
-
-        // collectedfrom
-        HashSet<String> lCf = getCollectedFromIds(left);
-        HashSet<String> rCf = getCollectedFromIds(right);
-        if (lCf.contains("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2") && !rCf.contains("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2")) {
-            return -1;
-        } else if (!lCf.contains("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2") && rCf.contains("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2")) {
-            return 1;
-        }
-
-
-
-        SubEntityType lClass = SubEntityType.fromClass(left.getClass());
-        SubEntityType rClass = SubEntityType.fromClass(right.getClass());
-        return lClass.ordinal() - rClass.ordinal();
-
-    }
-
-    protected HashSet<String> getCollectedFromIds(Oaf left) {
-        return (HashSet) Optional.ofNullable(left.getCollectedfrom()).map((cf) -> {
-            return (HashSet) cf.stream().map(KeyValue::getKey).collect(Collectors.toCollection(HashSet::new));
-        }).orElse(new HashSet());
-    }
-
-    enum SubEntityType {
-        publication, dataset, software, otherresearchproduct, datasource, organization, project;
-
-        /**
-         * Resolves the EntityType, given the relative class name
-         *
-         * @param clazz the given class name
-         * @param <T> actual OafEntity subclass
-         * @return the EntityType associated to the given class
-         */
-        public static <T extends Oaf> SubEntityType fromClass(Class<T> clazz) {
-            return valueOf(clazz.getSimpleName().toLowerCase());
-        }
-    }
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java

@@ -1,707 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-import eu.dnetlib.dhp.schema.common.AccessRightComparator;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.common.ModelSupport;
-import eu.dnetlib.dhp.schema.oaf.*;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.commons.lang3.tuple.Pair;
-
-import java.text.ParseException;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static com.google.common.base.Objects.firstNonNull;
-import static com.google.common.base.Preconditions.checkArgument;
-
-public class MergeUtils {
-
-    public static <T extends Oaf> T checkedMerge(final T left, final T right) {
-        return (T) merge(left, right, false);
-    }
-
-    public static Oaf merge(final Oaf left, final Oaf right) {
-        return merge(left, right, false);
-    }
-
-    public static Oaf merge(final Oaf left, final Oaf right, boolean checkDelegatedAuthority) {
-        if (sameClass(left, right, OafEntity.class)) {
-            return mergeEntities(left, right, checkDelegatedAuthority);
-        } else if (sameClass(left, right, Relation.class)) {
-            return mergeRelation((Relation) left, (Relation) right);
-        } else {
-            throw new RuntimeException(
-                    String
-                            .format(
-                                    "MERGE_FROM_AND_GET incompatible types: %s, %s",
-                                    left.getClass().getCanonicalName(), right.getClass().getCanonicalName()));
-        }
-    }
-
-    private static <T extends Oaf> boolean sameClass(Object left, Object right, Class<T> cls) {
-        return cls.isAssignableFrom(left.getClass()) && cls.isAssignableFrom(right.getClass());
-    }
-
-    private static Oaf mergeEntities(Oaf left, Oaf right, boolean checkDelegatedAuthority) {
-
-        if (sameClass(left, right, Result.class)) {
-            if (!left.getClass().equals(right.getClass()) || checkDelegatedAuthority) {
-                return mergeResultsOfDifferentTypes((Result)left, (Result) right);
-            }
-
-            if (sameClass(left, right, Publication.class)) {
-                return mergePublication((Publication) left, (Publication) right);
-            }
-            if (sameClass(left, right, Dataset.class)) {
-                return mergeDataset((Dataset) left, (Dataset) right);
-            }
-            if (sameClass(left, right, OtherResearchProduct.class)) {
-                return mergeORP((OtherResearchProduct) left, (OtherResearchProduct) right);
-            }
-            if (sameClass(left, right, Software.class)) {
-                return mergeSoftware((Software) left, (Software) right);
-            }
-
-            return mergeResult((Result) left, (Result) right);
-        } else if (sameClass(left, right, Datasource.class)) {
-            // TODO
-            final int trust = compareTrust(left, right);
-            return mergeOafEntityFields((Datasource) left, (Datasource) right, trust);
-        } else if (sameClass(left, right, Organization.class)) {
-            return mergeOrganization((Organization) left, (Organization) right);
-        } else if (sameClass(left, right, Project.class)) {
-            return mergeProject((Project) left, (Project) right);
-        } else {
-            throw new RuntimeException(
-                    String
-                            .format(
-                                    "MERGE_FROM_AND_GET incompatible types: %s, %s",
-                                    left.getClass().getCanonicalName(), right.getClass().getCanonicalName()));
-        }
-    }
-
-    /**
-     * This method is used in the global result grouping phase. It checks if one of the two is from a delegated authority
-     * https://graph.openaire.eu/docs/data-model/pids-and-identifiers#delegated-authorities and in that case it prefers
-     * such version.
-     * <p>
-     * Otherwise, it considers a resulttype priority order implemented in {@link ResultTypeComparator}
-     * and proceeds with the canonical property merging.
-     *
-     * @param left
-     * @param right
-     * @return
-     */
-    private static <T extends Result> T mergeResultsOfDifferentTypes(T left, T right) {
-
-        final boolean leftFromDelegatedAuthority = isFromDelegatedAuthority(left);
-        final boolean rightFromDelegatedAuthority = isFromDelegatedAuthority(right);
-
-        if (leftFromDelegatedAuthority && !rightFromDelegatedAuthority) {
-            return left;
-        }
-        if (!leftFromDelegatedAuthority && rightFromDelegatedAuthority) {
-            return right;
-        }
-        //TODO: raise trust to have preferred fields from one or the other??
-        if (new ResultTypeComparator().compare(left, right) < 0) {
-            return mergeResult(left, right);
-        } else {
-            return mergeResult(right, left);
-        }
-    }
-
-    private static DataInfo chooseDataInfo(DataInfo left, DataInfo right, int trust) {
-        if (trust > 0) {
-            return left;
-        } else if (trust == 0) {
-            if (left == null || (left.getInvisible() != null && left.getInvisible().equals(Boolean.TRUE))) {
-                return right;
-            } else {
-                return left;
-            }
-        } else {
-            return right;
-        }
-    }
-
-    private static String chooseString(String left, String right, int trust) {
-        if (trust > 0) {
-            return left;
-        } else if (trust == 0) {
-            return StringUtils.isNotBlank(left) ? left : right;
-        } else {
-            return right;
-        }
-    }
-
-    private static <T> T chooseReference(T left, T right, int trust) {
-        if (trust > 0) {
-            return left;
-        } else if (trust == 0) {
-            return left != null ? left : right;
-        } else {
-            return right;
-        }
-    }
-
-    private static Long max(Long left, Long right) {
-        if (left == null)
-            return right;
-        if (right == null)
-            return left;
-
-        return Math.max(left, right);
-    }
-
-    // trust ??
-    private static Boolean booleanOR(Boolean a, Boolean b) {
-        if (a == null) {
-            return b;
-        } else if (b == null) {
-            return a;
-        }
-
-        return a || b;
-    }
-
-
-    private static <T> List<T> unionDistinctLists(final List<T> left, final List<T> right, int trust) {
-        if (left == null) {
-            return right;
-        } else if (right == null) {
-            return left;
-        }
-
-        List<T> h = trust >= 0 ? left : right;
-        List<T> l = trust >= 0 ? right : left;
-
-        return Stream.concat(h.stream(), l.stream())
-                .filter(Objects::nonNull)
-                .distinct()
-                .collect(Collectors.toList());
-    }
-
-    private static List<String> unionDistinctListOfString(final List<String> l, final List<String> r) {
-        if (l == null) {
-            return r;
-        } else if (r == null) {
-            return l;
-        }
-
-        return Stream.concat(l.stream(), r.stream())
-                .filter(StringUtils::isNotBlank)
-                .distinct()
-                .collect(Collectors.toList());
-    }
-
-    //TODO review
-    private static List<KeyValue> mergeKeyValue(List<KeyValue> left, List<KeyValue> right, int trust) {
-        if (trust < 0) {
-            List<KeyValue> s = left;
-            left = right;
-            right = s;
-        }
-
-        HashMap<String, KeyValue> values = new HashMap<>();
-        left.forEach(kv -> values.put(kv.getKey(), kv));
-        right.forEach(kv -> values.putIfAbsent(kv.getKey(), kv));
-
-        return new ArrayList<>(values.values());
-    }
-
-    private static List<StructuredProperty> unionTitle(List<StructuredProperty> left, List<StructuredProperty> right, int trust) {
-        if (left == null) {
-            return right;
-        } else if (right == null) {
-            return left;
-        }
-
-        List<StructuredProperty> h = trust >= 0 ? left : right;
-        List<StructuredProperty> l = trust >= 0 ? right : left;
-
-        return Stream.concat(h.stream(), l.stream())
-                .filter(Objects::isNull)
-                .distinct()
-                .collect(Collectors.toList());
-    }
-
-    /**
-     * Internal utility that merges the common OafEntity fields
-     *
-     * @param merged
-     * @param enrich
-     * @param <T>
-     * @return
-     */
-    private static <T extends Oaf> T mergeOafFields(T merged, T enrich, int trust) {
-
-        //TODO: union of all values, but what does it mean with KeyValue pairs???
-        merged.setCollectedfrom(mergeKeyValue(merged.getCollectedfrom(), enrich.getCollectedfrom(), trust));
-        merged.setDataInfo(chooseDataInfo(merged.getDataInfo(), enrich.getDataInfo(), trust));
-        merged.setLastupdatetimestamp(max(merged.getLastupdatetimestamp(), enrich.getLastupdatetimestamp()));
-
-        return merged;
-    }
-
-    /**
-     * Internal utility that merges the common OafEntity fields
-     *
-     * @param original
-     * @param enrich
-     * @param <T>
-     * @return
-     */
-    private static <T extends OafEntity> T mergeOafEntityFields(T original, T enrich, int trust) {
-        final T merged = mergeOafFields(original, enrich, trust);
-
-        merged.setOriginalId(unionDistinctListOfString(merged.getOriginalId(), enrich.getOriginalId()));
-        merged.setPid(unionDistinctLists(merged.getPid(), enrich.getPid(), trust));
-        // dateofcollection mettere today quando si fa merge
-        merged.setDateofcollection(chooseString(merged.getDateofcollection(), enrich.getDateofcollection(), trust));
-        // setDateoftransformation mettere vuota in dedup, nota per Claudio
-        merged.setDateoftransformation(chooseString(merged.getDateoftransformation(), enrich.getDateoftransformation(), trust));
-        // TODO: was missing in OafEntity.merge
-        merged.setExtraInfo(unionDistinctLists(merged.getExtraInfo(), enrich.getExtraInfo(), trust));
-        //oaiprovenanze da mettere a null quando si genera merge
-        merged.setOaiprovenance(chooseReference(merged.getOaiprovenance(), enrich.getOaiprovenance(), trust));
-        merged.setMeasures(unionDistinctLists(merged.getMeasures(), enrich.getMeasures(), trust));
-
-        return merged;
-    }
-
-
-    public static <T extends Relation>  T mergeRelation(T original, T enrich) {
-        int trust = compareTrust(original, enrich);
-        T merge = mergeOafFields(original, enrich, trust);
-
-        checkArgument(Objects.equals(merge.getSource(), enrich.getSource()), "source ids must be equal");
-        checkArgument(Objects.equals(merge.getTarget(), enrich.getTarget()), "target ids must be equal");
-        checkArgument(Objects.equals(merge.getRelType(), enrich.getRelType()), "relType(s) must be equal");
-        checkArgument(
-                Objects.equals(merge.getSubRelType(), enrich.getSubRelType()), "subRelType(s) must be equal");
-        checkArgument(Objects.equals(merge.getRelClass(), enrich.getRelClass()), "relClass(es) must be equal");
-
-        //merge.setProvenance(mergeLists(merge.getProvenance(), enrich.getProvenance()));
-
-        //TODO: trust ??
-        merge.setValidated(booleanOR(merge.getValidated(), enrich.getValidated()));
-        try {
-            merge.setValidationDate(ModelSupport.oldest(merge.getValidationDate(), enrich.getValidationDate()));
-        } catch (ParseException e) {
-            throw new IllegalArgumentException(String
-                    .format(
-                            "invalid validation date format in relation [s:%s, t:%s]: %s", merge.getSource(),
-                            merge.getTarget(),
-                            merge.getValidationDate()));
-        }
-
-        // TODO keyvalue merge
-        merge.setProperties(mergeKeyValue(merge.getProperties(), enrich.getProperties(), trust));
-
-        return merge;
-    }
-
-    public static <T extends Result> T mergeResult(T original, T enrich) {
-        final int trust = compareTrust(original, enrich);
-        T merge = mergeOafEntityFields(original, enrich, trust);
-
-        if (merge.getProcessingchargeamount() == null || StringUtils.isBlank(merge.getProcessingchargeamount().getValue())) {
-            merge.setProcessingchargeamount(enrich.getProcessingchargeamount());
-            merge.setProcessingchargecurrency(enrich.getProcessingchargecurrency());
-        }
-
-        // author = usare la stessa logica che in dedup
-        merge.setAuthor(chooseReference(merge.getAuthor(), enrich.getAuthor(), trust));
-        // il primo che mi arriva secondo l'ordinamento per priorita'
-        merge.setResulttype(chooseReference(merge.getResulttype(), enrich.getResulttype(), trust));
-        // gestito come il resulttype perche' e' un subtype
-        merge.setMetaResourceType(chooseReference(merge.getMetaResourceType(), enrich.getMetaResourceType(), trust));
-        // spostiamo nell'instance e qui prendo il primo che arriva
-        merge.setLanguage(chooseReference(merge.getLanguage(), enrich.getLanguage(), trust));
-        // country lasicamo,o cosi' -> parentesi sul datainfo
-        merge.setCountry(unionDistinctLists(merge.getCountry(), enrich.getCountry(), trust));
-        //ok
-        merge.setSubject(unionDistinctLists(merge.getSubject(), enrich.getSubject(), trust));
-        // union per priority quindi vanno in append
-        merge.setTitle(unionTitle(merge.getTitle(), enrich.getTitle(), trust));
-        //ok
-        merge.setRelevantdate(unionDistinctLists(merge.getRelevantdate(), enrich.getRelevantdate(), trust));
-        // prima trust e poi longest list
-        merge.setDescription(longestLists(merge.getDescription(), enrich.getDescription()));
-        // trust piu' alto e poi piu' vecchia
-        merge.setDateofacceptance(chooseReference(merge.getDateofacceptance(), enrich.getDateofacceptance(), trust));
-        // ok, ma publisher va messo ripetibile
-        merge.setPublisher(chooseReference(merge.getPublisher(), enrich.getPublisher(), trust));
-        // ok
-        merge.setEmbargoenddate(chooseReference(merge.getEmbargoenddate(), enrich.getEmbargoenddate(), trust));
-        // ok
-        merge.setSource(unionDistinctLists(merge.getSource(), enrich.getSource(), trust));
-        // ok
-        merge.setFulltext(unionDistinctLists(merge.getFulltext(), enrich.getFulltext(), trust));
-        // ok
-        merge.setFormat(unionDistinctLists(merge.getFormat(), enrich.getFormat(), trust));
-        // ok
-        merge.setContributor(unionDistinctLists(merge.getContributor(), enrich.getContributor(), trust));
-
-        // prima prendo l'higher trust, su questo prendo il valore migliore nelle istanze TODO
-        // trust maggiore ma a parita' di trust il piu' specifico (base del vocabolario)
-        // vedi note
-        merge.setResourcetype(firstNonNull(merge.getResourcetype(), enrich.getResourcetype()));
-
-        // ok
-        merge.setCoverage(unionDistinctLists(merge.getCoverage(), enrich.getCoverage(), trust));
-
-        // most open ok
-        if (enrich.getBestaccessright() != null
-                && new AccessRightComparator<>()
-                .compare(enrich.getBestaccessright(), merge.getBestaccessright()) < 0) {
-            merge.setBestaccessright(enrich.getBestaccessright());
-        }
-
-        // TODO merge of datainfo given same id
-        merge.setContext(unionDistinctLists(merge.getContext(), enrich.getContext(), trust));
-
-        //ok
-        merge.setExternalReference(unionDistinctLists(merge.getExternalReference(), enrich.getExternalReference(), trust));
-
-        //instance enrichment or union
-        // review instance equals => add pid to comparision
-        if (!isAnEnrichment(merge) && !isAnEnrichment(enrich))
-            merge.setInstance(unionDistinctLists(merge.getInstance(), enrich.getInstance(), trust));
-        else {
-            final List<Instance> enrichmentInstances = isAnEnrichment(merge) ? merge.getInstance()
-                    : enrich.getInstance();
-            final List<Instance> enrichedInstances = isAnEnrichment(merge) ? enrich.getInstance()
-                    : merge.getInstance();
-            if (isAnEnrichment(merge))
-                merge.setDataInfo(enrich.getDataInfo());
-            merge.setInstance(enrichInstances(enrichedInstances, enrichmentInstances));
-        }
-
-        merge.setEoscifguidelines(unionDistinctLists(merge.getEoscifguidelines(), enrich.getEoscifguidelines(), trust));
-        merge.setIsGreen(booleanOR(merge.getIsGreen(), enrich.getIsGreen()));
-        // OK but should be list of values
-        merge.setOpenAccessColor(chooseReference(merge.getOpenAccessColor(), enrich.getOpenAccessColor(), trust));
-        merge.setIsInDiamondJournal(booleanOR(merge.getIsInDiamondJournal(), enrich.getIsInDiamondJournal()));
-        merge.setPubliclyFunded(booleanOR(merge.getPubliclyFunded(), enrich.getPubliclyFunded()));
-
-        return merge;
-    }
-
-    private static <T extends OtherResearchProduct> T mergeORP(T original, T enrich) {
-        int trust = compareTrust(original, enrich);
-        final T merge = mergeResult(original, enrich);
-
-        merge.setContactperson(unionDistinctLists(merge.getContactperson(), enrich.getContactperson(), trust));
-        merge.setContactgroup(unionDistinctLists(merge.getContactgroup(), enrich.getContactgroup(), trust));
-        merge.setTool(unionDistinctLists(merge.getTool(), enrich.getTool(), trust));
-
-        return merge;
-    }
-
-    private static <T extends Software> T mergeSoftware(T original, T enrich) {
-        int trust = compareTrust(original, enrich);
-        final T merge = mergeResult(original, enrich);
-
-        merge.setDocumentationUrl(unionDistinctLists(merge.getDocumentationUrl(), enrich.getDocumentationUrl(), trust));
-        merge.setLicense(unionDistinctLists(merge.getLicense(), enrich.getLicense(), trust));
-        merge.setCodeRepositoryUrl(chooseReference(merge.getCodeRepositoryUrl(), enrich.getCodeRepositoryUrl(), trust));
-        merge.setProgrammingLanguage(chooseReference(merge.getProgrammingLanguage(), enrich.getProgrammingLanguage(), trust));
-
-        return merge;
-    }
-
-    private static <T extends Dataset> T mergeDataset(T original, T enrich) {
-        int trust = compareTrust(original, enrich);
-        T merge = mergeResult(original, enrich);
-
-        merge.setStoragedate(chooseReference(merge.getStoragedate(), enrich.getStoragedate(), trust));
-        merge.setDevice(chooseReference(merge.getDevice(), enrich.getDevice(), trust));
-        merge.setSize(chooseReference(merge.getSize(), enrich.getSize(), trust));
-        merge.setVersion(chooseReference(merge.getVersion(), enrich.getVersion(), trust));
-        merge.setLastmetadataupdate(chooseReference(merge.getLastmetadataupdate(), enrich.getLastmetadataupdate(), trust));
-        merge.setMetadataversionnumber(chooseReference(merge.getMetadataversionnumber(), enrich.getMetadataversionnumber(), trust));
-        merge.setGeolocation(unionDistinctLists(merge.getGeolocation(), enrich.getGeolocation(), trust));
-
-        return merge;
-    }
-
-    public static <T extends Publication> T mergePublication(T original, T enrich) {
-        final int trust = compareTrust(original, enrich);
-        T merged = mergeResult(original, enrich);
-
-        merged.setJournal(chooseReference(merged.getJournal(), enrich.getJournal(), trust));
-
-        return merged;
-    }
-
-    private static <T extends Organization> T mergeOrganization(T left, T enrich) {
-        int trust = compareTrust(left, enrich);
-        T merged = mergeOafEntityFields(left, enrich, trust);
-
-        merged.setLegalshortname(chooseReference(merged.getLegalshortname(), enrich.getLegalshortname(), trust));
-        merged.setLegalname(chooseReference(merged.getLegalname(), enrich.getLegalname(), trust));
-        merged.setAlternativeNames(unionDistinctLists(enrich.getAlternativeNames(), merged.getAlternativeNames(), trust));
-        merged.setWebsiteurl(chooseReference(merged.getWebsiteurl(), enrich.getWebsiteurl(), trust));
-        merged.setLogourl(chooseReference(merged.getLogourl(), enrich.getLogourl(), trust));
-        merged.setEclegalbody(chooseReference(merged.getEclegalbody(), enrich.getEclegalbody(), trust));
-        merged.setEclegalperson(chooseReference(merged.getEclegalperson(), enrich.getEclegalperson(), trust));
-        merged.setEcnonprofit(chooseReference(merged.getEcnonprofit(), enrich.getEcnonprofit(), trust));
-        merged.setEcresearchorganization(chooseReference(merged.getEcresearchorganization(), enrich.getEcresearchorganization(), trust));
-        merged.setEchighereducation(chooseReference(merged.getEchighereducation(), enrich.getEchighereducation(), trust));
-        merged.setEcinternationalorganizationeurinterests(chooseReference(merged.getEcinternationalorganizationeurinterests(), enrich.getEcinternationalorganizationeurinterests(), trust));
-        merged.setEcinternationalorganization(chooseReference(merged.getEcinternationalorganization(), enrich.getEcinternationalorganization(), trust));
-        merged.setEcenterprise(chooseReference(merged.getEcenterprise(), enrich.getEcenterprise(), trust));
-        merged.setEcsmevalidated(chooseReference(merged.getEcsmevalidated(), enrich.getEcsmevalidated(), trust));
-        merged.setEcnutscode(chooseReference(merged.getEcnutscode(), enrich.getEcnutscode(), trust));
-        merged.setCountry(chooseReference(merged.getCountry(), enrich.getCountry(), trust));
-
-        return merged;
-    }
-
-    public static <T extends Project> T mergeProject(T original, T enrich) {
-        int trust = compareTrust(original, enrich);
-        T merged = mergeOafEntityFields(original, enrich, trust);
-
-        merged.setWebsiteurl(chooseReference(merged.getWebsiteurl(), enrich.getWebsiteurl(), trust));
-        merged.setCode(chooseReference(merged.getCode(), enrich.getCode(), trust));
-        merged.setAcronym(chooseReference(merged.getAcronym(), enrich.getAcronym(), trust));
-        merged.setTitle(chooseReference(merged.getTitle(), enrich.getTitle(), trust));
-        merged.setStartdate(chooseReference(merged.getStartdate(), enrich.getStartdate(), trust));
-        merged.setEnddate(chooseReference(merged.getEnddate(), enrich.getEnddate(), trust));
-        merged.setCallidentifier(chooseReference(merged.getCallidentifier(), enrich.getCallidentifier(), trust));
-        merged.setKeywords(chooseReference(merged.getKeywords(), enrich.getKeywords(), trust));
-        merged.setDuration(chooseReference(merged.getDuration(), enrich.getDuration(), trust));
-        merged.setEcsc39(chooseReference(merged.getEcsc39(), enrich.getEcsc39(), trust));
-        merged.setOamandatepublications(chooseReference(merged.getOamandatepublications(), enrich.getOamandatepublications(), trust));
-        merged.setEcarticle29_3(chooseReference(merged.getEcarticle29_3(), enrich.getEcarticle29_3(), trust));
-        merged.setSubjects(unionDistinctLists(merged.getSubjects(), enrich.getSubjects(), trust));
-        merged.setFundingtree(unionDistinctLists(merged.getFundingtree(), enrich.getFundingtree(), trust));
-        merged.setContracttype(chooseReference(merged.getContracttype(), enrich.getContracttype(), trust));
-        merged.setOptional1(chooseReference(merged.getOptional1(), enrich.getOptional1(), trust));
-        merged.setOptional2(chooseReference(merged.getOptional2(), enrich.getOptional2(), trust));
-        merged.setJsonextrainfo(chooseReference(merged.getJsonextrainfo(), enrich.getJsonextrainfo(), trust));
-        merged.setContactfullname(chooseReference(merged.getContactfullname(), enrich.getContactfullname(), trust));
-        merged.setContactfax(chooseReference(merged.getContactfax(), enrich.getContactfax(), trust));
-        merged.setContactphone(chooseReference(merged.getContactphone(), enrich.getContactphone(), trust));
-        merged.setContactemail(chooseReference(merged.getContactemail(), enrich.getContactemail(), trust));
-        merged.setSummary(chooseReference(merged.getSummary(), enrich.getSummary(), trust));
-        merged.setCurrency(chooseReference(merged.getCurrency(), enrich.getCurrency(), trust));
-
-        //missin in Project.merge
-        merged.setTotalcost(chooseReference(merged.getTotalcost(), enrich.getTotalcost(), trust));
-        merged.setFundedamount(chooseReference(merged.getFundedamount(), enrich.getFundedamount(), trust));
-
-        // trust ??
-        if (enrich.getH2020topiccode() != null && StringUtils.isEmpty(merged.getH2020topiccode())) {
-            merged.setH2020topiccode(enrich.getH2020topiccode());
-            merged.setH2020topicdescription(enrich.getH2020topicdescription());
-        }
-
-        merged.setH2020classification(unionDistinctLists(merged.getH2020classification(), enrich.getH2020classification(), trust));
-
-        return merged;
-    }
-
-
-    /**
-     * Longest lists list.
-     *
-     * @param a the a
-     * @param b the b
-     * @return the list
-     */
-    public static List<Field<String>> longestLists(List<Field<String>> a, List<Field<String>> b) {
-        if (a == null || b == null)
-            return a == null ? b : a;
-
-        return a.size() >= b.size() ? a : b;
-    }
-
-    /**
-     * This main method apply the enrichment of the instances
-     *
-     * @param toEnrichInstances   the instances that could be enriched
-     * @param enrichmentInstances the enrichment instances
-     * @return list of instances possibly enriched
-     */
-    private static List<Instance> enrichInstances(final List<Instance> toEnrichInstances,
-                                                  final List<Instance> enrichmentInstances) {
-        final List<Instance> enrichmentResult = new ArrayList<>();
-
-        if (toEnrichInstances == null) {
-            return enrichmentResult;
-        }
-        if (enrichmentInstances == null) {
-            return enrichmentResult;
-        }
-        Map<String, Instance> ri = toInstanceMap(enrichmentInstances);
-
-        toEnrichInstances.forEach(i -> {
-            final List<Instance> e = findEnrichmentsByPID(i.getPid(), ri);
-            if (e != null && e.size() > 0) {
-                e.forEach(enr -> applyEnrichment(i, enr));
-            } else {
-                final List<Instance> a = findEnrichmentsByPID(i.getAlternateIdentifier(), ri);
-                if (a != null && a.size() > 0) {
-                    a.forEach(enr -> applyEnrichment(i, enr));
-                }
-            }
-            enrichmentResult.add(i);
-        });
-        return enrichmentResult;
-    }
-
-    /**
-     * This method converts the list of instance enrichments
-     * into a Map where the key is the normalized identifier
-     * and the value is the instance itself
-     *
-     * @param ri the list of enrichment instances
-     * @return the result map
-     */
-    private static Map<String, Instance> toInstanceMap(final List<Instance> ri) {
-        return ri
-                .stream()
-                .filter(i -> i.getPid() != null || i.getAlternateIdentifier() != null)
-                .flatMap(i -> {
-                    final List<Pair<String, Instance>> result = new ArrayList<>();
-                    if (i.getPid() != null)
-                        i
-                                .getPid()
-                                .stream()
-                                .filter(MergeUtils::validPid)
-                                .forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
-                    if (i.getAlternateIdentifier() != null)
-                        i
-                                .getAlternateIdentifier()
-                                .stream()
-                                .filter(MergeUtils::validPid)
-                                .forEach(p -> result.add(new ImmutablePair<>(extractKeyFromPid(p), i)));
-                    return result.stream();
-                })
-                .collect(
-                        Collectors
-                                .toMap(
-                                        Pair::getLeft,
-                                        Pair::getRight,
-                                        (a, b) -> a));
-    }
-
-    private static boolean isFromDelegatedAuthority(Result r) {
-        return Optional
-                .ofNullable(r.getInstance())
-                .map(
-                        instance -> instance
-                                .stream()
-                                .filter(i -> Objects.nonNull(i.getCollectedfrom()))
-                                .map(i -> i.getCollectedfrom().getKey())
-                                .anyMatch(cfId -> IdentifierFactory.delegatedAuthorityDatasourceIds().contains(cfId)))
-                .orElse(false);
-    }
-
-    /**
-     * Valid pid boolean.
-     *
-     * @param p the p
-     * @return the boolean
-     */
-    private static boolean validPid(final StructuredProperty p) {
-        return p.getValue() != null && p.getQualifier() != null && p.getQualifier().getClassid() != null;
-    }
-
-    /**
-     * Normalize pid string.
-     *
-     * @param pid the pid
-     * @return the string
-     */
-    private static String extractKeyFromPid(final StructuredProperty pid) {
-        if (pid == null)
-            return null;
-        final StructuredProperty normalizedPid = CleaningFunctions.normalizePidValue(pid);
-
-        return String.format("%s::%s", normalizedPid.getQualifier().getClassid(), normalizedPid.getValue());
-    }
-
-    /**
-     * This utility method finds the list of enrichment instances
-     * that match one or more PIDs in the input list
-     *
-     * @param pids        the list of PIDs
-     * @param enrichments the List of enrichment instances having the same pid
-     * @return the list
-     */
-    private static List<Instance> findEnrichmentsByPID(final List<StructuredProperty> pids,
-                                                       final Map<String, Instance> enrichments) {
-        if (pids == null || enrichments == null)
-            return null;
-        return pids
-                .stream()
-                .map(MergeUtils::extractKeyFromPid)
-                .map(enrichments::get)
-                .filter(Objects::nonNull)
-                .collect(Collectors.toList());
-    }
-
-    /**
-     * Is an enrichment boolean.
-     *
-     * @param e the e
-     * @return the boolean
-     */
-    private static boolean isAnEnrichment(OafEntity e) {
-        return e.getDataInfo() != null &&
-                e.getDataInfo().getProvenanceaction() != null
-                && ModelConstants.PROVENANCE_ENRICH.equalsIgnoreCase(e.getDataInfo().getProvenanceaction().getClassid());
-    }
-
-    /**
-     * This method apply enrichment on a single instance
-     * The enrichment consists of replacing values on
-     * single attribute only if in the current instance is missing
-     * The only repeatable field enriched is measures
-     *
-     * @param merge the current instance
-     * @param enrichment      the enrichment instance
-     */
-    private static void applyEnrichment(final Instance merge, final Instance enrichment) {
-        if (merge == null || enrichment == null)
-            return;
-
-        merge.setLicense(firstNonNull(merge.getLicense(), enrichment.getLicense()));
-        merge.setAccessright(firstNonNull(merge.getAccessright(), enrichment.getAccessright()));
-        merge.setInstancetype(firstNonNull(merge.getInstancetype(), enrichment.getInstancetype()));
-        merge.setInstanceTypeMapping(firstNonNull(merge.getInstanceTypeMapping(), enrichment.getInstanceTypeMapping()));
-        merge.setHostedby(firstNonNull(merge.getHostedby(), enrichment.getHostedby()));
-        merge.setUrl(unionDistinctLists(merge.getUrl(), enrichment.getUrl(), 0));
-        merge.setDistributionlocation(firstNonNull(merge.getDistributionlocation(), enrichment.getDistributionlocation()));
-        merge.setCollectedfrom(firstNonNull(merge.getCollectedfrom(), enrichment.getCollectedfrom()));
-        // pid and alternateId are used for matching
-        merge.setDateofacceptance(firstNonNull(merge.getDateofacceptance(), enrichment.getDateofacceptance()));
-        merge.setProcessingchargeamount(firstNonNull(merge.getProcessingchargeamount(), enrichment.getProcessingchargeamount()));
-        merge.setProcessingchargecurrency(firstNonNull(merge.getProcessingchargecurrency(), enrichment.getProcessingchargecurrency()));
-        merge.setRefereed(firstNonNull(merge.getRefereed(), enrichment.getRefereed()));
-        merge.setMeasures(unionDistinctLists(merge.getMeasures(), enrichment.getMeasures(), 0));
-        merge.setFulltext(firstNonNull(merge.getFulltext(), enrichment.getFulltext()));
-    }
-
-    private static int compareTrust(Oaf a, Oaf b) {
-        String left = Optional
-                .ofNullable(a.getDataInfo())
-                .map(DataInfo::getTrust)
-                .orElse("0.0");
-
-        String right = Optional
-                .ofNullable(b.getDataInfo())
-                .map(DataInfo::getTrust)
-                .orElse("0.0");
-
-        return left.compareTo(right);
-    }
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java

@@ -14,6 +14,7 @@ import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 
 import eu.dnetlib.dhp.schema.common.AccessRightComparator;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.schema.oaf.*;
 
 public class OafMapperUtils {
@@ -21,6 +22,65 @@ public class OafMapperUtils {
 	private OafMapperUtils() {
 	}
 
+	public static Oaf merge(final Oaf left, final Oaf right) {
+		if (ModelSupport.isSubClass(left, OafEntity.class)) {
+			return mergeEntities((OafEntity) left, (OafEntity) right);
+		} else if (ModelSupport.isSubClass(left, Relation.class)) {
+			((Relation) left).mergeFrom((Relation) right);
+		} else {
+			throw new IllegalArgumentException("invalid Oaf type:" + left.getClass().getCanonicalName());
+		}
+		return left;
+	}
+
+	public static OafEntity mergeEntities(OafEntity left, OafEntity right) {
+		if (ModelSupport.isSubClass(left, Result.class)) {
+			return mergeResults((Result) left, (Result) right);
+		} else if (ModelSupport.isSubClass(left, Datasource.class)) {
+			left.mergeFrom(right);
+		} else if (ModelSupport.isSubClass(left, Organization.class)) {
+			left.mergeFrom(right);
+		} else if (ModelSupport.isSubClass(left, Project.class)) {
+			left.mergeFrom(right);
+		} else {
+			throw new IllegalArgumentException("invalid OafEntity subtype:" + left.getClass().getCanonicalName());
+		}
+		return left;
+	}
+
+	public static Result mergeResults(Result left, Result right) {
+
+		final boolean leftFromDelegatedAuthority = isFromDelegatedAuthority(left);
+		final boolean rightFromDelegatedAuthority = isFromDelegatedAuthority(right);
+
+		if (leftFromDelegatedAuthority && !rightFromDelegatedAuthority) {
+			return left;
+		}
+		if (!leftFromDelegatedAuthority && rightFromDelegatedAuthority) {
+			return right;
+		}
+
+		if (new ResultTypeComparator().compare(left, right) < 0) {
+			left.mergeFrom(right);
+			return left;
+		} else {
+			right.mergeFrom(left);
+			return right;
+		}
+	}
+
+	private static boolean isFromDelegatedAuthority(Result r) {
+		return Optional
+			.ofNullable(r.getInstance())
+			.map(
+				instance -> instance
+					.stream()
+					.filter(i -> Objects.nonNull(i.getCollectedfrom()))
+					.map(i -> i.getCollectedfrom().getKey())
+					.anyMatch(cfId -> IdentifierFactory.delegatedAuthorityDatasourceIds().contains(cfId)))
+			.orElse(false);
+	}
+
 	public static KeyValue keyValue(final String k, final String v) {
 		final KeyValue kv = new KeyValue();
 		kv.setKey(k);

dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java

@@ -1,109 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-@Disabled
-class ZenodoAPIClientTest {
-
-	private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
-	private final String ACCESS_TOKEN = "";
-
-	private final String CONCEPT_REC_ID = "657113";
-
-	private final String depositionId = "674915";
-
-	@Test
-	void testUploadOldDeposition() throws IOException, MissingConceptDoiException {
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-		Assertions.assertEquals(200, client.uploadOpenDeposition(depositionId));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz"));
-
-		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
-		Assertions.assertEquals(200, client.sendMretadata(metadata));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	void testNewDeposition() throws IOException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-		Assertions.assertEquals(201, client.newDeposition());
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz"));
-
-		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
-		Assertions.assertEquals(200, client.sendMretadata(metadata));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	void testNewVersionNewName() throws IOException, MissingConceptDoiException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-
-		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/newVersion")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition"));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	void testNewVersionOldName() throws IOException, MissingConceptDoiException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-
-		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/newVersion2")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition"));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-}

dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtilsTest.java

@@ -1,111 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-import static org.junit.jupiter.api.Assertions.*;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import eu.dnetlib.dhp.schema.common.ModelSupport;
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.Dataset;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.Publication;
-import eu.dnetlib.dhp.schema.oaf.Result;
-
-public class MergeUtilsTest {
-
-	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
-		.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
-	@Test
-	void testMergePubs() throws IOException {
-		Publication p1 = read("publication_1.json", Publication.class);
-		Publication p2 = read("publication_2.json", Publication.class);
-		Dataset d1 = read("dataset_1.json", Dataset.class);
-		Dataset d2 = read("dataset_2.json", Dataset.class);
-
-		assertEquals(1, p1.getCollectedfrom().size());
-		assertEquals(ModelConstants.CROSSREF_ID, p1.getCollectedfrom().get(0).getKey());
-		assertEquals(1, d2.getCollectedfrom().size());
-		assertFalse(cfId(d2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
-
-		assertEquals(1, p2.getCollectedfrom().size());
-		assertFalse(cfId(p2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
-		assertEquals(1, d1.getCollectedfrom().size());
-		assertTrue(cfId(d1.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
-
-		final Result p1d2 = MergeUtils.checkedMerge(p1, d2);
-		assertEquals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID,  p1d2.getResulttype().getClassid());
-		assertTrue(p1d2 instanceof Publication);
-		assertEquals(p1.getId(), p1d2.getId());
-	}
-
-	@Test
-	void testMergePubs_1() throws IOException {
-		Publication p2 = read("publication_2.json", Publication.class);
-		Dataset d1 = read("dataset_1.json", Dataset.class);
-
-		final Result p2d1 = MergeUtils.checkedMerge(p2, d1);
-		assertEquals((ModelConstants.DATASET_RESULTTYPE_CLASSID), p2d1.getResulttype().getClassid());
-		assertTrue(p2d1 instanceof Dataset);
-		assertEquals(d1.getId(), p2d1.getId());
-		assertEquals(2, p2d1.getCollectedfrom().size());
-	}
-
-	@Test
-	void testMergePubs_2() throws IOException {
-		Publication p1 = read("publication_1.json", Publication.class);
-		Publication p2 = read("publication_2.json", Publication.class);
-
-		Result p1p2 = MergeUtils.checkedMerge(p1, p2);
-		assertTrue(p1p2 instanceof Publication);
-		assertEquals(p1.getId(), p1p2.getId());
-		assertEquals(2, p1p2.getCollectedfrom().size());
-	}
-
-	@Test
-	void testDelegatedAuthority_1() throws IOException {
-		Dataset d1 = read("dataset_2.json", Dataset.class);
-		Dataset d2 = read("dataset_delegated.json", Dataset.class);
-
-		assertEquals(1, d2.getCollectedfrom().size());
-		assertTrue(cfId(d2.getCollectedfrom()).contains(ModelConstants.ZENODO_OD_ID));
-
-		Result res = (Result) MergeUtils.merge(d1, d2, true);
-
-		assertEquals(d2, res);
-	}
-
-	@Test
-	void testDelegatedAuthority_2() throws IOException {
-		Dataset p1 = read("publication_1.json", Dataset.class);
-		Dataset d2 = read("dataset_delegated.json", Dataset.class);
-
-		assertEquals(1, d2.getCollectedfrom().size());
-		assertTrue(cfId(d2.getCollectedfrom()).contains(ModelConstants.ZENODO_OD_ID));
-
-		Result res = (Result) MergeUtils.merge(p1, d2, true);
-
-		assertEquals(d2, res);
-	}
-
-	protected HashSet<String> cfId(List<KeyValue> collectedfrom) {
-		return collectedfrom.stream().map(KeyValue::getKey).collect(Collectors.toCollection(HashSet::new));
-	}
-
-	protected <T extends Result> T read(String filename, Class<T> clazz) throws IOException {
-		final String json = IOUtils.toString(getClass().getResourceAsStream(filename));
-		return OBJECT_MAPPER.readValue(json, clazz);
-	}
-
-}

dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java

@@ -149,7 +149,7 @@ class OafMapperUtilsTest {
 	void testDate() {
 		final String date = GraphCleaningFunctions.cleanDate("23-FEB-1998");
 		assertNotNull(date);
-		assertEquals("1998-02-23", date);
+		System.out.println(date);
 	}
 
 	@Test
@@ -166,8 +166,8 @@ class OafMapperUtilsTest {
 
 		assertEquals(
 			ModelConstants.PUBLICATION_RESULTTYPE_CLASSID,
-			MergeUtils
-				.mergeResult(p1, d2)
+			OafMapperUtils
+				.mergeResults(p1, d2)
 				.getResulttype()
 				.getClassid());
 
@@ -178,8 +178,8 @@ class OafMapperUtilsTest {
 
 		assertEquals(
 			ModelConstants.DATASET_RESULTTYPE_CLASSID,
-			MergeUtils
-				.mergeResult(p2, d1)
+			OafMapperUtils
+				.mergeResults(p2, d1)
 				.getResulttype()
 				.getClassid());
 	}
@@ -192,7 +192,7 @@ class OafMapperUtilsTest {
 		assertEquals(1, d2.getCollectedfrom().size());
 		assertTrue(cfId(d2.getCollectedfrom()).contains(ModelConstants.ZENODO_OD_ID));
 
-		Result res = MergeUtils.mergeResult(d1, d2);
+		Result res = OafMapperUtils.mergeResults(d1, d2);
 
 		assertEquals(d2, res);
 

dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java

@@ -1,9 +1,10 @@
 
-package eu.dnetlib.dhp.enrich.orcid;
+package eu.dnetlib.oa.merge;
+
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -13,9 +14,10 @@ import org.junit.platform.commons.util.StringUtils;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import eu.dnetlib.dhp.oa.merge.AuthorMerger;
 import eu.dnetlib.dhp.schema.oaf.Author;
 
-public class ORCIDAuthorEnricherTest {
+public class AuthorMergerTest {
 
 	@Test
 	public void testEnrcichAuthor() throws Exception {
@@ -24,13 +26,12 @@ public class ORCIDAuthorEnricherTest {
 		BufferedReader pr = new BufferedReader(new InputStreamReader(
 			Objects
 				.requireNonNull(
-					ORCIDAuthorEnricherTest.class
-						.getResourceAsStream("/eu/dnetlib/dhp/enrich/orcid/authors_publication_sample.json"))));
+					AuthorMergerTest.class
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json"))));
 		BufferedReader or = new BufferedReader(new InputStreamReader(
 			Objects
 				.requireNonNull(
-					ORCIDAuthorEnricherTest.class
-						.getResourceAsStream("/eu/dnetlib/dhp/enrich/orcid/authors_orcid_sample.json"))));
+					AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json"))));
 
 		TypeReference<List<Author>> aclass = new TypeReference<List<Author>>() {
 		};
@@ -66,8 +67,7 @@ public class ORCIDAuthorEnricherTest {
 				long start = System.currentTimeMillis();
 
 //                final List<Author> enrichedList = AuthorMerger.enrichOrcid(publicationAuthors, orcidAuthors);
-				final List<Author> enrichedList = Collections.emptyList(); // SparkEnrichGraphWithOrcidAuthors.enrichOrcid(publicationAuthors,
-																			// orcidAuthors);
+				final List<Author> enrichedList = AuthorMerger.enrichOrcid(publicationAuthors, orcidAuthors);
 
 				long enrichedAuthorWithPid = enrichedList
 					.stream()
@@ -91,4 +91,24 @@ public class ORCIDAuthorEnricherTest {
 		}
 	}
 
+	@Test
+	public void checkSimilarityTest() {
+		final Author left = new Author();
+		left.setName("Anand");
+		left.setSurname("Rachna");
+		left.setFullname("Anand, Rachna");
+
+		System.out.println(AuthorMerger.normalizeFullName(left.getFullname()));
+
+		final Author right = new Author();
+		right.setName("Rachna");
+		right.setSurname("Anand");
+		right.setFullname("Rachna, Anand");
+//        System.out.println(AuthorMerger.normalize(right.getFullname()));
+		boolean same = AuthorMerger.checkORCIDSimilarity(left, right);
+
+		assertTrue(same);
+
+	}
+
 }

dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/InstanceTypeMatch.java

@@ -23,18 +23,15 @@ public class InstanceTypeMatch extends AbstractListComparator {
 
 		// jolly types
 		translationMap.put("Conference object", "*");
-		translationMap.put("Research", "*");
 		translationMap.put("Other literature type", "*");
 		translationMap.put("Unknown", "*");
 		translationMap.put("UNKNOWN", "*");
 
 		// article types
 		translationMap.put("Article", "Article");
-		translationMap.put("Journal", "Article");
 		translationMap.put("Data Paper", "Article");
 		translationMap.put("Software Paper", "Article");
 		translationMap.put("Preprint", "Article");
-		translationMap.put("Part of book or chapter of book", "Article");
 
 		// thesis types
 		translationMap.put("Thesis", "Thesis");

dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/promote/MergeAndGet.java

@@ -1,13 +1,14 @@
 
 package eu.dnetlib.dhp.actionmanager.promote;
 
-import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.SerializableSupplier;
-import eu.dnetlib.dhp.schema.oaf.Oaf;
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
+import static eu.dnetlib.dhp.schema.common.ModelSupport.isSubClass;
 
 import java.util.function.BiFunction;
 
-import static eu.dnetlib.dhp.schema.common.ModelSupport.isSubClass;
+import eu.dnetlib.dhp.common.FunctionalInterfaceSupport.SerializableSupplier;
+import eu.dnetlib.dhp.schema.oaf.Oaf;
+import eu.dnetlib.dhp.schema.oaf.OafEntity;
+import eu.dnetlib.dhp.schema.oaf.Relation;
 
 /** OAF model merging support. */
 public class MergeAndGet {
@@ -45,7 +46,20 @@ public class MergeAndGet {
 	}
 
 	private static <G extends Oaf, A extends Oaf> G mergeFromAndGet(G x, A y) {
-		return (G) MergeUtils.merge(x, y);
+		if (isSubClass(x, Relation.class) && isSubClass(y, Relation.class)) {
+			((Relation) x).mergeFrom((Relation) y);
+			return x;
+		} else if (isSubClass(x, OafEntity.class)
+			&& isSubClass(y, OafEntity.class)
+			&& isSubClass(x, y)) {
+			((OafEntity) x).mergeFrom((OafEntity) y);
+			return x;
+		}
+		throw new RuntimeException(
+			String
+				.format(
+					"MERGE_FROM_AND_GET incompatible types: %s, %s",
+					x.getClass().getCanonicalName(), y.getClass().getCanonicalName()));
 	}
 
 	@SuppressWarnings("unchecked")

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java

@@ -64,9 +64,6 @@ public class PrepareAffiliationRelations implements Serializable {
 		final String pubmedInputPath = parser.get("pubmedInputPath");
 		log.info("pubmedInputPath: {}", pubmedInputPath);
 
-		final String openapcInputPath = parser.get("openapcInputPath");
-		log.info("openapcInputPath: {}", openapcInputPath);
-
 		final String outputPath = parser.get("outputPath");
 		log.info("outputPath: {}", outputPath);
 
@@ -88,14 +85,8 @@ public class PrepareAffiliationRelations implements Serializable {
 				JavaPairRDD<Text, Text> pubmedRelations = prepareAffiliationRelations(
 					spark, pubmedInputPath, collectedFromPubmed);
 
-				List<KeyValue> collectedFromOpenAPC = OafMapperUtils
-					.listKeyValues(ModelConstants.OPEN_APC_ID, "OpenAPC");
-				JavaPairRDD<Text, Text> openAPCRelations = prepareAffiliationRelations(
-					spark, openapcInputPath, collectedFromOpenAPC);
-
 				crossrefRelations
 					.union(pubmedRelations)
-					.union(openAPCRelations)
 					.saveAsHadoopFile(
 						outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class);
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java

@@ -95,7 +95,7 @@ public class SparkAtomicActionScoreJob implements Serializable {
 
 		return projectScores.map((MapFunction<BipProjectModel, Project>) bipProjectScores -> {
 			Project project = new Project();
-			//project.setId(bipProjectScores.getProjectId());
+			project.setId(bipProjectScores.getProjectId());
 			project.setMeasures(bipProjectScores.toMeasures());
 			return project;
 		}, Encoders.bean(Project.class))

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipmodel/score/deserializers/BipProjectModel.java

@@ -34,11 +34,6 @@ public class BipProjectModel {
 
 	String totalCitationCount;
 
-	public String getProjectId() {
-		return projectId;
-	}
-
-
 	// each project bip measure has exactly one value, hence one key-value pair
 	private Measure createMeasure(String measureId, String measureValue) {
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java

@@ -1,20 +1,12 @@
 
 package eu.dnetlib.dhp.actionmanager.project;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import eu.dnetlib.dhp.actionmanager.project.utils.model.CSVProgramme;
-import eu.dnetlib.dhp.actionmanager.project.utils.model.CSVProject;
-import eu.dnetlib.dhp.actionmanager.project.utils.model.JsonTopic;
-import eu.dnetlib.dhp.application.ArgumentApplicationParser;
-import eu.dnetlib.dhp.common.HdfsSupport;
-import eu.dnetlib.dhp.schema.action.AtomicAction;
-import eu.dnetlib.dhp.schema.common.ModelSupport;
-import eu.dnetlib.dhp.schema.oaf.H2020Classification;
-import eu.dnetlib.dhp.schema.oaf.H2020Programme;
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-import eu.dnetlib.dhp.schema.oaf.Project;
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
-import eu.dnetlib.dhp.utils.DHPUtils;
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@@ -26,14 +18,24 @@ import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SparkSession;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.actionmanager.project.utils.model.CSVProgramme;
+import eu.dnetlib.dhp.actionmanager.project.utils.model.CSVProject;
+import eu.dnetlib.dhp.actionmanager.project.utils.model.EXCELTopic;
+import eu.dnetlib.dhp.actionmanager.project.utils.model.JsonTopic;
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.H2020Classification;
+import eu.dnetlib.dhp.schema.oaf.H2020Programme;
+import eu.dnetlib.dhp.schema.oaf.OafEntity;
+import eu.dnetlib.dhp.schema.oaf.Project;
+import eu.dnetlib.dhp.utils.DHPUtils;
 import scala.Tuple2;
 
-import java.util.Arrays;
-import java.util.Objects;
-import java.util.Optional;
-
-import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
-
 /**
  * Class that makes the ActionSet. To prepare the AS two joins are needed
  *
@@ -158,11 +160,9 @@ public class SparkAtomicActionJob {
 				(MapFunction<Project, String>) OafEntity::getId,
 				Encoders.STRING())
 			.mapGroups((MapGroupsFunction<String, Project, Project>) (s, it) -> {
-				Project merge = it.next();
-				while (it.hasNext()) {
-					merge = MergeUtils.mergeProject(merge, it.next());
-				}
-				return merge;
+				Project first = it.next();
+				it.forEachRemaining(first::mergeFrom);
+				return first;
 			}, Encoders.bean(Project.class))
 			.toJavaRDD()
 			.map(p -> new AtomicAction(Project.class, p))

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/transformativeagreement/CreateActionSetSparkJob.java

@@ -1,195 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.transformativeagreement;
-
-import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.*;
-
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.GzipCodec;
-import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.function.FilterFunction;
-import org.apache.spark.api.java.function.FlatMapFunction;
-import org.apache.spark.api.java.function.MapFunction;
-import org.apache.spark.sql.Encoders;
-import org.apache.spark.sql.SparkSession;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.dhp.actionmanager.transformativeagreement.model.TransformativeAgreementModel;
-import eu.dnetlib.dhp.application.ArgumentApplicationParser;
-import eu.dnetlib.dhp.schema.action.AtomicAction;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.Country;
-import eu.dnetlib.dhp.schema.oaf.Relation;
-import eu.dnetlib.dhp.schema.oaf.Result;
-import eu.dnetlib.dhp.schema.oaf.utils.*;
-import scala.Tuple2;
-
-public class CreateActionSetSparkJob implements Serializable {
-
-	private static final Logger log = LoggerFactory.getLogger(CreateActionSetSparkJob.class);
-
-	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
-	private static final String IREL_PROJECT = "40|100018998___::1e5e62235d094afd01cd56e65112fc63";
-	private static final String TRANSFORMATIVE_AGREEMENT = "openapc::transformativeagreement";
-
-	public static void main(final String[] args) throws IOException, ParseException {
-
-		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
-			IOUtils
-				.toString(
-					Objects
-						.requireNonNull(
-							CreateActionSetSparkJob.class
-								.getResourceAsStream(
-									"/eu/dnetlib/dhp/actionmanager/transformativeagreement/as_parameters.json"))));
-
-		parser.parseArgument(args);
-
-		Boolean isSparkSessionManaged = Optional
-			.ofNullable(parser.get("isSparkSessionManaged"))
-			.map(Boolean::valueOf)
-			.orElse(Boolean.TRUE);
-
-		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
-
-		final String inputPath = parser.get("inputPath");
-		log.info("inputPath {}", inputPath);
-
-		final String outputPath = parser.get("outputPath");
-		log.info("outputPath {}", outputPath);
-
-		SparkConf conf = new SparkConf();
-		runWithSparkSession(
-			conf,
-			isSparkSessionManaged,
-			spark -> createActionSet(spark, inputPath, outputPath));
-
-	}
-
-	private static void createActionSet(SparkSession spark, String inputPath, String outputPath) {
-		JavaRDD<AtomicAction> relations = spark
-				.read()
-				.textFile(inputPath)
-				.map(
-						(MapFunction<String, TransformativeAgreementModel>) value -> OBJECT_MAPPER
-								.readValue(value, TransformativeAgreementModel.class),
-						Encoders.bean(TransformativeAgreementModel.class))
-				.flatMap(
-						(FlatMapFunction<TransformativeAgreementModel, Relation>) value -> createRelation(
-								value)
-								.iterator(),
-						Encoders.bean(Relation.class))
-				.filter((FilterFunction<Relation>) Objects::nonNull)
-				.toJavaRDD()
-				.map(p -> new AtomicAction(p.getClass(), p));
-//TODO relations in stand-by waiting to know if we need to create them or not In case we need just make a union before saving the sequence file
-				spark
-					.read()
-					.textFile(inputPath)
-					.map(
-						(MapFunction<String, TransformativeAgreementModel>) value -> OBJECT_MAPPER
-							.readValue(value, TransformativeAgreementModel.class),
-						Encoders.bean(TransformativeAgreementModel.class))
-					.map(
-						(MapFunction<TransformativeAgreementModel, Result>) value -> createResult(
-							value),
-						Encoders.bean(Result.class))
-					.filter((FilterFunction<Result>) r -> r != null)
-					.toJavaRDD()
-					.map(p -> new AtomicAction(p.getClass(), p))
-			.mapToPair(
-				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
-					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
-			.saveAsHadoopFile(
-				outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class);
-
-	}
-
-	private static Result createResult(TransformativeAgreementModel value) {
-		Result r = new Result();
-		r
-			.setId(
-				"50|doi_________::"
-					+ IdentifierFactory
-						.md5(PidCleaner.normalizePidValue(PidType.doi.toString(), value.getDoi())));
-		r.setTransformativeAgreement(value.getAgreement());
-		Country country = new Country();
-		country.setClassid(value.getCountry());
-		country.setClassname(value.getCountry());
-		country
-			.setDataInfo(
-				OafMapperUtils
-					.dataInfo(
-						false, ModelConstants.SYSIMPORT_ACTIONSET, false, false,
-						OafMapperUtils
-							.qualifier(
-								"openapc::transformativeagreement",
-								"Harvested from Trnasformative Agreement file from OpenAPC",
-								ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS),
-						"0.9"));
-		country.setSchemeid(ModelConstants.DNET_COUNTRY_TYPE);
-		country.setSchemename(ModelConstants.DNET_COUNTRY_TYPE);
-		r.setCountry(Arrays.asList(country));
-		return r;
-	}
-
-	private static List<Relation> createRelation(TransformativeAgreementModel value) {
-
-		List<Relation> relationList = new ArrayList<>();
-
-		if (value.getAgreement().startsWith("IReL")) {
-			String paper;
-
-			paper = "50|doi_________::"
-				+ IdentifierFactory
-					.md5(PidCleaner.normalizePidValue(PidType.doi.toString(), value.getDoi()));
-
-			relationList
-				.add(
-					getRelation(
-						paper,
-						IREL_PROJECT, ModelConstants.IS_PRODUCED_BY));
-
-			relationList.add(getRelation(IREL_PROJECT, paper, ModelConstants.PRODUCES));
-		}
-		return relationList;
-	}
-
-	public static Relation getRelation(
-		String source,
-		String target,
-		String relClass) {
-
-		return OafMapperUtils
-			.getRelation(
-				source,
-				target,
-				ModelConstants.RESULT_PROJECT,
-				ModelConstants.OUTCOME,
-				relClass,
-				Arrays
-					.asList(
-						OafMapperUtils.keyValue(ModelConstants.OPEN_APC_ID, ModelConstants.OPEN_APC_NAME)),
-				OafMapperUtils
-					.dataInfo(
-						false, null, false, false,
-						OafMapperUtils
-							.qualifier(
-								TRANSFORMATIVE_AGREEMENT, "Transformative Agreement",
-								ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS),
-						"0.9"),
-				null);
-	}
-
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/transformativeagreement/model/TransformativeAgreementModel.java

@@ -1,51 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.transformativeagreement.model;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-/**
- * @author miriam.baglioni
- * @Date 18/12/23
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-
-public class TransformativeAgreementModel implements Serializable {
-	private String institution;
-	private String doi;
-	private String agreement;
-	private String country;
-
-	public String getCountry() {
-		return country;
-	}
-
-	public void setCountry(String country) {
-		this.country = country;
-	}
-
-	public String getInstitution() {
-		return institution;
-	}
-
-	public void setInstitution(String institution) {
-		this.institution = institution;
-	}
-
-	public String getDoi() {
-		return doi;
-	}
-
-	public void setDoi(String doi) {
-		this.doi = doi;
-	}
-
-	public String getAgreement() {
-		return agreement;
-	}
-
-	public void setAgreement(String agreement) {
-		this.agreement = agreement;
-	}
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ORCIDWorker.java

@@ -1,244 +0,0 @@
-
-package eu.dnetlib.dhp.collection.orcid;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.concurrent.BlockingQueue;
-
-import javax.swing.*;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.http.HttpHeaders;
-import org.jetbrains.annotations.NotNull;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import eu.dnetlib.dhp.common.collection.HttpClientParams;
-
-public class ORCIDWorker extends Thread {
-
-	final static Logger log = LoggerFactory.getLogger(ORCIDWorker.class);
-
-	public static String JOB_COMPLETE = "JOB_COMPLETE";
-
-	private static final String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)";
-
-	private final BlockingQueue<String> queue;
-
-	private boolean hasComplete = false;
-
-	private final SequenceFile.Writer employments;
-
-	private final SequenceFile.Writer summary;
-	private final SequenceFile.Writer works;
-
-	private final String token;
-
-	private final String id;
-
-	public static ORCIDWorkerBuilder builder() {
-		return new ORCIDWorkerBuilder();
-	}
-
-	public ORCIDWorker(String id, BlockingQueue<String> myqueue, SequenceFile.Writer employments,
-		SequenceFile.Writer summary, SequenceFile.Writer works, String token) {
-		this.id = id;
-		this.queue = myqueue;
-		this.employments = employments;
-		this.summary = summary;
-		this.works = works;
-		this.token = token;
-	}
-
-	public static String retrieveURL(final String id, final String apiUrl, String token) {
-		try {
-			final HttpURLConnection urlConn = getHttpURLConnection(apiUrl, token);
-			if (urlConn.getResponseCode() > 199 && urlConn.getResponseCode() < 300) {
-				InputStream input = urlConn.getInputStream();
-				return IOUtils.toString(input);
-			} else {
-				log
-					.error(
-						"Thread {} UNABLE TO DOWNLOAD FROM THIS URL {} , status code {}", id, apiUrl,
-						urlConn.getResponseCode());
-			}
-		} catch (Exception e) {
-			log.error("Thread {}  Error on retrieving URL {} {}", id, apiUrl, e);
-		}
-		return null;
-	}
-
-	@NotNull
-	private static HttpURLConnection getHttpURLConnection(String apiUrl, String token) throws IOException {
-		final HttpURLConnection urlConn = (HttpURLConnection) new URL(apiUrl).openConnection();
-		final HttpClientParams clientParams = new HttpClientParams();
-		urlConn.setInstanceFollowRedirects(false);
-		urlConn.setReadTimeout(clientParams.getReadTimeOut() * 1000);
-		urlConn.setConnectTimeout(clientParams.getConnectTimeOut() * 1000);
-		urlConn.addRequestProperty(HttpHeaders.USER_AGENT, userAgent);
-		urlConn.addRequestProperty(HttpHeaders.AUTHORIZATION, String.format("Bearer %s", token));
-		return urlConn;
-	}
-
-	private static String generateSummaryURL(final String orcidId) {
-		return "https://api.orcid.org/v3.0/" + orcidId + "/record";
-	}
-
-	private static String generateWorksURL(final String orcidId) {
-		return "https://api.orcid.org/v3.0/" + orcidId + "/works";
-	}
-
-	private static String generateEmploymentsURL(final String orcidId) {
-		return "https://api.orcid.org/v3.0/" + orcidId + "/employments";
-	}
-
-	private static void writeResultToSequenceFile(String id, String url, String token, String orcidId,
-		SequenceFile.Writer file) throws IOException {
-		final String response = retrieveURL(id, url, token);
-		if (response != null) {
-			if (orcidId == null) {
-				log.error("Thread {}   {}   {}", id, orcidId, response);
-				throw new RuntimeException("null items ");
-			}
-
-			if (file == null) {
-				log.error("Thread {}   file is null for {}  URL:{}", id, url, orcidId);
-			} else {
-				file.append(new Text(orcidId), new Text(response));
-				file.hflush();
-			}
-
-		} else
-			log.error("Thread {} response is null for {}  URL:{}", id, url, orcidId);
-
-	}
-
-	@Override
-	public void run() {
-		final Text key = new Text();
-		final Text value = new Text();
-		long start;
-		long total_time;
-		String orcidId = "";
-		int requests = 0;
-		if (summary == null || employments == null || works == null)
-			throw new RuntimeException("Null files");
-
-		while (!hasComplete) {
-			try {
-
-				orcidId = queue.take();
-
-				if (orcidId.equalsIgnoreCase(JOB_COMPLETE)) {
-					hasComplete = true;
-				} else {
-					start = System.currentTimeMillis();
-					writeResultToSequenceFile(id, generateSummaryURL(orcidId), token, orcidId, summary);
-					total_time = System.currentTimeMillis() - start;
-					requests++;
-					if (total_time < 1000) {
-						// I know making a sleep on a thread is bad, but we need to stay to 24 requests per seconds,
-						// hence
-						// the time between two http request in a thread must be 1 second
-						Thread.sleep(1000L - total_time);
-					}
-					start = System.currentTimeMillis();
-					writeResultToSequenceFile(id, generateWorksURL(orcidId), token, orcidId, works);
-					total_time = System.currentTimeMillis() - start;
-					requests++;
-					if (total_time < 1000) {
-						// I know making a sleep on a thread is bad, but we need to stay to 24 requests per seconds,
-						// hence
-						// the time between two http request in a thread must be 1 second
-						Thread.sleep(1000L - total_time);
-					}
-					start = System.currentTimeMillis();
-					writeResultToSequenceFile(id, generateEmploymentsURL(orcidId), token, orcidId, employments);
-					total_time = System.currentTimeMillis() - start;
-					requests++;
-					if (total_time < 1000) {
-						// I know making a sleep on a thread is bad, but we need to stay to 24 requests per seconds,
-						// hence
-						// the time between two http request in a thread must be 1 second
-						Thread.sleep(1000L - total_time);
-					}
-					if (requests % 30 == 0) {
-						log.info("Thread {}   Downloaded {}", id, requests);
-					}
-				}
-
-			} catch (Throwable e) {
-
-				log.error("Thread {}  Unable to save ORICD: {} item error", id, orcidId, e);
-
-			}
-
-		}
-		try {
-			works.close();
-			summary.close();
-			employments.close();
-		} catch (Throwable e) {
-			throw new RuntimeException(e);
-		}
-
-		log.info("Thread {}  COMPLETE ", id);
-		log.info("Thread {}   Downloaded {}", id, requests);
-
-	}
-
-	public static class ORCIDWorkerBuilder {
-
-		private String id;
-		private SequenceFile.Writer employments;
-		private SequenceFile.Writer summary;
-		private SequenceFile.Writer works;
-		private BlockingQueue<String> queue;
-
-		private String token;
-
-		public ORCIDWorkerBuilder withId(final String id) {
-			this.id = id;
-			return this;
-		}
-
-		public ORCIDWorkerBuilder withEmployments(final SequenceFile.Writer sequenceFile) {
-			this.employments = sequenceFile;
-			return this;
-		}
-
-		public ORCIDWorkerBuilder withSummary(final SequenceFile.Writer sequenceFile) {
-			this.summary = sequenceFile;
-			return this;
-		}
-
-		public ORCIDWorkerBuilder withWorks(final SequenceFile.Writer sequenceFile) {
-			this.works = sequenceFile;
-			return this;
-		}
-
-		public ORCIDWorkerBuilder withAccessToken(final String accessToken) {
-			this.token = accessToken;
-			return this;
-		}
-
-		public ORCIDWorkerBuilder withBlockingQueue(final BlockingQueue<String> queue) {
-			this.queue = queue;
-			return this;
-		}
-
-		public ORCIDWorker build() {
-			if (this.summary == null || this.works == null || this.employments == null || StringUtils.isEmpty(token)
-				|| queue == null)
-				throw new RuntimeException("Unable to build missing required params");
-			return new ORCIDWorker(id, queue, employments, summary, works, token);
-		}
-
-	}
-
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/OrcidGetUpdatesFile.java

@@ -1,171 +0,0 @@
-
-package eu.dnetlib.dhp.collection.orcid;
-
-import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration;
-
-import java.io.*;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.spark.sql.SparkSession;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import eu.dnetlib.dhp.application.ArgumentApplicationParser;
-import eu.dnetlib.dhp.common.collection.HttpClientParams;
-
-public class OrcidGetUpdatesFile {
-
-	private static Logger log = LoggerFactory.getLogger(OrcidGetUpdatesFile.class);
-
-	public static void main(String[] args) throws Exception {
-
-		ArgumentApplicationParser parser = new ArgumentApplicationParser(
-			IOUtils
-				.toString(
-					Objects
-						.requireNonNull(
-							OrcidGetUpdatesFile.class
-								.getResourceAsStream(
-									"/eu/dnetlib/dhp/collection/orcid/download_orcid_update_parameter.json")))
-
-		);
-		parser.parseArgument(args);
-
-		final String namenode = parser.get("namenode");
-		log.info("got variable namenode: {}", namenode);
-
-		final String master = parser.get("master");
-		log.info("got variable master: {}", master);
-
-		final String targetPath = parser.get("targetPath");
-		log.info("got variable targetPath: {}", targetPath);
-
-		final String apiURL = parser.get("apiURL");
-		log.info("got variable apiURL: {}", apiURL);
-
-		final String accessToken = parser.get("accessToken");
-		log.info("got variable accessToken: {}", accessToken);
-
-		final String graphPath = parser.get("graphPath");
-		log.info("got variable graphPath: {}", graphPath);
-
-		final SparkSession spark = SparkSession
-			.builder()
-			.appName(OrcidGetUpdatesFile.class.getName())
-			.master(master)
-			.getOrCreate();
-
-		final String latestDate = spark
-			.read()
-			.load(graphPath + "/Authors")
-			.selectExpr("max(lastModifiedDate)")
-			.first()
-			.getString(0);
-
-		log.info("latest date is {}", latestDate);
-
-		final FileSystem fileSystem = FileSystem.get(getHadoopConfiguration(namenode));
-
-		new OrcidGetUpdatesFile().readTar(fileSystem, accessToken, apiURL, targetPath, latestDate);
-
-	}
-
-	private SequenceFile.Writer createFile(Path aPath, FileSystem fileSystem) throws IOException {
-		return SequenceFile
-			.createWriter(
-				fileSystem.getConf(),
-				SequenceFile.Writer.file(aPath),
-				SequenceFile.Writer.keyClass(Text.class),
-				SequenceFile.Writer.valueClass(Text.class));
-	}
-
-	private ORCIDWorker createWorker(final String id, final String targetPath, final BlockingQueue<String> queue,
-		final String accessToken, FileSystem fileSystem) throws Exception {
-		return ORCIDWorker
-			.builder()
-			.withId(id)
-			.withEmployments(createFile(new Path(String.format("%s/employments_%s", targetPath, id)), fileSystem))
-			.withSummary(createFile(new Path(String.format("%s/summary_%s", targetPath, id)), fileSystem))
-			.withWorks(createFile(new Path(String.format("%s/works_%s", targetPath, id)), fileSystem))
-			.withAccessToken(accessToken)
-			.withBlockingQueue(queue)
-			.build();
-	}
-
-	public void readTar(FileSystem fileSystem, final String accessToken, final String apiURL, final String targetPath,
-		final String startDate) throws Exception {
-		final HttpURLConnection urlConn = (HttpURLConnection) new URL(apiURL).openConnection();
-		final HttpClientParams clientParams = new HttpClientParams();
-		urlConn.setInstanceFollowRedirects(false);
-		urlConn.setReadTimeout(clientParams.getReadTimeOut() * 1000);
-		urlConn.setConnectTimeout(clientParams.getConnectTimeOut() * 1000);
-		if (urlConn.getResponseCode() > 199 && urlConn.getResponseCode() < 300) {
-			InputStream input = urlConn.getInputStream();
-
-			Path hdfsWritePath = new Path("/tmp/orcid_updates.tar.gz");
-			final FSDataOutputStream fsDataOutputStream = fileSystem.create(hdfsWritePath, true);
-			IOUtils.copy(input, fsDataOutputStream);
-			fsDataOutputStream.flush();
-			fsDataOutputStream.close();
-			FSDataInputStream updateFile = fileSystem.open(hdfsWritePath);
-			TarArchiveInputStream tais = new TarArchiveInputStream(new GzipCompressorInputStream(
-				new BufferedInputStream(
-					updateFile.getWrappedStream())));
-			TarArchiveEntry entry;
-
-			BlockingQueue<String> queue = new ArrayBlockingQueue<String>(3000);
-			final List<ORCIDWorker> workers = new ArrayList<>();
-			for (int i = 0; i < 22; i++) {
-				workers.add(createWorker("" + i, targetPath, queue, accessToken, fileSystem));
-			}
-			workers.forEach(Thread::start);
-
-			while ((entry = tais.getNextTarEntry()) != null) {
-
-				if (entry.isFile()) {
-
-					BufferedReader br = new BufferedReader(new InputStreamReader(tais));
-					System.out.println(br.readLine());
-					br
-						.lines()
-						.map(l -> l.split(","))
-						.filter(s -> StringUtils.compare(s[3].substring(0, 10), startDate) > 0)
-						.map(s -> s[0])
-						.forEach(s -> {
-							try {
-								queue.put(s);
-							} catch (InterruptedException e) {
-								throw new RuntimeException(e);
-							}
-						});
-
-				}
-			}
-
-			for (int i = 0; i < 22; i++) {
-				queue.put(ORCIDWorker.JOB_COMPLETE);
-			}
-			for (ORCIDWorker worker : workers) {
-				worker.join();
-			}
-		}
-
-	}
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/OrcidParser.java

@@ -1,15 +1,11 @@
 
 package eu.dnetlib.dhp.collection.orcid;
 
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
-import org.dom4j.Document;
-import org.dom4j.DocumentFactory;
-import org.dom4j.DocumentHelper;
-import org.dom4j.Node;
-import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,8 +40,8 @@ public class OrcidParser {
 	private static final String NS_ERROR = "error";
 	private static final String NS_HISTORY = "history";
 	private static final String NS_HISTORY_URL = "http://www.orcid.org/ns/history";
-	private static final String NS_EMPLOYMENT = "employment";
-	private static final String NS_EMPLOYMENT_URL = "http://www.orcid.org/ns/employment";
+	private static final String NS_BULK_URL = "http://www.orcid.org/ns/bulk";
+	private static final String NS_BULK = "bulk";
 	private static final String NS_EXTERNAL = "external-identifier";
 	private static final String NS_EXTERNAL_URL = "http://www.orcid.org/ns/external-identifier";
 
@@ -65,7 +61,6 @@ public class OrcidParser {
 		ap.declareXPathNameSpace(NS_WORK, NS_WORK_URL);
 		ap.declareXPathNameSpace(NS_EXTERNAL, NS_EXTERNAL_URL);
 		ap.declareXPathNameSpace(NS_ACTIVITIES, NS_ACTIVITIES_URL);
-		ap.declareXPathNameSpace(NS_EMPLOYMENT, NS_EMPLOYMENT_URL);
 	}
 
 	public Author parseSummary(final String xml) {
@@ -75,15 +70,13 @@ public class OrcidParser {
 			generateParsedDocument(xml);
 			List<VtdUtilityParser.Node> recordNodes = VtdUtilityParser
 				.getTextValuesWithAttributes(
-					ap, vn, "//record:record", Collections.singletonList("path"));
+					ap, vn, "//record:record", Arrays.asList("path"));
 			if (!recordNodes.isEmpty()) {
 				final String oid = (recordNodes.get(0).getAttributes().get("path")).substring(1);
 				author.setOrcid(oid);
 			} else {
 				return null;
 			}
-			final String ltm = VtdUtilityParser.getSingleValue(ap, vn, "//common:last-modified-date");
-			author.setLastModifiedDate(ltm);
 			List<VtdUtilityParser.Node> personNodes = VtdUtilityParser
 				.getTextValuesWithAttributes(
 					ap, vn, "//person:name", Arrays.asList("visibility"));
@@ -136,64 +129,6 @@ public class OrcidParser {
 		}
 	}
 
-	public List<Work> parseWorks(final String xml) {
-
-		try {
-			String oid;
-
-			generateParsedDocument(xml);
-			List<VtdUtilityParser.Node> workNodes = VtdUtilityParser
-				.getTextValuesWithAttributes(ap, vn, "//activities:works", Arrays.asList("path", "visibility"));
-			if (!workNodes.isEmpty()) {
-				oid = (workNodes.get(0).getAttributes().get("path")).split("/")[1];
-
-			} else {
-				return null;
-			}
-			final List<Work> works = new ArrayList<>();
-			ap.selectXPath("//work:work-summary");
-
-			while (ap.evalXPath() != -1) {
-				final Work work = new Work();
-				work.setOrcid(oid);
-				final AutoPilot ap1 = new AutoPilot(ap.getNav());
-				ap1.selectXPath("./work:title/common:title");
-				while (ap1.evalXPath() != -1) {
-					int it = vn.getText();
-					work.setTitle(vn.toNormalizedString(it));
-				}
-				ap1.selectXPath(".//common:external-id");
-				while (ap1.evalXPath() != -1) {
-					final Pid pid = new Pid();
-
-					final AutoPilot ap2 = new AutoPilot(ap1.getNav());
-
-					ap2.selectXPath("./common:external-id-type");
-					while (ap2.evalXPath() != -1) {
-						int it = vn.getText();
-						pid.setSchema(vn.toNormalizedString(it));
-					}
-					ap2.selectXPath("./common:external-id-value");
-					while (ap2.evalXPath() != -1) {
-						int it = vn.getText();
-						pid.setValue(vn.toNormalizedString(it));
-					}
-
-					work.addPid(pid);
-				}
-
-				works.add(work);
-			}
-			return works;
-
-		} catch (Throwable e) {
-			log.error("Error on parsing {}", xml);
-			log.error(e.getMessage());
-			return null;
-		}
-
-	}
-
 	public Work parseWork(final String xml) {
 
 		try {
@@ -241,15 +176,11 @@ public class OrcidParser {
 	}
 
 	private String extractEmploymentDate(final String xpath) throws Exception {
-		return extractEmploymentDate(xpath, ap);
-	}
 
-	private String extractEmploymentDate(final String xpath, AutoPilot pp) throws Exception {
-
-		pp.selectXPath(xpath);
+		ap.selectXPath(xpath);
 		StringBuilder sb = new StringBuilder();
-		while (pp.evalXPath() != -1) {
-			final AutoPilot ap1 = new AutoPilot(pp.getNav());
+		while (ap.evalXPath() != -1) {
+			final AutoPilot ap1 = new AutoPilot(ap.getNav());
 			ap1.selectXPath("./common:year");
 			while (ap1.evalXPath() != -1) {
 				int it = vn.getText();
@@ -272,104 +203,6 @@ public class OrcidParser {
 
 	}
 
-	public List<Employment> parseEmployments(final String xml) {
-		try {
-			String oid;
-			Map<String, String> nsContext = getNameSpaceMap();
-			DocumentFactory.getInstance().setXPathNamespaceURIs(nsContext);
-			Document doc = DocumentHelper.parseText(xml);
-			oid = doc.valueOf("//activities:employments/@path");
-			if (oid == null || StringUtils.isEmpty(oid))
-				return null;
-			final String orcid = oid.split("/")[1];
-
-			List<Node> nodes = doc.selectNodes("//employment:employment-summary");
-			return nodes.stream().map(n -> {
-				final Employment e = new Employment();
-				e.setOrcid(orcid);
-
-				final String depName = n.valueOf(".//common:department-name");
-				if (StringUtils.isNotBlank(depName))
-					e.setDepartmentName(depName);
-				final String roleTitle = n.valueOf(".//common:role-title");
-				e.setRoleTitle(roleTitle);
-				final String organizationName = n.valueOf(".//common:organization/common:name");
-				if (StringUtils.isEmpty(e.getDepartmentName()))
-					e.setDepartmentName(organizationName);
-				final Pid p = new Pid();
-				final String pid = n
-					.valueOf(
-						"./common:organization/common:disambiguated-organization/common:disambiguated-organization-identifier");
-				p.setValue(pid);
-				final String pidType = n
-					.valueOf("./common:organization/common:disambiguated-organization/common:disambiguation-source");
-				p.setSchema(pidType);
-				e.setAffiliationId(p);
-
-				final StringBuilder aDate = new StringBuilder();
-				final String sy = n.valueOf("./common:start-date/common:year");
-				if (StringUtils.isNotBlank(sy)) {
-					aDate.append(sy);
-					final String sm = n.valueOf("./common:start-date/common:month");
-					final String sd = n.valueOf("./common:start-date/common:day");
-					aDate.append("-");
-					if (StringUtils.isNotBlank(sm))
-						aDate.append(sm);
-					else
-						aDate.append("01");
-					aDate.append("-");
-					if (StringUtils.isNotBlank(sd))
-						aDate.append(sd);
-					else
-						aDate.append("01");
-					e.setEndDate(aDate.toString());
-				}
-
-				final String ey = n.valueOf("./common:end-date/common:year");
-				if (StringUtils.isNotBlank(ey)) {
-					aDate.append(ey);
-					final String em = n.valueOf("./common:end-date/common:month");
-					final String ed = n.valueOf("./common:end-date/common:day");
-					aDate.append("-");
-					if (StringUtils.isNotBlank(em))
-						aDate.append(em);
-					else
-						aDate.append("01");
-					aDate.append("-");
-					if (StringUtils.isNotBlank(ed))
-						aDate.append(ed);
-					else
-						aDate.append("01");
-					e.setEndDate(aDate.toString());
-				}
-
-				return e;
-
-			}).collect(Collectors.toList());
-		} catch (Throwable e) {
-			log.error("Error on parsing {}", xml);
-			log.error(e.getMessage());
-			return null;
-		}
-	}
-
-	@NotNull
-	private static Map<String, String> getNameSpaceMap() {
-		Map<String, String> nsContext = new HashMap<>();
-		nsContext.put(NS_COMMON, NS_COMMON_URL);
-		nsContext.put(NS_PERSON, NS_PERSON_URL);
-		nsContext.put(NS_DETAILS, NS_DETAILS_URL);
-		nsContext.put(NS_OTHER, NS_OTHER_URL);
-		nsContext.put(NS_RECORD, NS_RECORD_URL);
-		nsContext.put(NS_ERROR, NS_ERROR_URL);
-		nsContext.put(NS_HISTORY, NS_HISTORY_URL);
-		nsContext.put(NS_WORK, NS_WORK_URL);
-		nsContext.put(NS_EXTERNAL, NS_EXTERNAL_URL);
-		nsContext.put(NS_ACTIVITIES, NS_ACTIVITIES_URL);
-		nsContext.put(NS_EMPLOYMENT, NS_EMPLOYMENT_URL);
-		return nsContext;
-	}
-
 	public Employment parseEmployment(final String xml) {
 		try {
 			final Employment employment = new Employment();

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Author.java

@@ -18,8 +18,6 @@ public class Author extends ORCIDItem {
 
 	private String biography;
 
-	private String lastModifiedDate;
-
 	public String getBiography() {
 		return biography;
 	}
@@ -76,14 +74,6 @@ public class Author extends ORCIDItem {
 		this.otherPids = otherPids;
 	}
 
-	public String getLastModifiedDate() {
-		return lastModifiedDate;
-	}
-
-	public void setLastModifiedDate(String lastModifiedDate) {
-		this.lastModifiedDate = lastModifiedDate;
-	}
-
 	public void addOtherPid(final Pid pid) {
 
 		if (otherPids == null)

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java

@@ -52,6 +52,8 @@ public class RestIterator implements Iterator<String> {
 
 	private final String BASIC = "basic";
 
+	private final JsonUtils jsonUtils;
+
 	private final String baseUrl;
 	private final String resumptionType;
 	private final String resumptionParam;
@@ -104,6 +106,7 @@ public class RestIterator implements Iterator<String> {
 		final String resultOutputFormat) {
 
 		this.clientParams = clientParams;
+		this.jsonUtils = new JsonUtils();
 		this.baseUrl = baseUrl;
 		this.resumptionType = resumptionType;
 		this.resumptionParam = resumptionParam;
@@ -123,7 +126,6 @@ public class RestIterator implements Iterator<String> {
 		} catch (Exception e) {
 			throw new IllegalStateException("xml transformation init failed: " + e.getMessage());
 		}
-
 		initQueue();
 	}
 
@@ -188,7 +190,7 @@ public class RestIterator implements Iterator<String> {
 		String resultJson;
 		String resultXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
 		String nextQuery = "";
-		String emptyXml = resultXml + "<" + JsonUtils.XML_WRAP_TAG + "></" + JsonUtils.XML_WRAP_TAG + ">";
+		String emptyXml = resultXml + "<" + JsonUtils.wrapName + "></" + JsonUtils.wrapName + ">";
 		Node resultNode = null;
 		NodeList nodeList = null;
 		String qUrlArgument = "";
@@ -229,7 +231,7 @@ public class RestIterator implements Iterator<String> {
 			resultStream = theHttpInputStream;
 			if ("json".equals(resultOutputFormat)) {
 				resultJson = IOUtils.toString(resultStream, StandardCharsets.UTF_8);
-				resultXml = JsonUtils.convertToXML(resultJson);
+				resultXml = jsonUtils.convertToXML(resultJson);
 				resultStream = IOUtils.toInputStream(resultXml, UTF_8);
 			}
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/JsonUtils.java

@@ -3,142 +3,82 @@ package eu.dnetlib.dhp.collection.plugin.utils;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.json.JSONArray;
-import org.json.JSONObject;
 
 public class JsonUtils {
-	public static final String XML_WRAP_TAG = "recordWrap";
-	private static final String XML_HEADER = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
-	private static final String INVALID_XMLTAG_CHARS = "!\"#$%&'()*+,/;<=>?@[\\]^`{|}~,";
 
 	private static final Log log = LogFactory.getLog(JsonUtils.class);
 
+	public static final String wrapName = "recordWrap";
+
 	/**
-	 * cleanup in JSON-KeyName
+	 * convert in JSON-KeyName 'whitespace(s)' to '_' and '/' to '_', '(' and ')' to ''
 	 * check W3C XML syntax: https://www.w3.org/TR/2006/REC-xml11-20060816/#sec-starttags for valid tag names
 	 * and work-around for the JSON to XML converting of org.json.XML-package.
 	 *
-	 * @param input
-	 * @return converted json object
+	 * known bugs:     doesn't prevent     "key name":" ["sexy name",": penari","erotic dance"],
+	 *
+	 * @param jsonInput
+	 * @return convertedJsonKeynameOutput
 	 */
-	public static JSONObject cleanJsonObject(final JSONObject input) {
-		if (null == input) {
-			return null;
+	public String syntaxConvertJsonKeyNames(String jsonInput) {
+
+		log.trace("before convertJsonKeyNames: " + jsonInput);
+		// pre-clean json - rid spaces of element names (misinterpreted as elements with attributes in xml)
+		// replace ' 's in JSON Namens with '_'
+		while (jsonInput.matches(".*\"([^\"]*)\\s+([^\"]*)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([^\"]*)\\s+([^\"]*)\":", "\"$1_$2\":");
 		}
 
-		JSONObject result = new JSONObject();
-
-		for (String key : input.keySet()) {
-			Object value = input.opt(key);
-			if (value != null) {
-				result.put(cleanKey(key), cleanValue(value));
-			}
+		// replace forward-slash (sign '/' ) in JSON Names with '_'
+		while (jsonInput.matches(".*\"([^\"]*)/([^\"]*)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([^\"]*)/([^\"]*)\":", "\"$1_$2\":");
 		}
 
-		return result;
+		// replace '(' in JSON Names with ''
+		while (jsonInput.matches(".*\"([^\"]*)[(]([^\"]*)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([^\"]*)[(]([^\"]*)\":", "\"$1$2\":");
+		}
+
+		// replace ')' in JSON Names with ''
+		while (jsonInput.matches(".*\"([^\"]*)[)]([^\"]*)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([^\"]*)[)]([^\"]*)\":", "\"$1$2\":");
+		}
+
+		// add prefix of startNumbers in JSON Keynames with 'n_'
+		while (jsonInput.matches(".*\"([^\"][0-9])([^\"]*)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([^\"][0-9])([^\"]*)\":", "\"n_$1$2\":");
+		}
+		// add prefix of only numbers in JSON Keynames with 'm_'
+		while (jsonInput.matches(".*\"([0-9]+)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([0-9]+)\":", "\"m_$1\":");
+		}
+
+		// replace ':' between number like '2018-08-28T11:05:00Z' in JSON keynames with ''
+		while (jsonInput.matches(".*\"([^\"]*[0-9]):([0-9][^\"]*)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([^\"]*[0-9]):([0-9][^\"]*)\":", "\"$1$2\":");
+		}
+
+		// replace ',' in JSON Keynames with '.' to prevent , in xml tagnames.
+		// while (jsonInput.matches(".*\"([^\"]*),([^\"]*)\":.*")) {
+		// jsonInput = jsonInput.replaceAll("\"([^\"]*),([^\"]*)\":", "\"$1.$2\":");
+		// }
+
+		// replace '=' in JSON Keynames with '-'
+		while (jsonInput.matches(".*\"([^\"]*)=([^\"]*)\":.*")) {
+			jsonInput = jsonInput.replaceAll("\"([^\"]*)=([^\"]*)\":", "\"$1-$2\":");
+		}
+
+		log.trace("after syntaxConvertJsonKeyNames: " + jsonInput);
+		return jsonInput;
 	}
 
-	private static Object cleanValue(Object object) {
-		if (object instanceof JSONObject) {
-			return cleanJsonObject((JSONObject) object);
-		} else if (object instanceof JSONArray) {
-			JSONArray array = (JSONArray) object;
-			JSONArray res = new JSONArray();
-
-			for (int i = array.length() - 1; i >= 0; i--) {
-				res.put(i, cleanValue(array.opt(i)));
-			}
-			return res;
-		} else if (object instanceof String) {
-			String value = (String) object;
-
-			// XML 1.0 Allowed characters
-			// Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
-
-			return value
-				.codePoints()
-				.filter(
-					cp -> cp == 0x9 || cp == 0xA || cp == 0xD || (cp >= 0x20 && cp <= 0xD7FF)
-						|| (cp >= 0xE000 && cp <= 0xFFFD)
-						|| (cp >= 0x10000 && cp <= 0x10FFFF))
-				.collect(
-					StringBuilder::new,
-					StringBuilder::appendCodePoint,
-					StringBuilder::append)
-				.toString();
-		}
-
-		return object;
-	}
-
-	private static String cleanKey(String key) {
-		if (key == null || key.isEmpty()) {
-			return key;
-		}
-
-		// xml tag cannot begin with "-", ".", or a numeric digit.
-		switch (key.charAt(0)) {
-			case '-':
-			case '.':
-				key = "_" + key.substring(1);
-				break;
-		}
-
-		if (Character.isDigit(key.charAt(0))) {
-			if (key.matches("^[0-9]+$")) {
-				// add prefix of only numbers in JSON Keynames with 'm_'
-				key = "m_" + key;
-			} else {
-				// add prefix of startNumbers in JSON Keynames with 'n_'
-				key = "n_" + key;
-			}
-		}
-
-		StringBuilder res = new StringBuilder(key.length());
-		for (int i = 0; i < key.length(); i++) {
-			char c = key.charAt(i);
-
-			// sequence of whitespaces are rendered as a single '_'
-			if (Character.isWhitespace(c)) {
-				while (i + 1 < key.length() && Character.isWhitespace(key.charAt(i + 1))) {
-					i++;
-				}
-				res.append('_');
-			}
-			// remove invalid chars for xml tags with the expception of '=' and '/'
-			else if (INVALID_XMLTAG_CHARS.indexOf(c) >= 0) {
-				switch (c) {
-					case '=':
-						res.append('-');
-						break;
-					case '/':
-						res.append('_');
-						break;
-					default:
-						break;
-				}
-				// nothing
-			}
-			// all other chars are kept
-			else {
-				res.append(c);
-			}
-		}
-
-		return res.toString();
-	}
-
-	static public String convertToXML(final String jsonRecord) {
-		if (log.isTraceEnabled()) {
-			log.trace("input json: " + jsonRecord);
-		}
-
-		JSONObject jsonObject = cleanJsonObject(new org.json.JSONObject(jsonRecord));
-		String res = XML_HEADER + org.json.XML.toString(jsonObject, XML_WRAP_TAG); // wrap xml in single root element
-
-		if (log.isTraceEnabled()) {
-			log.trace("outout xml: " + res);
-		}
-		return res;
+	public String convertToXML(final String jsonRecord) {
+		String resultXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+		org.json.JSONObject jsonObject = new org.json.JSONObject(syntaxConvertJsonKeyNames(jsonRecord));
+		resultXml += org.json.XML.toString(jsonObject, wrapName); // wrap xml in single root element
+		log.trace("before inputStream: " + resultXml);
+		resultXml = XmlCleaner.cleanAllEntities(resultXml);
+		log.trace("after cleaning: " + resultXml);
+		return resultXml;
 	}
 }

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json

@@ -17,12 +17,6 @@
     "paramDescription": "the path to get the input data from Pubmed",
     "paramRequired": true
   },
-  {
-    "paramName": "oip",
-    "paramLongName": "openapcInputPath",
-    "paramDescription": "the path to get the input data from OpenAPC",
-    "paramRequired": true
-  },
   {
     "paramName": "o",
     "paramLongName": "outputPath",

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties

@@ -31,7 +31,6 @@ spark2SqlQueryExecutionListeners=com.cloudera.spark.lineage.NavigatorQueryListen
 # The following is needed as a property of a workflow
 oozie.wf.application.path=${oozieTopWfApplicationPath}
 
-crossrefInputPath=/data/bip-affiliations/crossref-data.json
+crossrefInputPath=/data/bip-affiliations/data.json
 pubmedInputPath=/data/bip-affiliations/pubmed-data.json
-openapcInputPath=/data/bip-affiliations/openapc-data.json
 outputPath=/tmp/crossref-affiliations-output-v5

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml

@@ -9,10 +9,6 @@
             <name>pubmedInputPath</name>
             <description>the path where to find the inferred affiliation relations from Pubmed</description>
         </property>
-        <property>
-            <name>openapcInputPath</name>
-            <description>the path where to find the inferred affiliation relations from OpenAPC</description>
-        </property>
         <property>
             <name>outputPath</name>
             <description>the path where to store the actionset</description>
@@ -106,7 +102,6 @@
             </spark-opts>
             <arg>--crossrefInputPath</arg><arg>${crossrefInputPath}</arg>
             <arg>--pubmedInputPath</arg><arg>${pubmedInputPath}</arg>
-            <arg>--openapcInputPath</arg><arg>${openapcInputPath}</arg>
             <arg>--outputPath</arg><arg>${outputPath}</arg>
         </spark>
         <ok to="End"/>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/transformativeagreement/as_parameters.json

@@ -1,20 +0,0 @@
-[
-  {
-    "paramName": "ip",
-    "paramLongName": "inputPath",
-    "paramDescription": "the zipped opencitations file",
-    "paramRequired": true
-  },
-  {
-    "paramName": "op",
-    "paramLongName": "outputPath",
-    "paramDescription": "the working path",
-    "paramRequired": true
-  },
-  {
-    "paramName": "issm",
-    "paramLongName": "isSparkSessionManaged",
-    "paramDescription": "the hdfs name node",
-    "paramRequired": false
-  }
-]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/transformativeagreement/input_read_parameters.json

@@ -1,30 +0,0 @@
-[
-
-  {
-    "paramName": "issm",
-    "paramLongName": "isSparkSessionManaged",
-    "paramDescription": "the hdfs name node",
-    "paramRequired": false
-  },
-  {
-    "paramName": "d",
-    "paramLongName": "delimiter",
-    "paramDescription": "the hdfs name node",
-    "paramRequired": false
-  },
-  {
-    "paramName": "op",
-    "paramLongName": "outputPath",
-    "paramDescription": "the hdfs name node",
-    "paramRequired": true
-  },
-  {
-    "paramName": "if",
-    "paramLongName": "inputFile",
-    "paramDescription": "the hdfs name node",
-    "paramRequired": true
-  }
-]
-
-
-

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/transformativeagreement/oozie_app/config-default.xml

@@ -1,58 +0,0 @@
-<configuration>
-    <property>
-        <name>jobTracker</name>
-        <value>yarnRM</value>
-    </property>
-    <property>
-        <name>nameNode</name>
-        <value>hdfs://nameservice1</value>
-    </property>
-    <property>
-        <name>oozie.use.system.libpath</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>oozie.action.sharelib.for.spark</name>
-        <value>spark2</value>
-    </property>
-    <property>
-        <name>hive_metastore_uris</name>
-        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
-    </property>
-    <property>
-        <name>spark2YarnHistoryServerAddress</name>
-        <value>http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089</value>
-    </property>
-    <property>
-        <name>spark2ExtraListeners</name>
-        <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
-    </property>
-    <property>
-        <name>spark2SqlQueryExecutionListeners</name>
-        <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
-    </property>
-    <property>
-        <name>oozie.launcher.mapreduce.user.classpath.first</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>sparkExecutorNumber</name>
-        <value>4</value>
-    </property>
-    <property>
-        <name>spark2EventLogDir</name>
-        <value>/user/spark/spark2ApplicationHistory</value>
-    </property>
-    <property>
-        <name>sparkDriverMemory</name>
-        <value>15G</value>
-    </property>
-    <property>
-        <name>sparkExecutorMemory</name>
-        <value>6G</value>
-    </property>
-    <property>
-        <name>sparkExecutorCores</name>
-        <value>1</value>
-    </property>
-</configuration>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/transformativeagreement/oozie_app/download.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-curl -L $1  | hdfs dfs -put - $2

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/transformativeagreement/oozie_app/workflow.xml

@@ -1,82 +0,0 @@
-<workflow-app name="Transfomative Agreement Integration" xmlns="uri:oozie:workflow:0.5">
-
-    <global>
-        <job-tracker>${jobTracker}</job-tracker>
-        <name-node>${nameNode}</name-node>
-        <configuration>
-            <property>
-                <name>mapreduce.job.queuename</name>
-                <value>${queueName}</value>
-            </property>
-            <property>
-                <name>oozie.launcher.mapred.job.queue.name</name>
-                <value>${oozieLauncherQueueName}</value>
-            </property>
-            <property>
-                <name>oozie.action.sharelib.for.spark</name>
-                <value>${oozieActionShareLibForSpark2}</value>
-            </property>
-
-        </configuration>
-    </global>
-
-    <start to="resume_from"/>
-
-    <decision name="resume_from">
-        <switch>
-            <case to="download">${wf:conf('resumeFrom') eq 'DownloadDump'}</case>
-            <default to="create_actionset"/> <!-- first action to be done when downloadDump is to be performed -->
-        </switch>
-    </decision>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-    <action name="download">
-        <shell xmlns="uri:oozie:shell-action:0.2">
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <configuration>
-                <property>
-                    <name>mapred.job.queue.name</name>
-                    <value>${queueName}</value>
-                </property>
-            </configuration>
-            <exec>download.sh</exec>
-            <argument>${inputFile}</argument>
-            <argument>${workingDir}/transformativeagreement/transformativeAgreement.json</argument>
-            <env-var>HADOOP_USER_NAME=${wf:user()}</env-var>
-            <file>download.sh</file>
-            <capture-output/>
-        </shell>
-        <ok to="create_actionset"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="create_actionset">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Produces the AS for the Transformative Agreement</name>
-            <class>eu.dnetlib.dhp.actionmanager.transformativeagreement.CreateActionSetSparkJob</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-                --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
-            </spark-opts>
-            <arg>--inputPath</arg><arg>${workingDir}/transformativeagreement/</arg>
-            <arg>--outputPath</arg><arg>${outputPath}</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-    <end name="End"/>
-</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/apply_orcid_table_parameter.json

@@ -1,26 +0,0 @@
-[
-  {
-    "paramName": "m",
-    "paramLongName": "master",
-    "paramDescription": "the master name",
-    "paramRequired": true
-  },
-  {
-    "paramName": "t",
-    "paramLongName": "targetPath",
-    "paramDescription": "the target PATH of the DF tables",
-    "paramRequired": true
-  },
-  {
-    "paramName": "g",
-    "paramLongName": "graphPath",
-    "paramDescription": "the PATH of the current graph path",
-    "paramRequired": true
-  },
-  {
-    "paramName": "u",
-    "paramLongName": "updatePath",
-    "paramDescription": "the PATH of the current graph update path",
-    "paramRequired": true
-  }
-  ]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/download_orcid_update_parameter.json

@@ -1,37 +0,0 @@
-[ {
-    "paramName": "n",
-    "paramLongName": "namenode",
-    "paramDescription": "the Name Node URI",
-    "paramRequired": true
-  },
-  {
-    "paramName": "m",
-    "paramLongName": "master",
-    "paramDescription": "the master name",
-    "paramRequired": true
-  },
-  {
-    "paramName": "t",
-    "paramLongName": "targetPath",
-    "paramDescription": "the target PATH where download the files",
-    "paramRequired": true
-  },
-  {
-    "paramName": "a",
-    "paramLongName": "apiURL",
-    "paramDescription": "the URL  to download the tar file",
-    "paramRequired": true
-  },
-  {
-    "paramName": "g",
-    "paramLongName": "graphPath",
-    "paramDescription": "the path of the input graph",
-    "paramRequired": true
-  },
-  {
-    "paramName": "at",
-    "paramLongName": "accessToken",
-    "paramDescription": "the accessToken to contact API",
-    "paramRequired": true
-  }
-]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/generate_orcid_table_parameter.json

@@ -16,12 +16,6 @@
     "paramLongName": "sourcePath",
     "paramDescription": "the PATH of the ORCID sequence file",
     "paramRequired": true
-  },
-  {
-    "paramName": "fu",
-    "paramLongName": "fromUpdate",
-    "paramDescription": "whether we have to generate table from dump or from update",
-    "paramRequired": false
   }
 
 ]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/update/oozie_app/config-default.xml

@@ -1,23 +0,0 @@
-<configuration>
-    <property>
-        <name>jobTracker</name>
-        <value>yarnRM</value>
-    </property>
-    <property>
-        <name>nameNode</name>
-        <value>hdfs://nameservice1</value>
-    </property>
-    <property>
-        <name>oozie.use.system.libpath</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>oozie.action.sharelib.for.spark</name>
-        <value>spark2</value>
-    </property>
-
-    <property>
-        <name>oozie.launcher.mapreduce.user.classpath.first</name>
-        <value>true</value>
-    </property>
-</configuration>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/update/oozie_app/workflow.xml

@@ -1,114 +0,0 @@
-<workflow-app name="download_Update_ORCID" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>graphPath</name>
-            <description>the  path  to store the original ORCID dump</description>
-        </property>
-        <property>
-            <name>targetPath</name>
-            <description>the  path  to store the original ORCID dump</description>
-        </property>
-        <property>
-            <name>apiURL</name>
-            <value>http://74804fb637bd8e2fba5b-e0a029c2f87486cddec3b416996a6057.r3.cf1.rackcdn.com/last_modified.csv.tar</value>
-            <description>The URL of the update CSV list </description>
-        </property>
-        <property>
-            <name>accessToken</name>
-            <description>The access token</description>
-        </property>
-
-
-    </parameters>
-
-    <start to="startUpdate"/>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-    <action name="startUpdate">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Check Latest Orcid and Download updates</name>
-            <class>eu.dnetlib.dhp.collection.orcid.OrcidGetUpdatesFile</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.executor.memoryOverhead=2g
-                --conf spark.sql.shuffle.partitions=3000
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--master</arg><arg>yarn</arg>
-            <arg>--namenode</arg><arg>${nameNode}</arg>
-            <arg>--graphPath</arg><arg>${graphPath}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}</arg>
-            <arg>--apiURL</arg><arg>${apiURL}</arg>
-            <arg>--accessToken</arg><arg>${accessToken}</arg>
-        </spark>
-        <ok to="generateTables"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="generateTables">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Generate ORCID Tables</name>
-            <class>eu.dnetlib.dhp.collection.orcid.SparkGenerateORCIDTable</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.executor.memoryOverhead=2g
-                --conf spark.sql.shuffle.partitions=3000
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${targetPath}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/updateTable</arg>
-            <arg>--fromUpdate</arg><arg>true</arg>
-            <arg>--master</arg><arg>yarn</arg>
-        </spark>
-        <ok to="updateTable"/>
-        <error to="Kill"/>
-    </action>
-
-    <action name="updateTable">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn</master>
-            <mode>cluster</mode>
-            <name>Update ORCID Tables</name>
-            <class>eu.dnetlib.dhp.collection.orcid.SparkApplyUpdate</class>
-            <jar>dhp-aggregation-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.executor.memoryOverhead=2g
-                --conf spark.sql.shuffle.partitions=3000
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--graphPath</arg><arg>${graphPath}</arg>
-            <arg>--updatePath</arg><arg>${targetPath}/updateTable</arg>
-            <arg>--targetPath</arg><arg>${targetPath}/newTable</arg>
-            <arg>--master</arg><arg>yarn</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-    <end name="End"/>
-</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/sql/base.sql

@@ -108,7 +108,7 @@ INSERT INTO dsm_apiparams(
 	'api_________::openaire____::base_search::dump@@acceptedNormTypes',
 	'api_________::openaire____::base_search::dump',
 	'acceptedNormTypes',
-	'1,11,111,121,14,15,18,181,182,183,1A,6,7'
+	'1,11,111,121,13,14,15,18,181,182,183,1A,6,7'
 );
 
 COMMIT;

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/xml/base2oaf.transformationRule.xml

@@ -59,13 +59,16 @@ base_dc:global_id (I used oai:identifier)
 base_dc:collection/text()
 
 base_dc:continent
-base_dc:country
 base_dc:year (I used dc:date)
 dc:coverage
 dc:language (I used base_dc:lang)
 base_dc:link (I used dc:identifier)
 -->
 
+
+								<xsl:variable name="varBaseNormType" select="vocabulary:clean(//base_dc:typenorm, 'base:normalized_types')" />
+								
+
 								<metadata>
 									<xsl:call-template name="allElements">
 										<xsl:with-param name="sourceElement" select="//dc:title" />
@@ -108,13 +111,9 @@ base_dc:link (I used dc:identifier)
 										<xsl:with-param name="targetElement" select="'dc:format'" />
 									</xsl:call-template>
 									
-									
-									<xsl:for-each select="//base_dc:typenorm">
-										<dc:type>
-											<xsl:value-of select="vocabulary:clean(., 'base:normalized_types')" />
-										</dc:type>
-									</xsl:for-each>
-									
+									<dc:type>
+										<xsl:value-of select="$varBaseNormType" />
+									</dc:type>
 									<xsl:call-template name="allElements">
 										<xsl:with-param name="sourceElement" select="//dc:type" />
 										<xsl:with-param name="targetElement" select="'dc:type'" />
@@ -162,150 +161,16 @@ base_dc:link (I used dc:identifier)
 										</xsl:if>
 									</xsl:for-each>
 
-									<xsl:choose>
-										<!-- I used an inline mapping because the field typenorm could be repeated and I have to specify a list of priority  -->
-										
-										<!-- Book part -->
-										<xsl:when test="//base_dc:typenorm = '111'">
-											<dr:CobjCategory type="publication">0013</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Book -->
-										<xsl:when test="//base_dc:typenorm = '11'">
-											<dr:CobjCategory type="publication">0002</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Article contribution -->
-										<xsl:when test="//base_dc:typenorm = '121'">
-											<dr:CobjCategory type="publication">0001</dr:CobjCategory>
-										</xsl:when>
-										
-																				
-										<!-- Journal/Newspaper -->
-										<xsl:when test="//base_dc:typenorm = '12'">
-											<dr:CobjCategory type="publication">0043</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Report -->
-										<xsl:when test="//base_dc:typenorm = '14'">
-											<dr:CobjCategory type="publication">0017</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Review -->
-										<xsl:when test="//base_dc:typenorm = '15'">
-											<dr:CobjCategory type="publication">0015</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Lecture -->
-										<xsl:when test="//base_dc:typenorm = '17'">
-											<dr:CobjCategory type="publication">0010</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Bachelor's thesis -->
-										<xsl:when test="//base_dc:typenorm = '181'">
-											<dr:CobjCategory type="publication">0008</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Master's thesis -->
-										<xsl:when test="//base_dc:typenorm = '182'">
-											<dr:CobjCategory type="publication">0007</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Doctoral and postdoctoral thesis -->
-										<xsl:when test="//base_dc:typenorm = '183'">
-											<dr:CobjCategory type="publication">0006</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Thesis -->
-										<xsl:when test="//base_dc:typenorm = '18'">
-											<dr:CobjCategory type="publication">0044</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Patent -->
-										<xsl:when test="//base_dc:typenorm = '1A'">
-											<dr:CobjCategory type="publication">0019</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Text -->
-										<xsl:when test="//base_dc:typenorm = '1'">
-											<dr:CobjCategory type="publication">0001</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Software -->
-										<xsl:when test="//base_dc:typenorm = '6'">
-											<dr:CobjCategory type="software">0029</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Dataset -->										
-										<xsl:when test="//base_dc:typenorm = '7'">
-											<dr:CobjCategory type="dataset">0021</dr:CobjCategory>
-										</xsl:when>										
-																				
-										<!-- Still image -->
-										<xsl:when test="//base_dc:typenorm = '51'">
-											<dr:CobjCategory type="other">0025</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Moving image/Video -->										
-										<xsl:when test="//base_dc:typenorm = '52'">
-											<dr:CobjCategory type="other">0024</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Image/Video -->
-										<xsl:when test="//base_dc:typenorm = '5'">
-											<dr:CobjCategory type="other">0033</dr:CobjCategory>
-										</xsl:when>
-
-										<!-- Audio -->
-										<xsl:when test="//base_dc:typenorm = '4'">
-											<dr:CobjCategory type="other">0030</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Musical notation -->
-										<xsl:when test="//base_dc:typenorm = '2'">
-											<dr:CobjCategory type="other">0020</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Map -->
-										<xsl:when test="//base_dc:typenorm = '3'">
-											<dr:CobjCategory type="other">0020</dr:CobjCategory>
-										</xsl:when>										
-										
-										<!-- Other non-article -->
-										<xsl:when test="//base_dc:typenorm = '122'">
-											<dr:CobjCategory type="publication">0038</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Course material -->
-										<xsl:when test="//base_dc:typenorm = '16'">
-											<dr:CobjCategory type="publication">0038</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Manuscript -->
-										<xsl:when test="//base_dc:typenorm = '19'">
-											<dr:CobjCategory type="publication">0038</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Conference object -->
-										<xsl:when test="//base_dc:typenorm = '13'">
-											<dr:CobjCategory type="publication">0004</dr:CobjCategory>
-										</xsl:when>
-
-										<!-- Unknown -->
-										<xsl:when test="//base_dc:typenorm = 'F'">
-											<dr:CobjCategory type="other">0000</dr:CobjCategory>
-										</xsl:when>
-										<xsl:otherwise>
-											<dr:CobjCategory type="other">0000</dr:CobjCategory>
-										</xsl:otherwise>
-									</xsl:choose>
-									
+									<dr:CobjCategory>
+										<xsl:variable name="varCobjCategory" select="vocabulary:clean($varBaseNormType, 'dnet:publication_resource')" />
+										<xsl:variable name="varSuperType" select="vocabulary:clean($varCobjCategory, 'dnet:result_typologies')" />
+										<xsl:attribute name="type" select="$varSuperType" />
+										<xsl:value-of select="$varCobjCategory" />
+									</dr:CobjCategory>
 									
 									<oaf:accessrights>
 										<xsl:choose>
-											<xsl:when test="//base_dc:oa[.='0']">CLOSED</xsl:when>
 											<xsl:when test="//base_dc:oa[.='1']">OPEN</xsl:when>
-											<xsl:when test="//base_dc:oa[.='2']">UNKNOWN</xsl:when>
 											<xsl:when test="//base_dc:rightsnorm">
 												<xsl:value-of select="vocabulary:clean(//base_dc:rightsnorm, 'dnet:access_modes')" />
 											</xsl:when>
@@ -342,7 +207,7 @@ base_dc:link (I used dc:identifier)
 									
 									<oaf:identifier identifierType="oai-original">
 										<xsl:value-of
-											select="//oai:header/oai:identifier" />
+											select="//*[local-name() = 'about']/*[local-name() = 'provenance']//*[local-name() = 'originDescription' and not(./*[local-name() = 'originDescription'])]/*[local-name() = 'identifier']" />
 									</oaf:identifier>
 									
 									<oaf:hostedBy>
@@ -388,8 +253,13 @@ base_dc:link (I used dc:identifier)
 													<xsl:value-of select="concat('ror_________::https://ror.org/', normalize-space(.))" />
 												</xsl:otherwise>
 											</xsl:choose>
-										</oaf:relation>
+										</oaf:relation>										
 									</xsl:for-each>
+
+									<xsl:for-each select="//base_dc:country">
+										<oaf:country><xsl:value-of select="vocabulary:clean(., 'dnet:countries')" /></oaf:country>
+									</xsl:for-each>
+									
 								</metadata>
 								<xsl:copy-of select="//*[local-name() = 'about']" />
 							</record>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/plugin/base/xml/base2odf.transformationRule.xml

@@ -42,7 +42,6 @@
 									base_dc:global_id (I used oai:identifier) 
 									base_dc:collection/text() 
 									base_dc:continent 
-									base_dc:country  
 									dc:coverage
 									dc:source
 									dc:relation
@@ -51,6 +50,10 @@
 									base_dc:link (I used dc:identifier)
 								 -->
 
+
+								<xsl:variable name="varBaseNormType" select="vocabulary:clean(//base_dc:typenorm, 'base:normalized_types')" />
+
+
 								<metadata>
 									<datacite:resource>
 
@@ -82,17 +85,14 @@
 
 											<datacite:identifier alternateIdentifierType="oai-original">
 												<xsl:value-of
-													select="//oai:header/oai:identifier" />
+													select="//*[local-name() = 'about']/*[local-name() = 'provenance']//*[local-name() = 'originDescription' and not(./*[local-name() = 'originDescription'])]/*[local-name() = 'identifier']" />
 											</datacite:identifier>
-											
 										</datacite:alternateIdentifiers>
 
 										<datacite:relatedIdentifiers />
 
 
-										<xsl:for-each select="//base_dc:typenorm">
-											<datacite:resourceType><xsl:value-of select="vocabulary:clean(., 'base:normalized_types')" /></datacite:resourceType>
-										</xsl:for-each>
+										<datacite:resourceType><xsl:value-of select="$varBaseNormType" /></datacite:resourceType>
 
 										<datacite:titles>
 											<xsl:for-each select="//dc:title">
@@ -153,18 +153,14 @@
 											</xsl:for-each>
 										</datacite:subjects>
 										
-										<xsl:for-each select="//dc:publisher">
-											<datacite:publisher>
-												<xsl:value-of select="normalize-space(.)" />
-											</datacite:publisher>
-										</xsl:for-each>
+										<datacite:publisher>
+											<xsl:value-of select="normalize-space(//dc:publisher)" />
+										</datacite:publisher>
+										
+										<datacite:publicationYear>
+											<xsl:value-of select="normalize-space(//base_dc:year)" />
+										</datacite:publicationYear>
 										
-										<xsl:for-each select="//base_dc:year">
-											<datacite:publicationYear>
-												<xsl:value-of select="normalize-space(.)" />
-											</datacite:publicationYear>
-										</xsl:for-each>
-																				
 										<datacite:formats>
 											<xsl:for-each select="//dc:format">
 												<datacite:format>
@@ -178,9 +174,6 @@
 										</datacite:language>
 
 										<oaf:accessrights>
-											<xsl:if test="//base_dc:oa[.='0']">
-												<datacite:rights rightsURI="http://purl.org/coar/access_right/c_16ec">restricted access</datacite:rights>
-											</xsl:if>
 											<xsl:if test="//base_dc:oa[.='1']">
 												<datacite:rights rightsURI="http://purl.org/coar/access_right/c_abf2">open access</datacite:rights>
 											</xsl:if>
@@ -191,162 +184,29 @@
 
 									</datacite:resource>
 
-									<xsl:for-each select="//dc:relation">
-										<xsl:if test="matches(normalize-space(.), '(info:eu-repo/grantagreement/ec/fp7/)(\d\d\d\d\d\d)(.*)', 'i')">
-											<oaf:projectid>
-												<xsl:value-of select="concat($varFP7, replace(normalize-space(.), '(info:eu-repo/grantagreement/ec/fp7/)(\d\d\d\d\d\d)(.*)', '$2', 'i'))" />
-											</oaf:projectid>
-										</xsl:if>
-										<xsl:if test="matches(normalize-space(.), '(info:eu-repo/grantagreement/ec/h2020/)(\d\d\d\d\d\d)(.*)', 'i')">
-											<oaf:projectid>
-												<xsl:value-of select="concat($varH2020, replace(normalize-space(.), '(info:eu-repo/grantagreement/ec/h2020/)(\d\d\d\d\d\d)(.*)', '$2', 'i'))" />
-											</oaf:projectid>
-										</xsl:if>
-									</xsl:for-each>
+										<xsl:for-each select="//dc:relation">
+											<xsl:if test="matches(normalize-space(.), '(info:eu-repo/grantagreement/ec/fp7/)(\d\d\d\d\d\d)(.*)', 'i')">
+												<oaf:projectid>
+													<xsl:value-of select="concat($varFP7, replace(normalize-space(.), '(info:eu-repo/grantagreement/ec/fp7/)(\d\d\d\d\d\d)(.*)', '$2', 'i'))" />
+												</oaf:projectid>
+											</xsl:if>
+											<xsl:if test="matches(normalize-space(.), '(info:eu-repo/grantagreement/ec/h2020/)(\d\d\d\d\d\d)(.*)', 'i')">
+												<oaf:projectid>
+													<xsl:value-of select="concat($varH2020, replace(normalize-space(.), '(info:eu-repo/grantagreement/ec/h2020/)(\d\d\d\d\d\d)(.*)', '$2', 'i'))" />
+												</oaf:projectid>
+											</xsl:if>
+										</xsl:for-each>
 
-									<xsl:choose>
-										<!-- I used an inline mapping because the field typenorm could be repeated and I have to specify a list of priority  -->
-										
-										<!-- Book part -->
-										<xsl:when test="//base_dc:typenorm = '111'">
-											<dr:CobjCategory type="publication">0013</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Book -->
-										<xsl:when test="//base_dc:typenorm = '11'">
-											<dr:CobjCategory type="publication">0002</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Article contribution -->
-										<xsl:when test="//base_dc:typenorm = '121'">
-											<dr:CobjCategory type="publication">0001</dr:CobjCategory>
-										</xsl:when>
-										
-																				
-										<!-- Journal/Newspaper -->
-										<xsl:when test="//base_dc:typenorm = '12'">
-											<dr:CobjCategory type="publication">0043</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Report -->
-										<xsl:when test="//base_dc:typenorm = '14'">
-											<dr:CobjCategory type="publication">0017</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Review -->
-										<xsl:when test="//base_dc:typenorm = '15'">
-											<dr:CobjCategory type="publication">0015</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Lecture -->
-										<xsl:when test="//base_dc:typenorm = '17'">
-											<dr:CobjCategory type="publication">0010</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Bachelor's thesis -->
-										<xsl:when test="//base_dc:typenorm = '181'">
-											<dr:CobjCategory type="publication">0008</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Master's thesis -->
-										<xsl:when test="//base_dc:typenorm = '182'">
-											<dr:CobjCategory type="publication">0007</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Doctoral and postdoctoral thesis -->
-										<xsl:when test="//base_dc:typenorm = '183'">
-											<dr:CobjCategory type="publication">0006</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Thesis -->
-										<xsl:when test="//base_dc:typenorm = '18'">
-											<dr:CobjCategory type="publication">0044</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Patent -->
-										<xsl:when test="//base_dc:typenorm = '1A'">
-											<dr:CobjCategory type="publication">0019</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Text -->
-										<xsl:when test="//base_dc:typenorm = '1'">
-											<dr:CobjCategory type="publication">0001</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Software -->
-										<xsl:when test="//base_dc:typenorm = '6'">
-											<dr:CobjCategory type="software">0029</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Dataset -->										
-										<xsl:when test="//base_dc:typenorm = '7'">
-											<dr:CobjCategory type="dataset">0021</dr:CobjCategory>
-										</xsl:when>										
-																				
-										<!-- Still image -->
-										<xsl:when test="//base_dc:typenorm = '51'">
-											<dr:CobjCategory type="other">0025</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Moving image/Video -->										
-										<xsl:when test="//base_dc:typenorm = '52'">
-											<dr:CobjCategory type="other">0024</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Image/Video -->
-										<xsl:when test="//base_dc:typenorm = '5'">
-											<dr:CobjCategory type="other">0033</dr:CobjCategory>
-										</xsl:when>
+									<dr:CobjCategory>
+										<xsl:variable name="varCobjCategory" select="vocabulary:clean($varBaseNormType, 'dnet:publication_resource')" />
+										<xsl:variable name="varSuperType" select="vocabulary:clean($varCobjCategory, 'dnet:result_typologies')" />
+										<xsl:attribute name="type" select="$varSuperType" />
+										<xsl:value-of select="$varCobjCategory" />
+									</dr:CobjCategory>
 
-										<!-- Audio -->
-										<xsl:when test="//base_dc:typenorm = '4'">
-											<dr:CobjCategory type="other">0030</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Musical notation -->
-										<xsl:when test="//base_dc:typenorm = '2'">
-											<dr:CobjCategory type="other">0020</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Map -->
-										<xsl:when test="//base_dc:typenorm = '3'">
-											<dr:CobjCategory type="other">0020</dr:CobjCategory>
-										</xsl:when>										
-										
-										<!-- Other non-article -->
-										<xsl:when test="//base_dc:typenorm = '122'">
-											<dr:CobjCategory type="publication">0038</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Course material -->
-										<xsl:when test="//base_dc:typenorm = '16'">
-											<dr:CobjCategory type="publication">0038</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Manuscript -->
-										<xsl:when test="//base_dc:typenorm = '19'">
-											<dr:CobjCategory type="publication">0038</dr:CobjCategory>
-										</xsl:when>
-										
-										<!-- Conference object -->
-										<xsl:when test="//base_dc:typenorm = '13'">
-											<dr:CobjCategory type="publication">0004</dr:CobjCategory>
-										</xsl:when>
-
-										<!-- Unknown -->
-										<xsl:when test="//base_dc:typenorm = 'F'">
-											<dr:CobjCategory type="other">0000</dr:CobjCategory>
-										</xsl:when>
-										<xsl:otherwise>
-											<dr:CobjCategory type="other">0000</dr:CobjCategory>
-										</xsl:otherwise>
-									</xsl:choose>
-									
 									<oaf:accessrights>
 										<xsl:choose>
-											<xsl:when test="//base_dc:oa[.='0']">CLOSED</xsl:when>
 											<xsl:when test="//base_dc:oa[.='1']">OPEN</xsl:when>
-											<xsl:when test="//base_dc:oa[.='2']">UNKNOWN</xsl:when>
 											<xsl:when test="//base_dc:rightsnorm">
 												<xsl:value-of select="vocabulary:clean(//base_dc:rightsnorm, 'dnet:access_modes')" />
 											</xsl:when>
@@ -384,7 +244,7 @@
 
 									<oaf:identifier identifierType="oai-original">
 										<xsl:value-of
-											select="//oai:header/oai:identifier" />
+											select="//*[local-name() = 'about']/*[local-name() = 'provenance']//*[local-name() = 'originDescription' and not(./*[local-name() = 'originDescription'])]/*[local-name() = 'identifier']" />
 									</oaf:identifier>
 
 									<oaf:hostedBy>
@@ -429,6 +289,11 @@
 											</xsl:choose>
 										</oaf:relation>
 									</xsl:for-each>
+									
+									<xsl:for-each select="//base_dc:country">
+										<oaf:country><xsl:value-of select="vocabulary:clean(., 'dnet:countries')" /></oaf:country>
+									</xsl:for-each>
+									
 								</metadata>
 								<xsl:copy-of select="//*[local-name() = 'about']" />
 							</record>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/hostedBy_map.json

@@ -1048,5 +1048,10 @@
   "openaire_id": "re3data_____::r3d100010399",
   "datacite_name": "ZEW Forschungsdatenzentrum",
   "official_name": "ZEW Forschungsdatenzentrum"
+ },
+ "HBP.NEUROINF": {
+  "openaire_id": "fairsharing_::2975",
+  "datacite_name": "EBRAINS",
+  "official_name": "EBRAINS"
  }
 }

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/db/oozie_app/workflow.xml

@@ -1,4 +1,4 @@
-    <workflow-app name="Transform_BioEntity_Workflow" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="Transform_BioEntity_Workflow" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
             <name>sourcePath</name>
@@ -8,19 +8,40 @@
             <name>database</name>
             <description>the PDB Database Working Path</description>
         </property>
-
         <property>
-            <name>targetPath</name>
-            <description>the Target Working dir path</description>
+            <name>mdStoreOutputId</name>
+            <description>the identifier of the cleaned MDStore</description>
+        </property>
+        <property>
+            <name>mdStoreManagerURI</name>
+            <description>the path of the cleaned mdstore</description>
         </property>
     </parameters>
 
-    <start to="ConvertDB"/>
+    <start to="StartTransaction"/>
+
 
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
+    <action name="StartTransaction">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>NEW_VERSION</arg>
+            <arg>--mdStoreID</arg><arg>${mdStoreOutputId}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            <capture-output/>
+        </java>
+        <ok to="ConvertDB"/>
+        <error to="RollBack"/>
+    </action>
     <action name="ConvertDB">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn</master>
@@ -41,11 +62,48 @@
             <arg>--master</arg><arg>yarn</arg>
             <arg>--dbPath</arg><arg>${sourcePath}</arg>
             <arg>--database</arg><arg>${database}</arg>
-            <arg>--targetPath</arg><arg>${targetPath}</arg>
+            <arg>--mdstoreOutputVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
         </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
+        <ok to="CommitVersion"/>
+        <error to="RollBack"/>
+
     </action>
-    <end name="End"/>
+        <action name="CommitVersion">
+            <java>
+                <configuration>
+                    <property>
+                        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                        <value>true</value>
+                    </property>
+                </configuration>
+                <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+                <arg>--action</arg><arg>COMMIT</arg>
+                <arg>--namenode</arg><arg>${nameNode}</arg>
+                <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+                <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            </java>
+            <ok to="End"/>
+            <error to="Kill"/>
+        </action>
+
+        <action name="RollBack">
+            <java>
+                <configuration>
+                    <property>
+                        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                        <value>true</value>
+                    </property>
+                </configuration>
+                <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+                <arg>--action</arg><arg>ROLLBACK</arg>
+                <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+                <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            </java>
+            <ok to="Kill"/>
+            <error to="Kill"/>
+        </action>
+
+
+        <end name="End"/>
 
 </workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/bio_to_oaf_params.json

@@ -2,5 +2,5 @@
   {"paramName":"mt",  "paramLongName":"master",       "paramDescription": "should be local or yarn",                  "paramRequired": true},
   {"paramName":"db",  "paramLongName":"database",     "paramDescription": "should be PDB or UNIPROT",                 "paramRequired": true},
   {"paramName":"p",   "paramLongName":"dbPath",       "paramDescription": "the path of the database to transform",    "paramRequired": true},
-  {"paramName":"t",   "paramLongName":"targetPath",   "paramDescription": "the OAF target path ",                     "paramRequired": true}
+  {"paramName":"mo",   "paramLongName":"mdstoreOutputVersion",     "paramDescription": "the oaf path ",                "paramRequired": true}
 ]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/ebi_to_df_params.json

@@ -1,5 +1,20 @@
 [
-  {"paramName":"mt",  "paramLongName":"master",     "paramDescription": "should be local or yarn",                  "paramRequired": true},
-  {"paramName":"s",   "paramLongName":"sourcePath","paramDescription": "the source Path",                              "paramRequired": true},
-  {"paramName":"t",   "paramLongName":"targetPath","paramDescription": "the  oaf path ",  "paramRequired": true}
+  {
+    "paramName": "mt",
+    "paramLongName": "master",
+    "paramDescription": "should be local or yarn",
+    "paramRequired": true
+  },
+  {
+    "paramName": "s",
+    "paramLongName": "sourcePath",
+    "paramDescription": "the source Path",
+    "paramRequired": true
+  },
+  {
+    "paramName": "mo",
+    "paramLongName": "mdstoreOutputVersion",
+    "paramDescription": "the oaf path ",
+    "paramRequired": true
+  }
 ]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/sx/bio/ebi/oozie_app/workflow.xml

@@ -9,34 +9,26 @@
             <description>the Working Path</description>
         </property>
         <property>
-            <name>targetPath</name>
-            <description>the OAF MDStore Path</description>
+            <name>mdStoreOutputId</name>
+            <description>the identifier of the cleaned MDStore</description>
         </property>
         <property>
-            <name>sparkDriverMemory</name>
-            <description>memory for driver process</description>
-        </property>
-        <property>
-            <name>sparkExecutorMemory</name>
-            <description>memory for individual executor</description>
-        </property>
-        <property>
-            <name>sparkExecutorCores</name>
-            <description>number of cores used by single executor</description>
+            <name>mdStoreManagerURI</name>
+            <description>the path of the cleaned mdstore</description>
         </property>
         <property>
             <name>resumeFrom</name>
-            <value>DownloadEBILinks</value>
+            <value>CreateEBIDataSet</value>
             <description>node to start</description>
         </property>
     </parameters>
 
-    <start to="resume_from"/>
+    <start to="StartTransaction"/>
 
     <decision name="resume_from">
         <switch>
             <case to="DownloadEBILinks">${wf:conf('resumeFrom') eq 'DownloadEBILinks'}</case>
-            <case to="CreateEBIDataSet">${wf:conf('resumeFrom') eq 'CreateEBIDataSet'}</case>
+            <case to="StartTransaction">${wf:conf('resumeFrom') eq 'CreateEBIDataSet'}</case>
             <default to="DownloadEBILinks"/>
         </switch>
     </decision>
@@ -77,9 +69,29 @@
             <move source="${sourcePath}/ebi_links_dataset" target="${sourcePath}/ebi_links_dataset_old"/>
             <move source="${workingPath}/links_final" target="${sourcePath}/ebi_links_dataset"/>
         </fs>
-        <ok to="CreateEBIDataSet"/>
+        <ok to="StartTransaction"/>
         <error to="Kill"/>
     </action>
+
+    <action name="StartTransaction">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>NEW_VERSION</arg>
+            <arg>--mdStoreID</arg><arg>${mdStoreOutputId}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+            <capture-output/>
+        </java>
+        <ok to="CreateEBIDataSet"/>
+        <error to="RollBack"/>
+    </action>
+
+
     <action name="CreateEBIDataSet">
         <spark xmlns="uri:oozie:spark-action:0.2">
             <master>yarn-cluster</master>
@@ -95,11 +107,49 @@
                 ${sparkExtraOPT}
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/ebi_links_dataset</arg>
-            <arg>--targetPath</arg><arg>${targetPath}</arg>
+            <arg>--mdstoreOutputVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
             <arg>--master</arg><arg>yarn</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
     </action>
+
+
+    <action name="CommitVersion">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>COMMIT</arg>
+            <arg>--namenode</arg><arg>${nameNode}</arg>
+            <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+        </java>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="RollBack">
+        <java>
+            <configuration>
+                <property>
+                    <name>oozie.launcher.mapreduce.user.classpath.first</name>
+                    <value>true</value>
+                </property>
+            </configuration>
+            <main-class>eu.dnetlib.dhp.aggregation.mdstore.MDStoreActionNode</main-class>
+            <arg>--action</arg><arg>ROLLBACK</arg>
+            <arg>--mdStoreVersion</arg><arg>${wf:actionData('StartTransaction')['mdStoreVersion']}</arg>
+            <arg>--mdStoreManagerURI</arg><arg>${mdStoreManagerURI}</arg>
+        </java>
+        <ok to="Kill"/>
+        <error to="Kill"/>
+    </action>
+
     <end name="End"/>
+
 </workflow-app>

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/orcid/SparkApplyUpdate.scala

@@ -1,120 +0,0 @@
-package eu.dnetlib.dhp.collection.orcid
-
-import eu.dnetlib.dhp.application.AbstractScalaApplication
-import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
-import org.slf4j.{Logger, LoggerFactory}
-
-class SparkApplyUpdate(propertyPath: String, args: Array[String], log: Logger)
-    extends AbstractScalaApplication(propertyPath, args, log: Logger) {
-
-  /** Here all the spark applications runs this method
-    * where the whole logic of the spark node is defined
-    */
-  override def run(): Unit = {
-
-    val graphPath: String = parser.get("graphPath")
-    log.info("found parameters graphPath: {}", graphPath)
-    val updatePath: String = parser.get("updatePath")
-    log.info("found parameters updatePath: {}", updatePath)
-    val targetPath: String = parser.get("targetPath")
-    log.info("found parameters targetPath: {}", targetPath)
-    applyTableUpdate(spark, graphPath, updatePath, targetPath)
-    checkUpdate(spark, graphPath, targetPath)
-    moveTable(spark, graphPath, targetPath)
-
-  }
-
-  private def moveTable(spark: SparkSession, graphPath: String, updatePath: String): Unit = {
-    spark.read
-      .load(s"$updatePath/Authors")
-      .repartition(1000)
-      .write
-      .mode(SaveMode.Overwrite)
-      .save(s"$graphPath/Authors")
-    spark.read
-      .load(s"$updatePath/Works")
-      .repartition(1000)
-      .write
-      .mode(SaveMode.Overwrite)
-      .save(s"$graphPath/Works")
-    spark.read
-      .load(s"$updatePath/Employments")
-      .repartition(1000)
-      .write
-      .mode(SaveMode.Overwrite)
-      .save(s"$graphPath/Employments")
-
-  }
-
-  private def updateDataset(
-    inputDataset: DataFrame,
-    idUpdate: DataFrame,
-    updateDataframe: DataFrame,
-    targetPath: String
-  ): Unit = {
-    inputDataset
-      .join(idUpdate, inputDataset("orcid").equalTo(idUpdate("orcid")), "leftanti")
-      .select(inputDataset("*"))
-      .unionByName(updateDataframe)
-      .write
-      .mode(SaveMode.Overwrite)
-      .save(targetPath)
-  }
-
-  private def checkUpdate(spark: SparkSession, graphPath: String, updatePath: String): Unit = {
-    val totalOriginalAuthors = spark.read.load(s"$graphPath/Authors").count
-    val totalOriginalWorks = spark.read.load(s"$graphPath/Works").count
-    val totalOriginalEmployments = spark.read.load(s"$graphPath/Employments").count
-    val totalUpdateAuthors = spark.read.load(s"$updatePath/Authors").count
-    val totalUpdateWorks = spark.read.load(s"$updatePath/Works").count
-    val totalUpdateEmployments = spark.read.load(s"$updatePath/Employments").count
-
-    log.info("totalOriginalAuthors: {}", totalOriginalAuthors)
-    log.info("totalOriginalWorks: {}", totalOriginalWorks)
-    log.info("totalOriginalEmployments: {}", totalOriginalEmployments)
-    log.info("totalUpdateAuthors: {}", totalUpdateAuthors)
-    log.info("totalUpdateWorks: {}", totalUpdateWorks)
-    log.info("totalUpdateEmployments: {}", totalUpdateEmployments)
-    if (
-      totalUpdateAuthors < totalOriginalAuthors || totalUpdateEmployments < totalOriginalEmployments || totalUpdateWorks < totalOriginalWorks
-    )
-      throw new RuntimeException("The updated Graph contains less elements of the original one")
-
-  }
-
-  private def applyTableUpdate(spark: SparkSession, graphPath: String, updatePath: String, targetPath: String): Unit = {
-    val orcidIDUpdate = spark.read.load(s"$updatePath/Authors").select("orcid")
-    updateDataset(
-      spark.read.load(s"$graphPath/Authors"),
-      orcidIDUpdate,
-      spark.read.load(s"$updatePath/Authors"),
-      s"$targetPath/Authors"
-    )
-    updateDataset(
-      spark.read.load(s"$graphPath/Employments"),
-      orcidIDUpdate,
-      spark.read.load(s"$updatePath/Employments"),
-      s"$targetPath/Employments"
-    )
-    updateDataset(
-      spark.read.load(s"$graphPath/Works"),
-      orcidIDUpdate,
-      spark.read.load(s"$updatePath/Works"),
-      s"$targetPath/Works"
-    )
-  }
-
-}
-
-object SparkApplyUpdate {
-
-  val log: Logger = LoggerFactory.getLogger(SparkGenerateORCIDTable.getClass)
-
-  def main(args: Array[String]): Unit = {
-
-    new SparkApplyUpdate("/eu/dnetlib/dhp/collection/orcid/apply_orcid_table_parameter.json", args, log)
-      .initialize()
-      .run()
-
-  }
-}

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/orcid/SparkGenerateORCIDTable.scala

@@ -6,7 +6,6 @@ import org.apache.hadoop.io.Text
 import org.apache.spark.SparkContext
 import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
-import scala.collection.JavaConverters._
 
 class SparkGenerateORCIDTable(propertyPath: String, args: Array[String], log: Logger)
     extends AbstractScalaApplication(propertyPath, args, log: Logger) {
@@ -19,16 +18,12 @@ class SparkGenerateORCIDTable(propertyPath: String, args: Array[String], log: Lo
     log.info("found parameters sourcePath: {}", sourcePath)
     val targetPath: String = parser.get("targetPath")
     log.info("found parameters targetPath: {}", targetPath)
-    val fromUpdate = "true".equals(parser.get("fromUpdate"))
-    val sourceSummaryPath = if (fromUpdate) s"$sourcePath/summary*" else sourcePath
-    val sourceEmploymentsPath = if (fromUpdate) s"$sourcePath/employments*" else sourcePath
-    val sourceWorksPath = if (fromUpdate) s"$sourcePath/works*" else sourcePath
-    extractORCIDTable(spark, sourceSummaryPath, targetPath, fromUpdate)
-    extractORCIDEmploymentsTable(spark, sourceEmploymentsPath, targetPath, fromUpdate)
-    extractORCIDWorksTable(spark, sourceWorksPath, targetPath, fromUpdate)
+    extractORCIDTable(spark, sourcePath, targetPath)
+    extractORCIDEmploymentsTable(spark, sourcePath, targetPath)
+    extractORCIDWorksTable(spark, sourcePath, targetPath)
   }
 
-  def extractORCIDTable(spark: SparkSession, sourcePath: String, targetPath: String, skipFilterByKey: Boolean): Unit = {
+  def extractORCIDTable(spark: SparkSession, sourcePath: String, targetPath: String): Unit = {
     val sc: SparkContext = spark.sparkContext
     import spark.implicits._
     val df = sc
@@ -37,8 +32,8 @@ class SparkGenerateORCIDTable(propertyPath: String, args: Array[String], log: Lo
       .toDF
       .as[(String, String)]
     implicit val orcidAuthor: Encoder[Author] = Encoders.bean(classOf[Author])
-    val newDf = if (!skipFilterByKey) df.filter(r => r._1.contains("summaries")) else df
-    newDf
+//    implicit  val orcidPID:Encoder[Pid] = Encoders.bean(classOf[Pid])
+    df.filter(r => r._1.contains("summaries"))
       .map { r =>
         val p = new OrcidParser
         p.parseSummary(r._2)
@@ -49,12 +44,7 @@ class SparkGenerateORCIDTable(propertyPath: String, args: Array[String], log: Lo
       .save(s"$targetPath/Authors")
   }
 
-  def extractORCIDWorksTable(
-    spark: SparkSession,
-    sourcePath: String,
-    targetPath: String,
-    skipFilterByKey: Boolean
-  ): Unit = {
+  def extractORCIDWorksTable(spark: SparkSession, sourcePath: String, targetPath: String): Unit = {
     val sc: SparkContext = spark.sparkContext
     import spark.implicits._
     val df = sc
@@ -63,37 +53,19 @@ class SparkGenerateORCIDTable(propertyPath: String, args: Array[String], log: Lo
       .toDF
       .as[(String, String)]
     implicit val orcidWorkAuthor: Encoder[Work] = Encoders.bean(classOf[Work])
-
-    //We are in the case of parsing ORCID UPDATE
-    if (skipFilterByKey) {
-      df.flatMap { r =>
+    implicit val orcidPID: Encoder[Pid] = Encoders.bean(classOf[Pid])
+    df.filter(r => r._1.contains("works"))
+      .map { r =>
         val p = new OrcidParser
-        p.parseWorks(r._2).asScala
-      }.filter(p => p != null)
-        .write
-        .mode(SaveMode.Overwrite)
-        .save(s"$targetPath/Works")
-    }
-    //We are in the case of parsing ORCID DUMP
-    else {
-      df.filter(r => r._1.contains("works"))
-        .map { r =>
-          val p = new OrcidParser
-          p.parseWork(r._2)
-        }
-        .filter(p => p != null)
-        .write
-        .mode(SaveMode.Overwrite)
-        .save(s"$targetPath/Works")
-    }
+        p.parseWork(r._2)
+      }
+      .filter(p => p != null)
+      .write
+      .mode(SaveMode.Overwrite)
+      .save(s"$targetPath/Works")
   }
 
-  def extractORCIDEmploymentsTable(
-    spark: SparkSession,
-    sourcePath: String,
-    targetPath: String,
-    skipFilterByKey: Boolean
-  ): Unit = {
+  def extractORCIDEmploymentsTable(spark: SparkSession, sourcePath: String, targetPath: String): Unit = {
     val sc: SparkContext = spark.sparkContext
     import spark.implicits._
     val df = sc
@@ -102,27 +74,16 @@ class SparkGenerateORCIDTable(propertyPath: String, args: Array[String], log: Lo
       .toDF
       .as[(String, String)]
     implicit val orcidEmploymentAuthor: Encoder[Employment] = Encoders.bean(classOf[Employment])
-    if (skipFilterByKey) {
-      df.flatMap { r =>
+    implicit val orcidPID: Encoder[Pid] = Encoders.bean(classOf[Pid])
+    df.filter(r => r._1.contains("employments"))
+      .map { r =>
         val p = new OrcidParser
-        p.parseEmployments(r._2).asScala
-      }.filter(p => p != null)
-        .write
-        .mode(SaveMode.Overwrite)
-        .save(s"$targetPath/Employments")
-    }
-    //We are in the case of parsing ORCID DUMP
-    else {
-      df.filter(r => r._1.contains("employments"))
-        .map { r =>
-          val p = new OrcidParser
-          p.parseEmployment(r._2)
-        }
-        .filter(p => p != null)
-        .write
-        .mode(SaveMode.Overwrite)
-        .save(s"$targetPath/Employments")
-    }
+        p.parseEmployment(r._2)
+      }
+      .filter(p => p != null)
+      .write
+      .mode(SaveMode.Overwrite)
+      .save(s"$targetPath/Employments")
   }
 }
 

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala

@@ -231,7 +231,7 @@ object BioDBToOAF {
   def uniprotToOAF(input: String): List[Oaf] = {
     implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
     lazy val json = parse(input)
-    val pid = (json \ "pid").extract[String]
+    val pid = (json \ "pid").extract[String].trim()
 
     val d = new Dataset
 

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/SparkTransformBioDatabaseToOAF.scala

@@ -2,12 +2,15 @@ package eu.dnetlib.dhp.sx.bio
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.collection.CollectionUtils
+import eu.dnetlib.dhp.common.Constants.{MDSTORE_DATA_PATH, MDSTORE_SIZE_PATH}
+import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion
 import eu.dnetlib.dhp.schema.oaf.Oaf
 import eu.dnetlib.dhp.sx.bio.BioDBToOAF.ScholixResolved
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Encoder, Encoders, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
+import eu.dnetlib.dhp.utils.DHPUtils.{MAPPER, writeHdfsFile}
 
 object SparkTransformBioDatabaseToOAF {
 
@@ -25,8 +28,13 @@ object SparkTransformBioDatabaseToOAF {
 
     val dbPath: String = parser.get("dbPath")
     log.info("dbPath: {}", database)
-    val targetPath: String = parser.get("targetPath")
-    log.info("targetPath: {}", database)
+
+    val mdstoreOutputVersion = parser.get("mdstoreOutputVersion")
+    log.info("mdstoreOutputVersion: {}", mdstoreOutputVersion)
+
+    val cleanedMdStoreVersion = MAPPER.readValue(mdstoreOutputVersion, classOf[MDStoreVersion])
+    val outputBasePath = cleanedMdStoreVersion.getHdfsPath
+    log.info("outputBasePath: {}", outputBasePath)
 
     val spark: SparkSession =
       SparkSession
@@ -43,24 +51,28 @@ object SparkTransformBioDatabaseToOAF {
       case "UNIPROT" =>
         CollectionUtils.saveDataset(
           spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.uniprotToOAF(i))),
-          targetPath
+          s"$outputBasePath/$MDSTORE_DATA_PATH"
         )
       case "PDB" =>
         CollectionUtils.saveDataset(
           spark.createDataset(sc.textFile(dbPath).flatMap(i => BioDBToOAF.pdbTOOaf(i))),
-          targetPath
+          s"$outputBasePath/$MDSTORE_DATA_PATH"
         )
       case "SCHOLIX" =>
         CollectionUtils.saveDataset(
           spark.read.load(dbPath).as[ScholixResolved].map(i => BioDBToOAF.scholixResolvedToOAF(i)),
-          targetPath
+          s"$outputBasePath/$MDSTORE_DATA_PATH"
         )
       case "CROSSREF_LINKS" =>
         CollectionUtils.saveDataset(
           spark.createDataset(sc.textFile(dbPath).map(i => BioDBToOAF.crossrefLinksToOaf(i))),
-          targetPath
+          s"$outputBasePath/$MDSTORE_DATA_PATH"
         )
     }
+
+    val df = spark.read.text(s"$outputBasePath/$MDSTORE_DATA_PATH")
+    val mdStoreSize = df.count
+    writeHdfsFile(spark.sparkContext.hadoopConfiguration, s"$mdStoreSize", s"$outputBasePath/$MDSTORE_SIZE_PATH")
   }
 
 }

dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkEBILinksToOaf.scala

@@ -9,6 +9,9 @@ import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
 import org.apache.spark.sql._
 import org.slf4j.{Logger, LoggerFactory}
+import eu.dnetlib.dhp.common.Constants.{MDSTORE_DATA_PATH, MDSTORE_SIZE_PATH}
+import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion
+import eu.dnetlib.dhp.utils.DHPUtils.{MAPPER, writeHdfsFile}
 
 object SparkEBILinksToOaf {
 
@@ -32,8 +35,13 @@ object SparkEBILinksToOaf {
     import spark.implicits._
     val sourcePath = parser.get("sourcePath")
     log.info(s"sourcePath  -> $sourcePath")
-    val targetPath = parser.get("targetPath")
-    log.info(s"targetPath  -> $targetPath")
+    val mdstoreOutputVersion = parser.get("mdstoreOutputVersion")
+    log.info("mdstoreOutputVersion: {}", mdstoreOutputVersion)
+
+    val cleanedMdStoreVersion = MAPPER.readValue(mdstoreOutputVersion, classOf[MDStoreVersion])
+    val outputBasePath = cleanedMdStoreVersion.getHdfsPath
+    log.info("outputBasePath: {}", outputBasePath)
+
     implicit val PMEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
 
     val ebLinks: Dataset[EBILinkItem] = spark.read
@@ -46,7 +54,10 @@ object SparkEBILinksToOaf {
         .flatMap(j => BioDBToOAF.parse_ebi_links(j.links))
         .filter(p => BioDBToOAF.EBITargetLinksFilter(p))
         .flatMap(p => BioDBToOAF.convertEBILinksToOaf(p)),
-      targetPath
+      s"$outputBasePath/$MDSTORE_DATA_PATH"
     )
+    val df = spark.read.text(s"$outputBasePath/$MDSTORE_DATA_PATH")
+    val mdStoreSize = df.count
+    writeHdfsFile(spark.sparkContext.hadoopConfiguration, s"$mdStoreSize", s"$outputBasePath/$MDSTORE_SIZE_PATH")
   }
 }

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java

@@ -78,6 +78,10 @@ public class PrepareAffiliationRelationsTest {
 			.getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json")
 			.getPath();
 
+		String pubmedAffiliationRelationsPath = getClass()
+			.getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json")
+			.getPath();
+
 		String outputPath = workingDir.toString() + "/actionSet";
 
 		PrepareAffiliationRelations
@@ -85,8 +89,7 @@ public class PrepareAffiliationRelationsTest {
 				new String[] {
 					"-isSparkSessionManaged", Boolean.FALSE.toString(),
 					"-crossrefInputPath", crossrefAffiliationRelationPath,
-					"-pubmedInputPath", crossrefAffiliationRelationPath,
-					"-openapcInputPath", crossrefAffiliationRelationPath,
+					"-pubmedInputPath", pubmedAffiliationRelationsPath,
 					"-outputPath", outputPath
 				});
 
@@ -103,7 +106,7 @@ public class PrepareAffiliationRelationsTest {
 //            );
 //        }
 		// count the number of relations
-		assertEquals(60, tmp.count());
+		assertEquals(40, tmp.count());
 
 		Dataset<Relation> dataset = spark.createDataset(tmp.rdd(), Encoders.bean(Relation.class));
 		dataset.createOrReplaceTempView("result");
@@ -114,7 +117,7 @@ public class PrepareAffiliationRelationsTest {
 		// verify that we have equal number of bi-directional relations
 		Assertions
 			.assertEquals(
-				30, execVerification
+				20, execVerification
 					.filter(
 						"relClass='" + ModelConstants.HAS_AUTHOR_INSTITUTION + "'")
 					.collectAsList()
@@ -122,7 +125,7 @@ public class PrepareAffiliationRelationsTest {
 
 		Assertions
 			.assertEquals(
-				30, execVerification
+				20, execVerification
 					.filter(
 						"relClass='" + ModelConstants.IS_AUTHOR_INSTITUTION_OF + "'")
 					.collectAsList()

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/transformativeagreement/CreateTAActionSetTest.java

@@ -1,324 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.transformativeagreement;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.SparkSession;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob;
-import eu.dnetlib.dhp.actionmanager.opencitations.CreateOpenCitationsASTest;
-import eu.dnetlib.dhp.schema.action.AtomicAction;
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.Relation;
-import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
-import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
-
-/**
- * @author miriam.baglioni
- * @Date 13/02/24
- */
-public class CreateTAActionSetTest {
-	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
-	private static SparkSession spark;
-
-	private static Path workingDir;
-	private static final Logger log = LoggerFactory
-		.getLogger(CreateOpenCitationsASTest.class);
-
-	@BeforeAll
-	public static void beforeAll() throws IOException {
-		workingDir = Files
-			.createTempDirectory(CreateTAActionSetTest.class.getSimpleName());
-		log.info("using work dir {}", workingDir);
-
-		SparkConf conf = new SparkConf();
-		conf.setAppName(CreateTAActionSetTest.class.getSimpleName());
-
-		conf.setMaster("local[*]");
-		conf.set("spark.driver.host", "localhost");
-		conf.set("hive.metastore.local", "true");
-		conf.set("spark.ui.enabled", "false");
-		conf.set("spark.sql.warehouse.dir", workingDir.toString());
-		conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
-
-		spark = SparkSession
-			.builder()
-			.appName(CreateTAActionSetTest.class.getSimpleName())
-			.config(conf)
-			.getOrCreate();
-	}
-
-	@AfterAll
-	public static void afterAll() throws IOException {
-		FileUtils.deleteDirectory(workingDir.toFile());
-		spark.stop();
-	}
-
-	@Test
-	void createActionSet() throws Exception {
-
-		String inputPath = getClass()
-			.getResource(
-				"/eu/dnetlib/dhp/actionmanager/transformativeagreement/facts.json")
-			.getPath();
-
-		eu.dnetlib.dhp.actionmanager.transformativeagreement.CreateActionSetSparkJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged",
-					Boolean.FALSE.toString(),
-					"-inputPath",
-					inputPath,
-					"-outputPath",
-					workingDir.toString() + "/actionSet1"
-				});
-
-	}
-
-	@Test
-	void testNumberofRelations2() throws Exception {
-
-		String inputPath = getClass()
-			.getResource(
-				"/eu/dnetlib/dhp/actionmanager/opencitations/COCI")
-			.getPath();
-
-		eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged",
-					Boolean.FALSE.toString(),
-					"-inputPath",
-					inputPath,
-					"-outputPath",
-					workingDir.toString() + "/actionSet2"
-				});
-
-		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
-
-		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet2", Text.class, Text.class)
-			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
-			.map(aa -> ((Relation) aa.getPayload()));
-
-		assertEquals(23, tmp.count());
-
-		// tmp.foreach(r -> System.out.println(OBJECT_MAPPER.writeValueAsString(r)));
-
-	}
-
-	@Test
-	void testRelationsCollectedFrom() throws Exception {
-
-		String inputPath = getClass()
-			.getResource(
-				"/eu/dnetlib/dhp/actionmanager/opencitations/COCI")
-			.getPath();
-
-		eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged",
-					Boolean.FALSE.toString(),
-					"-inputPath",
-					inputPath,
-					"-outputPath",
-					workingDir.toString() + "/actionSet3"
-				});
-
-		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
-
-		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet3", Text.class, Text.class)
-			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
-			.map(aa -> ((Relation) aa.getPayload()));
-
-		tmp.foreach(r -> {
-			assertEquals(ModelConstants.OPENOCITATIONS_NAME, r.getCollectedfrom().get(0).getValue());
-			assertEquals(ModelConstants.OPENOCITATIONS_ID, r.getCollectedfrom().get(0).getKey());
-		});
-
-	}
-
-	@Test
-	void testRelationsDataInfo() throws Exception {
-
-		String inputPath = getClass()
-			.getResource(
-				"/eu/dnetlib/dhp/actionmanager/opencitations/COCI")
-			.getPath();
-
-		eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged",
-					Boolean.FALSE.toString(),
-					"-inputPath",
-					inputPath,
-					"-outputPath",
-					workingDir.toString() + "/actionSet4"
-				});
-
-		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
-
-		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet4", Text.class, Text.class)
-			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
-			.map(aa -> ((Relation) aa.getPayload()));
-
-		tmp.foreach(r -> {
-			assertEquals(false, r.getDataInfo().getInferred());
-			assertEquals(false, r.getDataInfo().getDeletedbyinference());
-			assertEquals("0.91", r.getDataInfo().getTrust());
-			assertEquals(
-				eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob.OPENCITATIONS_CLASSID,
-				r.getDataInfo().getProvenanceaction().getClassid());
-			assertEquals(
-				eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob.OPENCITATIONS_CLASSNAME,
-				r.getDataInfo().getProvenanceaction().getClassname());
-			assertEquals(ModelConstants.DNET_PROVENANCE_ACTIONS, r.getDataInfo().getProvenanceaction().getSchemeid());
-			assertEquals(ModelConstants.DNET_PROVENANCE_ACTIONS, r.getDataInfo().getProvenanceaction().getSchemename());
-		});
-
-	}
-
-	@Test
-	void testRelationsSemantics() throws Exception {
-
-		String inputPath = getClass()
-			.getResource(
-				"/eu/dnetlib/dhp/actionmanager/opencitations/COCI")
-			.getPath();
-
-		eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged",
-					Boolean.FALSE.toString(),
-					"-inputPath",
-					inputPath,
-					"-outputPath",
-					workingDir.toString() + "/actionSet5"
-				});
-
-		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
-
-		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet5", Text.class, Text.class)
-			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
-			.map(aa -> ((Relation) aa.getPayload()));
-
-		tmp.foreach(r -> {
-			assertEquals("citation", r.getSubRelType());
-			assertEquals("resultResult", r.getRelType());
-		});
-		assertEquals(23, tmp.filter(r -> r.getRelClass().equals("Cites")).count());
-		assertEquals(0, tmp.filter(r -> r.getRelClass().equals("IsCitedBy")).count());
-
-	}
-
-	@Test
-	void testRelationsSourceTargetPrefix() throws Exception {
-
-		String inputPath = getClass()
-			.getResource(
-				"/eu/dnetlib/dhp/actionmanager/opencitations/COCI")
-			.getPath();
-
-		eu.dnetlib.dhp.actionmanager.opencitations.CreateActionSetSparkJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged",
-					Boolean.FALSE.toString(),
-					"-inputPath",
-					inputPath,
-					"-outputPath",
-					workingDir.toString() + "/actionSet6"
-				});
-
-		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
-
-		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet6", Text.class, Text.class)
-			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
-			.map(aa -> ((Relation) aa.getPayload()));
-
-		tmp.foreach(r -> {
-			assertEquals("50|doi_________::", r.getSource().substring(0, 17));
-			assertEquals("50|doi_________::", r.getTarget().substring(0, 17));
-		});
-
-	}
-
-	@Test
-	void testRelationsSourceTargetCouple() throws Exception {
-		final String doi1 = "50|doi_________::"
-			+ IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1007/s10854-015-3684-x"));
-		final String doi2 = "50|doi_________::"
-			+ IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1111/j.1551-2916.2008.02408.x"));
-		final String doi3 = "50|doi_________::"
-			+ IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1007/s10854-014-2114-9"));
-		final String doi4 = "50|doi_________::"
-			+ IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1016/j.ceramint.2013.09.069"));
-		final String doi5 = "50|doi_________::"
-			+ IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1007/s10854-009-9913-4"));
-		final String doi6 = "50|doi_________::"
-			+ IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", "10.1016/0038-1098(72)90370-5"));
-
-		String inputPath = getClass()
-			.getResource(
-				"/eu/dnetlib/dhp/actionmanager/opencitations/COCI")
-			.getPath();
-
-		CreateActionSetSparkJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged",
-					Boolean.FALSE.toString(),
-					"-inputPath",
-					inputPath,
-					"-outputPath",
-					workingDir.toString() + "/actionSet7"
-				});
-
-		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
-
-		JavaRDD<Relation> tmp = sc
-			.sequenceFile(workingDir.toString() + "/actionSet7", Text.class, Text.class)
-			.map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class))
-			.map(aa -> ((Relation) aa.getPayload()));
-
-		JavaRDD<Relation> check = tmp.filter(r -> r.getSource().equals(doi1) || r.getTarget().equals(doi1));
-
-		assertEquals(5, check.count());
-
-//		check.foreach(r -> {
-//			if (r.getSource().equals(doi2) || r.getSource().equals(doi3) || r.getSource().equals(doi4) ||
-//				r.getSource().equals(doi5) || r.getSource().equals(doi6)) {
-//				assertEquals(ModelConstants.IS_CITED_BY, r.getRelClass());
-//				assertEquals(doi1, r.getTarget());
-//			}
-//		});
-
-		assertEquals(5, check.filter(r -> r.getSource().equals(doi1)).count());
-		check.filter(r -> r.getSource().equals(doi1)).foreach(r -> assertEquals(ModelConstants.CITES, r.getRelClass()));
-
-	}
-}

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java

@@ -2,7 +2,6 @@
 package eu.dnetlib.dhp.collection.orcid;
 
 import java.io.IOException;
-import java.net.URI;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
@@ -10,12 +9,7 @@ import java.util.Objects;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.apache.spark.SparkContext;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.Encoders;
@@ -33,7 +27,6 @@ import com.ximpleware.XPathParseException;
 
 import eu.dnetlib.dhp.collection.orcid.model.Author;
 import eu.dnetlib.dhp.collection.orcid.model.ORCIDItem;
-import eu.dnetlib.dhp.collection.orcid.model.Work;
 import eu.dnetlib.dhp.parser.utility.VtdException;
 
 public class DownloadORCIDTest {
@@ -89,34 +82,6 @@ public class DownloadORCIDTest {
 		});
 	}
 
-	@Test
-	public void testParsingOrcidUpdateEmployments() throws Exception {
-		final String xml = IOUtils
-			.toString(
-				Objects
-					.requireNonNull(
-						getClass().getResourceAsStream("/eu/dnetlib/dhp/collection/orcid/update_employments.xml")));
-
-		final OrcidParser parser = new OrcidParser();
-		final ObjectMapper mapper = new ObjectMapper();
-		System.out.println(mapper.writeValueAsString(parser.parseEmployments(xml)));
-	}
-
-	@Test
-	public void testParsingOrcidUpdateWorks() throws Exception {
-		final String xml = IOUtils
-			.toString(
-				Objects
-					.requireNonNull(
-						getClass().getResourceAsStream("/eu/dnetlib/dhp/collection/orcid/update_work.xml")));
-
-		final OrcidParser parser = new OrcidParser();
-		final List<Work> works = parser.parseWorks(xml);
-
-		final ObjectMapper mapper = new ObjectMapper();
-		System.out.println(mapper.writeValueAsString(works));
-	}
-
 	@Test
 	public void testParsingEmployments() throws Exception {
 

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/base/BaseTransfomationTest.java

@@ -1,8 +1,6 @@
 
 package eu.dnetlib.dhp.collection.plugin.base;
 
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
 import java.io.IOException;
 
 import org.apache.commons.io.IOUtils;
@@ -65,21 +63,6 @@ public class BaseTransfomationTest extends AbstractVocabularyTest {
 		System.out.println(result.getBody());
 	}
 
-	@Test
-	void testBase2ODF_wrong_date() throws Exception {
-
-		final MetadataRecord mr = new MetadataRecord();
-		mr.setProvenance(new Provenance("DSID", "DSNAME", "PREFIX"));
-		mr.setBody(IOUtils.toString(getClass().getResourceAsStream("record_wrong_1.xml")));
-
-		final XSLTTransformationFunction tr = loadTransformationRule("xml/base2oaf.transformationRule.xml");
-
-		assertThrows(NullPointerException.class, () -> {
-			final MetadataRecord result = tr.call(mr);
-			System.out.println(result.getBody());
-		});
-	}
-
 	private XSLTTransformationFunction loadTransformationRule(final String path) throws Exception {
 		final String xslt = new SAXReader()
 				.read(this.getClass().getResourceAsStream(path))

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestIteratorTest.java

@@ -9,7 +9,6 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import eu.dnetlib.dhp.common.collection.CollectorException;
 import eu.dnetlib.dhp.common.collection.HttpClientParams;
 
 /**
@@ -38,7 +37,7 @@ public class RestIteratorTest {
 
 	@Disabled
 	@Test
-	public void test() throws CollectorException {
+	public void test() {
 
 		HttpClientParams clientParams = new HttpClientParams();
 

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/utils/JsonUtilsTest.java

@@ -1,48 +0,0 @@
-
-package eu.dnetlib.dhp.collection.plugin.utils;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import org.junit.jupiter.api.Test;
-
-class JsonUtilsTest {
-
-	static private String wrapped(String xml) {
-		return "<?xml version=\"1.0\" encoding=\"UTF-8\"?><recordWrap>" + xml + "</recordWrap>";
-	}
-
-	@Test
-	void keyStartWithDigit() {
-		assertEquals(
-			wrapped("<m_100><n_200v>null</n_200v></m_100>"),
-			JsonUtils.convertToXML("{\"100\" : {\"200v\" : null}}"));
-	}
-
-	@Test
-	void keyStartWithSpecialchars() {
-		assertEquals(
-			wrapped("<_parent><_nest1><_nest2>null</_nest2></_nest1></_parent>"),
-			JsonUtils.convertToXML("{\"   parent\" : {\"-nest1\" : {\".nest2\" : null}}}"));
-	}
-
-	@Test
-	void encodeArray() {
-		assertEquals(
-			wrapped("<_parent.child>1</_parent.child><_parent.child>2</_parent.child>"),
-			JsonUtils.convertToXML("{\" parent.child\":[1, 2]}"));
-	}
-
-	@Test
-	void arrayOfObjects() {
-		assertEquals(
-			wrapped("<parent><id>1</id></parent><parent><id>2</id></parent>"),
-			JsonUtils.convertToXML("{\"parent\": [{\"id\": 1}, {\"id\": 2}]}"));
-	}
-
-	@Test
-	void removeControlCharacters() {
-		assertEquals(
-			wrapped("<m_100><n_200v>Test</n_200v></m_100>"),
-			JsonUtils.convertToXML("{\"100\" : {\"200v\" : \"\\u0000\\u000cTest\"}}"));
-	}
-}

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/summary2.xml

@@ -1,966 +0,0 @@
-<record:record path="/0000-0001-6816-8350" xmlns:internal="http://www.orcid.org/ns/internal" xmlns:education="http://www.orcid.org/ns/education" xmlns:distinction="http://www.orcid.org/ns/distinction" xmlns:deprecated="http://www.orcid.org/ns/deprecated" xmlns:other-name="http://www.orcid.org/ns/other-name" xmlns:membership="http://www.orcid.org/ns/membership" xmlns:error="http://www.orcid.org/ns/error" xmlns:common="http://www.orcid.org/ns/common" xmlns:record="http://www.orcid.org/ns/record" xmlns:personal-details="http://www.orcid.org/ns/personal-details" xmlns:keyword="http://www.orcid.org/ns/keyword" xmlns:email="http://www.orcid.org/ns/email" xmlns:external-identifier="http://www.orcid.org/ns/external-identifier" xmlns:funding="http://www.orcid.org/ns/funding" xmlns:preferences="http://www.orcid.org/ns/preferences" xmlns:address="http://www.orcid.org/ns/address" xmlns:invited-position="http://www.orcid.org/ns/invited-position" xmlns:work="http://www.orcid.org/ns/work" xmlns:history="http://www.orcid.org/ns/history" xmlns:employment="http://www.orcid.org/ns/employment" xmlns:qualification="http://www.orcid.org/ns/qualification" xmlns:service="http://www.orcid.org/ns/service" xmlns:person="http://www.orcid.org/ns/person" xmlns:activities="http://www.orcid.org/ns/activities" xmlns:researcher-url="http://www.orcid.org/ns/researcher-url" xmlns:peer-review="http://www.orcid.org/ns/peer-review" xmlns:bulk="http://www.orcid.org/ns/bulk" xmlns:research-resource="http://www.orcid.org/ns/research-resource">
-    <common:orcid-identifier>
-        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-        <common:path>0000-0001-6816-8350</common:path>
-        <common:host>orcid.org</common:host>
-    </common:orcid-identifier>
-    <preferences:preferences>
-        <preferences:locale>en</preferences:locale>
-    </preferences:preferences>
-    <history:history>
-        <history:creation-method>Direct</history:creation-method>
-        <history:submission-date>2016-01-06T05:08:45.720Z</history:submission-date>
-        <common:last-modified-date>2024-01-02T20:07:05.186Z</common:last-modified-date>
-        <history:claimed>true</history:claimed>
-        <history:verified-email>true</history:verified-email>
-        <history:verified-primary-email>true</history:verified-primary-email>
-    </history:history>
-    <person:person path="/0000-0001-6816-8350/person">
-        <common:last-modified-date>2023-12-02T13:32:05.269Z</common:last-modified-date>
-        <other-name:other-names path="/0000-0001-6816-8350/other-names"/>
-        <researcher-url:researcher-urls path="/0000-0001-6816-8350/researcher-urls">
-            <common:last-modified-date>2016-02-09T09:18:18.417Z</common:last-modified-date>
-            <researcher-url:researcher-url put-code="633431" visibility="public" path="/0000-0001-6816-8350/researcher-urls/633431" display-index="0">
-                <common:created-date>2016-02-09T09:18:18.416Z</common:created-date>
-                <common:last-modified-date>2016-02-09T09:18:18.417Z</common:last-modified-date>
-                <common:source>
-                    <common:source-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-orcid>
-                </common:source>
-                <researcher-url:url-name>Dr Michael Muchiri</researcher-url:url-name>
-                <researcher-url:url>http://www.rmit.edu.au/contact/staff-contacts/academic-staff/m/muchiri-dr-michael</researcher-url:url>
-            </researcher-url:researcher-url>
-        </researcher-url:researcher-urls>
-        <email:emails path="/0000-0001-6816-8350/email"/>
-        <address:addresses path="/0000-0001-6816-8350/address">
-            <common:last-modified-date>2023-12-02T13:32:05.269Z</common:last-modified-date>
-            <address:address put-code="897528" visibility="public" path="/0000-0001-6816-8350/address/897528" display-index="2">
-                <common:created-date>2018-02-13T02:32:04.094Z</common:created-date>
-                <common:last-modified-date>2023-12-02T13:32:05.269Z</common:last-modified-date>
-                <common:source>
-                    <common:source-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-orcid>
-                </common:source>
-                <address:country>AU</address:country>
-            </address:address>
-            <address:address put-code="3191142" visibility="public" path="/0000-0001-6816-8350/address/3191142" display-index="1">
-                <common:created-date>2023-12-02T13:32:05.260Z</common:created-date>
-                <common:last-modified-date>2023-12-02T13:32:05.260Z</common:last-modified-date>
-                <common:source>
-                    <common:source-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-orcid>
-                </common:source>
-                <address:country>SA</address:country>
-            </address:address>
-        </address:addresses>
-        <keyword:keywords path="/0000-0001-6816-8350/keywords">
-            <common:last-modified-date>2023-12-02T13:31:16.269Z</common:last-modified-date>
-            <keyword:keyword put-code="368304" visibility="public" path="/0000-0001-6816-8350/keywords/368304" display-index="4">
-                <common:created-date>2016-02-09T09:16:44.001Z</common:created-date>
-                <common:last-modified-date>2023-12-02T13:31:16.269Z</common:last-modified-date>
-                <common:source>
-                    <common:source-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-orcid>
-                </common:source>
-                <keyword:content>Organizational Behavior</keyword:content>
-            </keyword:keyword>
-            <keyword:keyword put-code="368303" visibility="public" path="/0000-0001-6816-8350/keywords/368303" display-index="3">
-                <common:created-date>2016-02-09T09:16:27.374Z</common:created-date>
-                <common:last-modified-date>2023-12-02T13:31:16.269Z</common:last-modified-date>
-                <common:source>
-                    <common:source-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-orcid>
-                </common:source>
-                <keyword:content>Organizational Leadership</keyword:content>
-            </keyword:keyword>
-            <keyword:keyword put-code="368306" visibility="public" path="/0000-0001-6816-8350/keywords/368306" display-index="2">
-                <common:created-date>2016-02-09T09:17:08.998Z</common:created-date>
-                <common:last-modified-date>2023-12-02T13:31:16.269Z</common:last-modified-date>
-                <common:source>
-                    <common:source-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-orcid>
-                </common:source>
-                <keyword:content>Organizational performance</keyword:content>
-            </keyword:keyword>
-            <keyword:keyword put-code="3590814" visibility="public" path="/0000-0001-6816-8350/keywords/3590814" display-index="1">
-                <common:created-date>2023-12-02T13:31:16.259Z</common:created-date>
-                <common:last-modified-date>2023-12-02T13:31:16.259Z</common:last-modified-date>
-                <common:source>
-                    <common:source-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-orcid>
-                </common:source>
-                <keyword:content>Thriving at work</keyword:content>
-            </keyword:keyword>
-        </keyword:keywords>
-        <external-identifier:external-identifiers path="/0000-0001-6816-8350/external-identifiers">
-            <common:last-modified-date>2018-04-10T00:49:55.386Z</common:last-modified-date>
-            <external-identifier:external-identifier put-code="998076" visibility="public" path="/0000-0001-6816-8350/external-identifiers/998076" display-index="0">
-                <common:created-date>2018-04-10T00:49:55.385Z</common:created-date>
-                <common:last-modified-date>2018-04-10T00:49:55.386Z</common:last-modified-date>
-                <common:source>
-                    <common:source-client-id>
-                        <common:uri>https://orcid.org/client/0000-0003-1377-5676</common:uri>
-                        <common:path>0000-0003-1377-5676</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:source-client-id>
-                    <common:source-name>ResearcherID</common:source-name>
-                    <common:assertion-origin-orcid>
-                        <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                        <common:path>0000-0001-6816-8350</common:path>
-                        <common:host>orcid.org</common:host>
-                    </common:assertion-origin-orcid>
-                </common:source>
-                <common:external-id-type>ResearcherID</common:external-id-type>
-                <common:external-id-value>D-1929-2018</common:external-id-value>
-                <common:external-id-url>http://www.researcherid.com/rid/D-1929-2018</common:external-id-url>
-                <common:external-id-relationship>self</common:external-id-relationship>
-            </external-identifier:external-identifier>
-        </external-identifier:external-identifiers>
-    </person:person>
-    <activities:activities-summary path="/0000-0001-6816-8350/activities">
-        <common:last-modified-date>2023-12-02T13:28:26.051Z</common:last-modified-date>
-        <activities:distinctions path="/0000-0001-6816-8350/distinctions"/>
-        <activities:educations path="/0000-0001-6816-8350/educations">
-            <common:last-modified-date>2018-02-13T02:33:38.225Z</common:last-modified-date>
-            <activities:affiliation-group>
-                <common:last-modified-date>2016-02-09T06:55:21.838Z</common:last-modified-date>
-                <common:external-ids/>
-                <education:education-summary put-code="1549986" display-index="0" path="/0000-0001-6816-8350/education/1549986" visibility="public">
-                    <common:created-date>2016-02-09T06:54:39.199Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T06:55:21.838Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Management</common:department-name>
-                    <common:role-title>PhD</common:role-title>
-                    <common:start-date>
-                        <common:year>2021</common:year>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2022</common:year>
-                        <common:month>02</common:month>
-                        <common:day>12</common:day>
-                    </common:end-date>
-                    <common:start-date>
-                        <common:year>2003</common:year>
-                        <common:month>03</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2007</common:year>
-                        <common:month>03</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>University of New England</common:name>
-                        <common:address>
-                            <common:city>Armidale</common:city>
-                            <common:region>NSW</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>1319</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </education:education-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2018-02-13T02:33:38.225Z</common:last-modified-date>
-                <common:external-ids/>
-                <education:education-summary put-code="1549990" display-index="0" path="/0000-0001-6816-8350/education/1549990" visibility="public">
-                    <common:created-date>2016-02-09T06:57:04.181Z</common:created-date>
-                    <common:last-modified-date>2018-02-13T02:33:38.225Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Psychology</common:department-name>
-                    <common:role-title>Master of Science (Industrial and Organizational) Psychology</common:role-title>
-                    <common:start-date>
-                        <common:year>1998</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2000</common:year>
-                        <common:month>01</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Universitas Gadjah Mada</common:name>
-                        <common:address>
-                            <common:city>Yogyakarta</common:city>
-                            <common:region>Daerah Istimewa Yogyakart</common:region>
-                            <common:country>ID</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>59166</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </education:education-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2018-02-13T02:33:35.821Z</common:last-modified-date>
-                <common:external-ids/>
-                <education:education-summary put-code="1549998" display-index="0" path="/0000-0001-6816-8350/education/1549998" visibility="public">
-                    <common:created-date>2016-02-09T06:58:59.869Z</common:created-date>
-                    <common:last-modified-date>2018-02-13T02:33:35.821Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Education</common:department-name>
-                    <common:role-title>Bachelor of Education (Honors)</common:role-title>
-                    <common:start-date>
-                        <common:year>1988</common:year>
-                        <common:month>03</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>1991</common:year>
-                        <common:month>03</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Kenyatta University</common:name>
-                        <common:address>
-                            <common:city>Nairobi</common:city>
-                            <common:region>Nairobi</common:region>
-                            <common:country>KE</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>107864</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </education:education-summary>
-            </activities:affiliation-group>
-        </activities:educations>
-        <activities:employments path="/0000-0001-6816-8350/employments">
-            <common:last-modified-date>2023-12-02T13:28:26.051Z</common:last-modified-date>
-            <activities:affiliation-group>
-                <common:last-modified-date>2023-12-02T13:28:26.051Z</common:last-modified-date>
-                <common:external-ids/>
-                <employment:employment-summary put-code="21884863" display-index="1" path="/0000-0001-6816-8350/employment/21884863" visibility="public">
-                    <common:created-date>2023-12-02T13:28:26.051Z</common:created-date>
-                    <common:last-modified-date>2023-12-02T13:28:26.051Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Management</common:department-name>
-                    <common:role-title>Associate Professor in Management</common:role-title>
-                    <common:start-date>
-                        <common:year>2023</common:year>
-                        <common:month>08</common:month>
-                        <common:day>20</common:day>
-                    </common:start-date>
-                    <common:organization>
-                        <common:name>Alfaisal University</common:name>
-                        <common:address>
-                            <common:city>Riyadh</common:city>
-                            <common:country>SA</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>https://ror.org/00cdrtq48</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>ROR</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                    <common:url>https://faculty.alfaisal.edu/user/mmuchiri</common:url>
-                </employment:employment-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2016-02-09T07:00:06.052Z</common:last-modified-date>
-                <common:external-ids/>
-                <employment:employment-summary put-code="1550002" display-index="0" path="/0000-0001-6816-8350/employment/1550002" visibility="public">
-                    <common:created-date>2016-02-09T07:00:06.052Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T07:00:06.052Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Management</common:department-name>
-                    <common:role-title>Senior Lecturer</common:role-title>
-                    <common:start-date>
-                        <common:year>2014</common:year>
-                        <common:month>02</common:month>
-                    </common:start-date>
-                    <common:organization>
-                        <common:name>RMIT University</common:name>
-                        <common:address>
-                            <common:city>Melbourne</common:city>
-                            <common:region>VIC</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>5376</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </employment:employment-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2016-02-09T07:01:08.398Z</common:last-modified-date>
-                <common:external-ids/>
-                <employment:employment-summary put-code="1550007" display-index="0" path="/0000-0001-6816-8350/employment/1550007" visibility="public">
-                    <common:created-date>2016-02-09T07:01:08.398Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T07:01:08.398Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Management</common:department-name>
-                    <common:role-title>Senior Lecturer in Human Resource Management</common:role-title>
-                    <common:start-date>
-                        <common:year>2010</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2014</common:year>
-                        <common:month>02</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Central Queensland University</common:name>
-                        <common:address>
-                            <common:city>Rockhampton</common:city>
-                            <common:region>QLD</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>273488</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </employment:employment-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2016-02-09T07:01:47.814Z</common:last-modified-date>
-                <common:external-ids/>
-                <employment:employment-summary put-code="1550010" display-index="0" path="/0000-0001-6816-8350/employment/1550010" visibility="public">
-                    <common:created-date>2016-02-09T07:01:47.814Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T07:01:47.814Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Management</common:department-name>
-                    <common:role-title>Lecturer in Management</common:role-title>
-                    <common:start-date>
-                        <common:year>2007</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2010</common:year>
-                        <common:month>01</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Central Queensland University</common:name>
-                        <common:address>
-                            <common:city>Rockhampton</common:city>
-                            <common:region>QLD</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>273488</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </employment:employment-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2018-02-13T02:33:13.213Z</common:last-modified-date>
-                <common:external-ids/>
-                <employment:employment-summary put-code="1550017" display-index="0" path="/0000-0001-6816-8350/employment/1550017" visibility="public">
-                    <common:created-date>2016-02-09T07:03:42.180Z</common:created-date>
-                    <common:last-modified-date>2018-02-13T02:33:13.213Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Human Resource Development Division</common:department-name>
-                    <common:role-title>Chief Human Resource Development Officer</common:role-title>
-                    <common:start-date>
-                        <common:year>2005</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2007</common:year>
-                        <common:month>01</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Government of Kenya Directorate of Personnel Management</common:name>
-                        <common:address>
-                            <common:city>Nairobi</common:city>
-                            <common:region>Nairobi</common:region>
-                            <common:country>KE</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>360256</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </employment:employment-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2016-02-09T07:05:02.300Z</common:last-modified-date>
-                <common:external-ids/>
-                <employment:employment-summary put-code="1550020" display-index="0" path="/0000-0001-6816-8350/employment/1550020" visibility="public">
-                    <common:created-date>2016-02-09T07:05:02.300Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T07:05:02.300Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Human Resource Development Division</common:department-name>
-                    <common:role-title>Human Resource Development Officer</common:role-title>
-                    <common:start-date>
-                        <common:year>2001</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2005</common:year>
-                        <common:month>01</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Government of Kenya Directorate of Personnel Management</common:name>
-                        <common:address>
-                            <common:city>Nairobi</common:city>
-                            <common:region>Nairobi</common:region>
-                            <common:country>KE</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>360256</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </employment:employment-summary>
-            </activities:affiliation-group>
-            <activities:affiliation-group>
-                <common:last-modified-date>2016-02-09T07:36:52.398Z</common:last-modified-date>
-                <common:external-ids/>
-                <employment:employment-summary put-code="1550050" display-index="0" path="/0000-0001-6816-8350/employment/1550050" visibility="public">
-                    <common:created-date>2016-02-09T07:36:52.398Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T07:36:52.398Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <common:department-name>Public Sector Management Technical Assistance Project</common:department-name>
-                    <common:role-title>Project Coordinator for Development Learning Centre</common:role-title>
-                    <common:start-date>
-                        <common:year>2002</common:year>
-                        <common:month>08</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2003</common:year>
-                        <common:month>03</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Government of Kenya Directorate of Personnel Management</common:name>
-                        <common:address>
-                            <common:city>Nairobi</common:city>
-                            <common:region>Nairobi</common:region>
-                            <common:country>KE</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>360256</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </employment:employment-summary>
-            </activities:affiliation-group>
-        </activities:employments>
-        <activities:fundings path="/0000-0001-6816-8350/fundings">
-            <common:last-modified-date>2016-02-09T09:05:27.100Z</common:last-modified-date>
-            <activities:group>
-                <common:last-modified-date>2016-02-09T09:05:27.100Z</common:last-modified-date>
-                <common:external-ids/>
-                <funding:funding-summary put-code="150520" path="/0000-0001-6816-8350/funding/150520" visibility="public" display-index="6">
-                    <common:created-date>2016-02-09T09:05:27.100Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T09:05:27.100Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <funding:title>
-                        <common:title>A cross-country examination of Employee Wellbeing,  Leadership, High Performance Work Systems and Innovative Behaviours</common:title>
-                    </funding:title>
-                    <funding:type>grant</funding:type>
-                    <common:start-date>
-                        <common:year>2016</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2016</common:year>
-                        <common:month>12</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>RMIT University</common:name>
-                        <common:address>
-                            <common:city>VIC</common:city>
-                            <common:region>VIC</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>http://dx.doi.org/10.13039/501100001780</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>FUNDREF</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </funding:funding-summary>
-            </activities:group>
-            <activities:group>
-                <common:last-modified-date>2016-02-09T09:03:51.641Z</common:last-modified-date>
-                <common:external-ids/>
-                <funding:funding-summary put-code="150518" path="/0000-0001-6816-8350/funding/150518" visibility="public" display-index="5">
-                    <common:created-date>2016-02-09T09:03:51.641Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T09:03:51.641Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <funding:title>
-                        <common:title>Leading Safe and Thriving Organisations: An Investigation of the Relationships between Leadership, Thriving Behaviour, Authentic Followership and Safety Climate in an Australian Multinational Enterprise</common:title>
-                    </funding:title>
-                    <funding:type>grant</funding:type>
-                    <common:start-date>
-                        <common:year>2015</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2015</common:year>
-                        <common:month>12</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>RMIT University</common:name>
-                        <common:address>
-                            <common:city>VIC</common:city>
-                            <common:region>VIC</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>http://dx.doi.org/10.13039/501100001780</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>FUNDREF</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </funding:funding-summary>
-            </activities:group>
-            <activities:group>
-                <common:last-modified-date>2016-02-09T09:02:28.297Z</common:last-modified-date>
-                <common:external-ids/>
-                <funding:funding-summary put-code="150516" path="/0000-0001-6816-8350/funding/150516" visibility="public" display-index="4">
-                    <common:created-date>2016-02-09T09:02:28.297Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T09:02:28.297Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <funding:title>
-                        <common:title>A multilevel, cross-country examination of leadership, followership and innovative behaviours in Australia and Indonesia. </common:title>
-                    </funding:title>
-                    <funding:type>grant</funding:type>
-                    <common:start-date>
-                        <common:year>2015</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2015</common:year>
-                        <common:month>12</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>RMIT University</common:name>
-                        <common:address>
-                            <common:city>VIC</common:city>
-                            <common:region>VIC</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>http://dx.doi.org/10.13039/501100001780</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>FUNDREF</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </funding:funding-summary>
-            </activities:group>
-            <activities:group>
-                <common:last-modified-date>2016-02-09T09:00:51.749Z</common:last-modified-date>
-                <common:external-ids/>
-                <funding:funding-summary put-code="150514" path="/0000-0001-6816-8350/funding/150514" visibility="public" display-index="3">
-                    <common:created-date>2016-02-09T09:00:51.749Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T09:00:51.749Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <funding:title>
-                        <common:title>Workplace safety and positive leadership: Exploring relationships between leader behaviours, organisational climate, safety climate, safety citizenship behaviours and innovative behaviours within city councils in Victoria </common:title>
-                    </funding:title>
-                    <funding:type>grant</funding:type>
-                    <common:start-date>
-                        <common:year>2014</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2014</common:year>
-                        <common:month>12</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>RMIT University</common:name>
-                        <common:address>
-                            <common:city>VIC</common:city>
-                            <common:region>VIC</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                        <common:disambiguated-organization>
-                            <common:disambiguated-organization-identifier>http://dx.doi.org/10.13039/501100001780</common:disambiguated-organization-identifier>
-                            <common:disambiguation-source>FUNDREF</common:disambiguation-source>
-                        </common:disambiguated-organization>
-                    </common:organization>
-                </funding:funding-summary>
-            </activities:group>
-            <activities:group>
-                <common:last-modified-date>2016-02-09T07:46:44.919Z</common:last-modified-date>
-                <common:external-ids/>
-                <funding:funding-summary put-code="150485" path="/0000-0001-6816-8350/funding/150485" visibility="public" display-index="0">
-                    <common:created-date>2016-02-09T07:46:44.919Z</common:created-date>
-                    <common:last-modified-date>2016-02-09T07:46:44.919Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-orcid>
-                            <common:uri>https://orcid.org/0000-0001-6816-8350</common:uri>
-                            <common:path>0000-0001-6816-8350</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-orcid>
-                    </common:source>
-                    <funding:title>
-                        <common:title>Sustainable Business Model for Central Queensland Regional Information Systems.</common:title>
-                    </funding:title>
-                    <funding:type>grant</funding:type>
-                    <common:start-date>
-                        <common:year>2008</common:year>
-                        <common:month>01</common:month>
-                    </common:start-date>
-                    <common:end-date>
-                        <common:year>2008</common:year>
-                        <common:month>12</common:month>
-                    </common:end-date>
-                    <common:organization>
-                        <common:name>Department of Local Government, Planning, Sport and Recreation, Queensland, Australia </common:name>
-                        <common:address>
-                            <common:city>Rockhampton</common:city>
-                            <common:region>Central Queensland</common:region>
-                            <common:country>AU</common:country>
-                        </common:address>
-                    </common:organization>
-                </funding:funding-summary>
-            </activities:group>
-        </activities:fundings>
-        <activities:invited-positions path="/0000-0001-6816-8350/invited-positions"/>
-        <activities:memberships path="/0000-0001-6816-8350/memberships"/>
-        <activities:peer-reviews path="/0000-0001-6816-8350/peer-reviews">
-            <common:last-modified-date>2023-05-31T05:53:44.542Z</common:last-modified-date>
-            <activities:group>
-                <common:last-modified-date>2023-05-31T05:53:44.542Z</common:last-modified-date>
-                <common:external-ids>
-                    <common:external-id>
-                        <common:external-id-type>peer-review</common:external-id-type>
-                        <common:external-id-value>issn:0167-4544</common:external-id-value>
-                    </common:external-id>
-                </common:external-ids>
-                <activities:peer-review-group>
-                    <common:last-modified-date>2023-02-28T06:51:52.426Z</common:last-modified-date>
-                    <common:external-ids>
-                        <common:external-id>
-                            <common:external-id-type>source-work-id</common:external-id-type>
-                            <common:external-id-value>c9bdf086-cfee-4cd9-bcfb-268cc5423248</common:external-id-value>
-                            <common:external-id-normalized transient="true">c9bdf086-cfee-4cd9-bcfb-268cc5423248</common:external-id-normalized>
-                            <common:external-id-url></common:external-id-url>
-                            <common:external-id-relationship>self</common:external-id-relationship>
-                        </common:external-id>
-                    </common:external-ids>
-                    <peer-review:peer-review-summary put-code="8741329" path="/0000-0001-6816-8350/peer-review/8741329" visibility="public" display-index="0">
-                        <common:created-date>2023-02-28T06:51:52.426Z</common:created-date>
-                        <common:last-modified-date>2023-02-28T06:51:52.426Z</common:last-modified-date>
-                        <common:source>
-                            <common:source-client-id>
-                                <common:uri>https://orcid.org/client/APP-945VYTN20C7BZXYT</common:uri>
-                                <common:path>APP-945VYTN20C7BZXYT</common:path>
-                                <common:host>orcid.org</common:host>
-                            </common:source-client-id>
-                            <common:source-name>Springer Nature @ Editorial Manager</common:source-name>
-                        </common:source>
-                        <peer-review:reviewer-role>reviewer</peer-review:reviewer-role>
-                        <common:external-ids>
-                            <common:external-id>
-                                <common:external-id-type>source-work-id</common:external-id-type>
-                                <common:external-id-value>c9bdf086-cfee-4cd9-bcfb-268cc5423248</common:external-id-value>
-                                <common:external-id-normalized transient="true">c9bdf086-cfee-4cd9-bcfb-268cc5423248</common:external-id-normalized>
-                                <common:external-id-url></common:external-id-url>
-                                <common:external-id-relationship>self</common:external-id-relationship>
-                            </common:external-id>
-                        </common:external-ids>
-                        <peer-review:review-type>review</peer-review:review-type>
-                        <peer-review:completion-date>
-                            <common:year>2023</common:year>
-                        </peer-review:completion-date>
-                        <peer-review:review-group-id>issn:0167-4544</peer-review:review-group-id>
-                        <peer-review:convening-organization>
-                            <common:name>Springer Nature</common:name>
-                            <common:address>
-                                <common:city>New York</common:city>
-                                <common:country>US</common:country>
-                            </common:address>
-                            <common:disambiguated-organization>
-                                <common:disambiguated-organization-identifier>grid.467660.5</common:disambiguated-organization-identifier>
-                                <common:disambiguation-source>GRID</common:disambiguation-source>
-                            </common:disambiguated-organization>
-                        </peer-review:convening-organization>
-                    </peer-review:peer-review-summary>
-                </activities:peer-review-group>
-                <activities:peer-review-group>
-                    <common:last-modified-date>2023-05-31T05:53:44.542Z</common:last-modified-date>
-                    <common:external-ids>
-                        <common:external-id>
-                            <common:external-id-type>source-work-id</common:external-id-type>
-                            <common:external-id-value>c442840b-5807-459d-802a-303d8ba4e25e</common:external-id-value>
-                            <common:external-id-normalized transient="true">c442840b-5807-459d-802a-303d8ba4e25e</common:external-id-normalized>
-                            <common:external-id-url></common:external-id-url>
-                            <common:external-id-relationship>self</common:external-id-relationship>
-                        </common:external-id>
-                    </common:external-ids>
-                    <peer-review:peer-review-summary put-code="9680570" path="/0000-0001-6816-8350/peer-review/9680570" visibility="public" display-index="0">
-                        <common:created-date>2023-05-31T05:53:44.542Z</common:created-date>
-                        <common:last-modified-date>2023-05-31T05:53:44.542Z</common:last-modified-date>
-                        <common:source>
-                            <common:source-client-id>
-                                <common:uri>https://orcid.org/client/APP-945VYTN20C7BZXYT</common:uri>
-                                <common:path>APP-945VYTN20C7BZXYT</common:path>
-                                <common:host>orcid.org</common:host>
-                            </common:source-client-id>
-                            <common:source-name>Springer Nature @ Editorial Manager</common:source-name>
-                        </common:source>
-                        <peer-review:reviewer-role>reviewer</peer-review:reviewer-role>
-                        <common:external-ids>
-                            <common:external-id>
-                                <common:external-id-type>source-work-id</common:external-id-type>
-                                <common:external-id-value>c442840b-5807-459d-802a-303d8ba4e25e</common:external-id-value>
-                                <common:external-id-normalized transient="true">c442840b-5807-459d-802a-303d8ba4e25e</common:external-id-normalized>
-                                <common:external-id-url></common:external-id-url>
-                                <common:external-id-relationship>self</common:external-id-relationship>
-                            </common:external-id>
-                        </common:external-ids>
-                        <peer-review:review-type>review</peer-review:review-type>
-                        <peer-review:completion-date>
-                            <common:year>2023</common:year>
-                        </peer-review:completion-date>
-                        <peer-review:review-group-id>issn:0167-4544</peer-review:review-group-id>
-                        <peer-review:convening-organization>
-                            <common:name>Springer Nature</common:name>
-                            <common:address>
-                                <common:city>New York</common:city>
-                                <common:country>US</common:country>
-                            </common:address>
-                            <common:disambiguated-organization>
-                                <common:disambiguated-organization-identifier>grid.467660.5</common:disambiguated-organization-identifier>
-                                <common:disambiguation-source>GRID</common:disambiguation-source>
-                            </common:disambiguated-organization>
-                        </peer-review:convening-organization>
-                    </peer-review:peer-review-summary>
-                </activities:peer-review-group>
-            </activities:group>
-        </activities:peer-reviews>
-        <activities:qualifications path="/0000-0001-6816-8350/qualifications"/>
-        <activities:research-resources path="/0000-0001-6816-8350/research-resources"/>
-        <activities:services path="/0000-0001-6816-8350/services"/>
-        <activities:works path="/0000-0001-6816-8350/works">
-            <common:last-modified-date>2023-06-02T20:12:00.338Z</common:last-modified-date>
-            <activities:group>
-                <common:last-modified-date>2023-06-02T20:12:00.338Z</common:last-modified-date>
-                <common:external-ids>
-                    <common:external-id>
-                        <common:external-id-type>doi</common:external-id-type>
-                        <common:external-id-value>10.4337/9781800881945.00020</common:external-id-value>
-                        <common:external-id-normalized transient="true">10.4337/9781800881945.00020</common:external-id-normalized>
-                        <common:external-id-url>https://doi.org/10.4337/9781800881945.00020</common:external-id-url>
-                        <common:external-id-relationship>self</common:external-id-relationship>
-                    </common:external-id>
-                </common:external-ids>
-                <work:work-summary put-code="134891279" path="/0000-0001-6816-8350/work/134891279" visibility="public" display-index="0">
-                    <common:created-date>2023-05-11T21:05:54.188Z</common:created-date>
-                    <common:last-modified-date>2023-06-02T20:12:00.338Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-client-id>
-                            <common:uri>https://orcid.org/client/0000-0001-9884-1913</common:uri>
-                            <common:path>0000-0001-9884-1913</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-client-id>
-                        <common:source-name>Crossref</common:source-name>
-                    </common:source>
-                    <work:title>
-                        <common:title>Ethical leadership as workplace innovation and enabler for employee commitment and innovative work behaviours in Vietnam</common:title>
-                    </work:title>
-                    <common:external-ids>
-                        <common:external-id>
-                            <common:external-id-type>doi</common:external-id-type>
-                            <common:external-id-value>10.4337/9781800881945.00020</common:external-id-value>
-                            <common:external-id-normalized transient="true">10.4337/9781800881945.00020</common:external-id-normalized>
-                            <common:external-id-url>https://doi.org/10.4337/9781800881945.00020</common:external-id-url>
-                            <common:external-id-relationship>self</common:external-id-relationship>
-                        </common:external-id>
-                    </common:external-ids>
-                    <common:url>https://doi.org/10.4337/9781800881945.00020</common:url>
-                    <work:type>book-chapter</work:type>
-                    <common:publication-date>
-                        <common:year>2023</common:year>
-                        <common:month>05</common:month>
-                        <common:day>26</common:day>
-                    </common:publication-date>
-                </work:work-summary>
-            </activities:group>
-            <activities:group>
-                <common:last-modified-date>2023-03-01T11:30:31.972Z</common:last-modified-date>
-                <common:external-ids>
-                    <common:external-id>
-                        <common:external-id-type>doi</common:external-id-type>
-                        <common:external-id-value>10.1007/s10551-022-05081-6</common:external-id-value>
-                        <common:external-id-normalized transient="true">10.1007/s10551-022-05081-6</common:external-id-normalized>
-                        <common:external-id-url>https://doi.org/10.1007/s10551-022-05081-6</common:external-id-url>
-                        <common:external-id-relationship>self</common:external-id-relationship>
-                    </common:external-id>
-                </common:external-ids>
-                <work:work-summary put-code="110048777" path="/0000-0001-6816-8350/work/110048777" visibility="public" display-index="0">
-                    <common:created-date>2022-03-18T03:36:55.927Z</common:created-date>
-                    <common:last-modified-date>2023-03-01T11:30:31.972Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-client-id>
-                            <common:uri>https://orcid.org/client/0000-0001-9884-1913</common:uri>
-                            <common:path>0000-0001-9884-1913</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-client-id>
-                        <common:source-name>Crossref</common:source-name>
-                    </common:source>
-                    <work:title>
-                        <common:title>Unethical Leadership: Review, Synthesis and Directions for Future Research</common:title>
-                    </work:title>
-                    <common:external-ids>
-                        <common:external-id>
-                            <common:external-id-type>doi</common:external-id-type>
-                            <common:external-id-value>10.1007/s10551-022-05081-6</common:external-id-value>
-                            <common:external-id-normalized transient="true">10.1007/s10551-022-05081-6</common:external-id-normalized>
-                            <common:external-id-url>https://doi.org/10.1007/s10551-022-05081-6</common:external-id-url>
-                            <common:external-id-relationship>self</common:external-id-relationship>
-                        </common:external-id>
-                    </common:external-ids>
-                    <common:url>https://doi.org/10.1007/s10551-022-05081-6</common:url>
-                    <work:type>journal-article</work:type>
-                    <common:publication-date>
-                        <common:year>2023</common:year>
-                        <common:month>03</common:month>
-                    </common:publication-date>
-                    <work:journal-title>Journal of Business Ethics</work:journal-title>
-                </work:work-summary>
-            </activities:group>
-            <activities:group>
-                <common:last-modified-date>2022-05-28T18:16:16.575Z</common:last-modified-date>
-                <common:external-ids>
-                    <common:external-id>
-                        <common:external-id-type>doi</common:external-id-type>
-                        <common:external-id-value>10.1017/jmo.2019.33</common:external-id-value>
-                        <common:external-id-normalized transient="true">10.1017/jmo.2019.33</common:external-id-normalized>
-                        <common:external-id-url>https://doi.org/10.1017/jmo.2019.33</common:external-id-url>
-                        <common:external-id-relationship>self</common:external-id-relationship>
-                    </common:external-id>
-                </common:external-ids>
-                <work:work-summary put-code="57272180" path="/0000-0001-6816-8350/work/57272180" visibility="public" display-index="0">
-                    <common:created-date>2019-05-10T07:23:14.608Z</common:created-date>
-                    <common:last-modified-date>2022-05-28T18:16:16.575Z</common:last-modified-date>
-                    <common:source>
-                        <common:source-client-id>
-                            <common:uri>https://orcid.org/client/0000-0001-9884-1913</common:uri>
-                            <common:path>0000-0001-9884-1913</common:path>
-                            <common:host>orcid.org</common:host>
-                        </common:source-client-id>
-                        <common:source-name>Crossref</common:source-name>
-                    </common:source>
-                    <work:title>
-                        <common:title>And now for something completely different: Reframing social processes of leadership theory using positive organisational behaviour</common:title>
-                    </work:title>
-                    <common:external-ids>
-                        <common:external-id>
-                            <common:external-id-type>doi</common:external-id-type>
-                            <common:external-id-value>10.1017/jmo.2019.33</common:external-id-value>
-                            <common:external-id-normalized transient="true">10.1017/jmo.2019.33</common:external-id-normalized>
-                            <common:external-id-url>https://doi.org/10.1017/jmo.2019.33</common:external-id-url>
-                            <common:external-id-relationship>self</common:external-id-relationship>
-                        </common:external-id>
-                    </common:external-ids>
-                    <common:url>https://doi.org/10.1017/jmo.2019.33</common:url>
-                    <work:type>journal-article</work:type>
-                    <common:publication-date>
-                        <common:year>2019</common:year>
-                        <common:month>05</common:month>
-                        <common:day>09</common:day>
-                    </common:publication-date>
-                    <work:journal-title>Journal of Management &amp; Organization</work:journal-title>
-                </work:work-summary>
-            </activities:group>
-        </activities:works>
-    </activities:activities-summary>
-</record:record>

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/update_employments.xml

@@ -1,60 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<activities:employments path="/0000-0002-0456-1185/employments" xmlns:internal="http://www.orcid.org/ns/internal" xmlns:education="http://www.orcid.org/ns/education" xmlns:distinction="http://www.orcid.org/ns/distinction" xmlns:deprecated="http://www.orcid.org/ns/deprecated" xmlns:other-name="http://www.orcid.org/ns/other-name" xmlns:membership="http://www.orcid.org/ns/membership" xmlns:error="http://www.orcid.org/ns/error" xmlns:common="http://www.orcid.org/ns/common" xmlns:record="http://www.orcid.org/ns/record" xmlns:personal-details="http://www.orcid.org/ns/personal-details" xmlns:keyword="http://www.orcid.org/ns/keyword" xmlns:email="http://www.orcid.org/ns/email" xmlns:external-identifier="http://www.orcid.org/ns/external-identifier" xmlns:funding="http://www.orcid.org/ns/funding" xmlns:preferences="http://www.orcid.org/ns/preferences" xmlns:address="http://www.orcid.org/ns/address" xmlns:invited-position="http://www.orcid.org/ns/invited-position" xmlns:work="http://www.orcid.org/ns/work" xmlns:history="http://www.orcid.org/ns/history" xmlns:employment="http://www.orcid.org/ns/employment" xmlns:qualification="http://www.orcid.org/ns/qualification" xmlns:service="http://www.orcid.org/ns/service" xmlns:person="http://www.orcid.org/ns/person" xmlns:activities="http://www.orcid.org/ns/activities" xmlns:researcher-url="http://www.orcid.org/ns/researcher-url" xmlns:peer-review="http://www.orcid.org/ns/peer-review" xmlns:bulk="http://www.orcid.org/ns/bulk" xmlns:research-resource="http://www.orcid.org/ns/research-resource">
-    <common:last-modified-date>2024-01-07T23:59:38.869Z</common:last-modified-date>
-    <activities:affiliation-group>
-        <common:last-modified-date>2024-01-07T23:59:38.869Z</common:last-modified-date>
-        <common:external-ids/>
-        <employment:employment-summary put-code="22127142" display-index="0" path="/0000-0002-0456-1185/employment/22127142" visibility="public">
-            <common:created-date>2024-01-07T23:59:38.869Z</common:created-date>
-            <common:last-modified-date>2024-01-07T23:59:38.869Z</common:last-modified-date>
-            <common:source>
-                <common:source-client-id>
-                    <common:uri>https://orcid.org/client/APP-N0TAO4G9BBK9PWHT</common:uri>
-                    <common:path>APP-N0TAO4G9BBK9PWHT</common:path>
-                    <common:host>orcid.org</common:host>
-                </common:source-client-id>
-                <common:source-name>Tampere University</common:source-name>
-            </common:source>
-            <common:organization>
-                <common:name>Tampere University</common:name>
-                <common:address>
-                    <common:city>Tampere</common:city>
-                    <common:country>FI</common:country>
-                </common:address>
-                <common:disambiguated-organization>
-                    <common:disambiguated-organization-identifier>https://ror.org/033003e23</common:disambiguated-organization-identifier>
-                    <common:disambiguation-source>ROR</common:disambiguation-source>
-                </common:disambiguated-organization>
-            </common:organization>
-        </employment:employment-summary>
-    </activities:affiliation-group>
-    <activities:affiliation-group>
-        <common:last-modified-date>2019-01-03T17:00:05.658Z</common:last-modified-date>
-        <common:external-ids/>
-        <employment:employment-summary put-code="3291239" display-index="1" path="/0000-0002-0456-1185/employment/3291239" visibility="public">
-            <common:created-date>2017-02-26T04:46:20.917Z</common:created-date>
-            <common:last-modified-date>2019-01-03T17:00:05.658Z</common:last-modified-date>
-            <common:source>
-                <common:source-orcid>
-                    <common:uri>https://orcid.org/0000-0002-0456-1185</common:uri>
-                    <common:path>0000-0002-0456-1185</common:path>
-                    <common:host>orcid.org</common:host>
-                </common:source-orcid>
-                <common:source-name>Tiina Manninen</common:source-name>
-            </common:source>
-            <common:department-name> Faculty of Medicine and Health Technology</common:department-name>
-            <common:role-title>Academy Research Fellow</common:role-title>
-            <common:organization>
-                <common:name>Tampere University</common:name>
-                <common:address>
-                    <common:city>Tampere</common:city>
-                    <common:country>FI</common:country>
-                </common:address>
-                <common:disambiguated-organization>
-                    <common:disambiguated-organization-identifier>7839</common:disambiguated-organization-identifier>
-                    <common:disambiguation-source>RINGGOLD</common:disambiguation-source>
-                </common:disambiguated-organization>
-            </common:organization>
-        </employment:employment-summary>
-    </activities:affiliation-group>
-</activities:employments>

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/plugin/base/record_wrong_1.xml

@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>                                          
-<record>
-  <header xmlns="http://www.openarchives.org/OAI/2.0/">
-    <identifier>ftunivminnesdc:oai:conservancy.umn.edu:11299/109914</identifier>
-    <datestamp>2023-07-18T20:05:40Z</datestamp>
-  </header>
-  <metadata xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:base_dc="http://oai.base-search.net/base_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dc="http://purl.org/dc/elements/1.1/">
-    <base_dc:dc xsi:schemaLocation="http://oai.base-search.net/base_dc/ http://oai.base-search.net/base_dc/base_dc.xsd">
-      <base_dc:global_id>ftunivminnesdc:oai:conservancy.umn.edu:11299/109914</base_dc:global_id>
-      <base_dc:continent>cna</base_dc:continent>
-      <base_dc:country>us</base_dc:country>
-      <base_dc:collection opendoar_id="1008">ftunivminnesdc</base_dc:collection>
-      <base_dc:collname>University of Minnesota Digital Conservancy</base_dc:collname>
-      <dc:title>An Experimental Investigation of the influence of an air bubble layer on radiated noise and surface pressure fluctuations in a turbulent boundary layer</dc:title>
-      <dc:creator>Killen, John M.</dc:creator>
-      <dc:subject>Boundary layer noise</dc:subject>
-      <dc:subject>Kinetic energy dissipation</dc:subject>
-      <dc:subject>Air bubble layer</dc:subject>
-      <dc:subject>Maximum stable bubble size</dc:subject>
-      <dc:subject>bubble noise spectra</dc:subject>
-      <dc:description>between the noise spectra of a layer of air bubbles in a turbulent flow and' a maximum stable bubble size which can exist in the same flow. An air bubbie layer with individual bubble sizes greater than a maximum stable size was introduced into the boundary layer of water flowing along a smooth flat plate. It was found that the maximum stable bubble size was related to the turbulent kinetic energy dissipation as had been previously shown by other investigators ; Office of Naval Research Arlington, Virginia 22217</dc:description>
-      <dc:date>1981-09-31T21:42:52Z</dc:date>
-      <base_dc:year>1981</base_dc:year>
-      <dc:type>Report</dc:type>
-      <base_dc:typenorm>14</base_dc:typenorm>
-      <dc:identifier>http://purl.umn.edu/109914</dc:identifier>
-      <base_dc:link>http://purl.umn.edu/109914</base_dc:link>
-      <dc:language>en_US</dc:language>
-      <dc:relation>Project Reports</dc:relation>
-      <dc:relation>202</dc:relation>
-      <dc:relation>http://purl.umn.edu/109914</dc:relation>
-      <base_dc:oa>2</base_dc:oa>
-      <base_dc:lang>eng</base_dc:lang>
-    </base_dc:dc>
-  </metadata>
-</record>

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pdb_dump

@@ -1,15 +1,44 @@
-{"pdb": "1CW0", "title": "crystal structure analysis of very short patch repair (vsr) endonuclease in complex with a duplex dna", "authors": ["S.E.Tsutakawa", "H.Jingami", "K.Morikawa"], "doi": "10.1016/S0092-8674(00)81550-0", "pmid": "10612397"}
-{"pdb": "2CWW", "title": "crystal structure of thermus thermophilus ttha1280, a putative sam- dependent rna methyltransferase, in complex with s-adenosyl-l- homocysteine", "authors": ["A.A.Pioszak", "K.Murayama", "N.Nakagawa", "A.Ebihara", "S.Kuramitsu", "M.Shirouzu", "S.Yokoyama", "Riken Structural Genomics/proteomics Initiative (Rsgi)"], "doi": "10.1107/S1744309105029842", "pmid": "16511182"}
-{"pdb": "6CWE", "title": "structure of alpha-gsa[8,6p] bound by cd1d and in complex with the va14vb8.2 tcr", "authors": ["J.Wang", "D.Zajonc"], "doi": null, "pmid": null}
-{"pdb": "5CWS", "title": "crystal structure of the intact chaetomium thermophilum nsp1-nup49- nup57 channel nucleoporin heterotrimer bound to its nic96 nuclear pore complex attachment site", "authors": ["C.J.Bley", "S.Petrovic", "M.Paduch", "V.Lu", "A.A.Kossiakoff", "A.Hoelz"], "doi": "10.1126/SCIENCE.AAC9176", "pmid": "26316600"}
-{"pdb": "5CWE", "title": "structure of cyp107l2 from streptomyces avermitilis with lauric acid", "authors": ["T.-V.Pham", "S.-H.Han", "J.-H.Kim", "D.-H.Kim", "L.-W.Kang"], "doi": null, "pmid": null}
-{"pdb": "7CW4", "title": "acetyl-coa acetyltransferase from bacillus cereus atcc 14579", "authors": ["J.Hong", "K.J.Kim"], "doi": "10.1016/J.BBRC.2020.09.048", "pmid": "32972748"}
-{"pdb": "2CWP", "title": "crystal structure of metrs related protein from pyrococcus horikoshii", "authors": ["K.Murayama", "M.Kato-Murayama", "M.Shirouzu", "S.Yokoyama", "Riken StructuralGenomics/proteomics Initiative (Rsgi)"], "doi": null, "pmid": null}
-{"pdb": "2CW7", "title": "crystal structure of intein homing endonuclease ii", "authors": ["H.Matsumura", "H.Takahashi", "T.Inoue", "H.Hashimoto", "M.Nishioka", "S.Fujiwara", "M.Takagi", "T.Imanaka", "Y.Kai"], "doi": "10.1002/PROT.20858", "pmid": "16493661"}
-{"pdb": "1CWU", "title": "brassica napus enoyl acp reductase a138g mutant complexed with nad+ and thienodiazaborine", "authors": ["A.Roujeinikova", "J.B.Rafferty", "D.W.Rice"], "doi": "10.1074/JBC.274.43.30811", "pmid": "10521472"}
-{"pdb": "3CWN", "title": "escherichia coli transaldolase b mutant f178y", "authors": ["T.Sandalova", "G.Schneider", "A.Samland"], "doi": "10.1074/JBC.M803184200", "pmid": "18687684"}
-{"pdb": "1CWL", "title": "human cyclophilin a complexed with 4 4-hydroxy-meleu cyclosporin", "authors": ["V.Mikol", "J.Kallen", "P.Taylor", "M.D.Walkinshaw"], "doi": "10.1006/JMBI.1998.2108", "pmid": "9769216"}
-{"pdb": "3CW2", "title": "crystal structure of the intact archaeal translation initiation factor 2 from sulfolobus solfataricus .", "authors": ["E.A.Stolboushkina", "S.V.Nikonov", "A.D.Nikulin", "U.Blaesi", "D.J.Manstein", "R.V.Fedorov", "M.B.Garber", "O.S.Nikonov"], "doi": "10.1016/J.JMB.2008.07.039", "pmid": "18675278"}
-{"pdb": "3CW9", "title": "4-chlorobenzoyl-coa ligase/synthetase in the thioester-forming conformation, bound to 4-chlorophenacyl-coa", "authors": ["A.S.Reger", "J.Cao", "R.Wu", "D.Dunaway-Mariano", "A.M.Gulick"], "doi": "10.1021/BI800696Y", "pmid": "18620418"}
-{"pdb": "3CWU", "title": "crystal structure of an alka host/guest complex 2'-fluoro-2'-deoxy-1, n6-ethenoadenine:thymine base pair", "authors": ["B.R.Bowman", "S.Lee", "S.Wang", "G.L.Verdine"], "doi": "10.1016/J.STR.2008.04.012", "pmid": "18682218"}
-{"pdb": "5CWF", "title": "crystal structure of de novo designed helical repeat protein dhr8", "authors": ["G.Bhabha", "D.C.Ekiert"], "doi": "10.1038/NATURE16162", "pmid": "26675729"}
+{"classification": "Signaling protein", "pdb": "5NM4", "deposition_date": "2017-04-05", "title": "A2a adenosine receptor room-temperature structure determined by serial Femtosecond crystallography", "Keywords": ["Oom-temperature", " serial crystallography", " signaling protein"], "authors": ["T.weinert", "R.cheng", "D.james", "D.gashi", "P.nogly", "K.jaeger", "M.hennig", "", "J.standfuss"], "pmid": "28912485", "doi": "10.1038/S41467-017-00630-4"}
+{"classification": "Oxidoreductase/oxidoreductase inhibitor", "pdb": "4KN3", "deposition_date": "2013-05-08", "title": "Structure of the y34ns91g double mutant of dehaloperoxidase from Amphitrite ornata with 2,4,6-trichlorophenol", "Keywords": ["Lobin", " oxygen storage", " peroxidase", " oxidoreductase", " oxidoreductase-", "Oxidoreductase inhibitor complex"], "authors": ["C.wang", "L.lovelace", "L.lebioda"], "pmid": "23952341", "doi": "10.1021/BI400627W"}
+{"classification": "Transport protein", "pdb": "8HKM", "deposition_date": "2022-11-27", "title": "Ion channel", "Keywords": ["On channel", " transport protein"], "authors": ["D.h.jiang", "J.t.zhang"], "pmid": "37494189", "doi": "10.1016/J.CELREP.2023.112858"}
+{"classification": "Signaling protein", "pdb": "6JT1", "deposition_date": "2019-04-08", "title": "Structure of human soluble guanylate cyclase in the heme oxidised State", "Keywords": ["Oluble guanylate cyclase", " signaling protein"], "authors": ["L.chen", "Y.kang", "R.liu", "J.-x.wu"], "pmid": "31514202", "doi": "10.1038/S41586-019-1584-6"}
+{"classification": "Immune system", "pdb": "7OW6", "deposition_date": "2021-06-16", "title": "Crystal structure of a tcr in complex with hla-a*11:01 bound to kras G12d peptide (vvvgadgvgk)", "Keywords": ["La", " kras", " tcr", " immune system"], "authors": ["V.karuppiah", "R.a.robinson"], "doi": "10.1038/S41467-022-32811-1"}
+{"classification": "Biosynthetic protein", "pdb": "5EQ8", "deposition_date": "2015-11-12", "title": "Crystal structure of medicago truncatula histidinol-phosphate Phosphatase (mthpp) in complex with l-histidinol", "Keywords": ["Istidine biosynthesis", " metabolic pathways", " dimer", " plant", "", "Biosynthetic protein"], "authors": ["M.ruszkowski", "Z.dauter"], "pmid": "26994138", "doi": "10.1074/JBC.M115.708727"}
+{"classification": "De novo protein", "pdb": "8CWA", "deposition_date": "2022-05-18", "title": "Solution nmr structure of 8-residue rosetta-designed cyclic peptide D8.21 in cdcl3 with cis/trans switching (tc conformation, 53%)", "Keywords": ["Yclic peptide", " non natural amino acids", " cis/trans", " switch peptides", "", "De novo design", "Membrane permeability", "De novo protein"], "authors": ["T.a.ramelot", "R.tejero", "G.t.montelione"], "pmid": "36041435", "doi": "10.1016/J.CELL.2022.07.019"}
+{"classification": "Hydrolase", "pdb": "3R6M", "deposition_date": "2011-03-21", "title": "Crystal structure of vibrio parahaemolyticus yeaz", "Keywords": ["Ctin/hsp70 nucleotide-binding fold", " bacterial resuscitation", " viable", "But non-culturable state", "Resuscitation promoting factor", "Ygjd", "", "Yjee", "Vibrio parahaemolyticus", "Hydrolase"], "authors": ["A.roujeinikova", "I.aydin"], "pmid": "21858042", "doi": "10.1371/JOURNAL.PONE.0023245"}
+{"classification": "Hydrolase", "pdb": "2W5J", "deposition_date": "2008-12-10", "title": "Structure of the c14-rotor ring of the proton translocating Chloroplast atp synthase", "Keywords": ["Ydrolase", " chloroplast", " atp synthase", " lipid-binding", " cf(0)", " membrane", "", "Transport", "Formylation", "Energy transduction", "Hydrogen ion transport", "", "Ion transport", "Transmembrane", "Membrane protein"], "authors": ["M.vollmar", "D.schlieper", "M.winn", "C.buechner", "G.groth"], "pmid": "19423706", "doi": "10.1074/JBC.M109.006916"}
+{"classification": "De novo protein", "pdb": "4GLU", "deposition_date": "2012-08-14", "title": "Crystal structure of the mirror image form of vegf-a", "Keywords": ["-protein", " covalent dimer", " cysteine knot protein", " growth factor", " de", "Novo protein"], "authors": ["K.mandal", "M.uppalapati", "D.ault-riche", "J.kenney", "J.lowitz", "S.sidhu", "", "S.b.h.kent"], "pmid": "22927390", "doi": "10.1073/PNAS.1210483109"}
+{"classification": "Hydrolase/hydrolase inhibitor", "pdb": "3WYL", "deposition_date": "2014-09-01", "title": "Crystal structure of the catalytic domain of pde10a complexed with 5- Methoxy-3-(1-phenyl-1h-pyrazol-5-yl)-1-(3-(trifluoromethyl)phenyl) Pyridazin-4(1h)-one", "Keywords": ["Ydrolase-hydrolase inhibitor complex"], "authors": ["H.oki", "Y.hayano"], "pmid": "25384088", "doi": "10.1021/JM5013648"}
+{"classification": "Isomerase", "pdb": "5BOR", "deposition_date": "2015-05-27", "title": "Structure of acetobacter aceti pure-s57c, sulfonate form", "Keywords": ["Cidophile", " pure", " purine biosynthesis", " isomerase"], "authors": ["K.l.sullivan", "T.j.kappock"]}
+{"classification": "Hydrolase", "pdb": "1X0C", "deposition_date": "2005-03-17", "title": "Improved crystal structure of isopullulanase from aspergillus niger Atcc 9642", "Keywords": ["Ullulan", " glycoside hydrolase family 49", " glycoprotein", " hydrolase"], "authors": ["M.mizuno", "T.tonozuka", "A.yamamura", "Y.miyasaka", "H.akeboshi", "S.kamitori", "", "A.nishikawa", "Y.sakano"], "pmid": "18155243", "doi": "10.1016/J.JMB.2007.11.098"}
+{"classification": "Oxidoreductase", "pdb": "7CUP", "deposition_date": "2020-08-23", "title": "Structure of 2,5-dihydroxypridine dioxygenase from pseudomonas putida Kt2440", "Keywords": ["On-heme dioxygenase", " oxidoreductase"], "authors": ["G.q.liu", "H.z.tang"]}
+{"classification": "Ligase", "pdb": "1VCN", "deposition_date": "2004-03-10", "title": "Crystal structure of t.th. hb8 ctp synthetase complex with sulfate Anion", "Keywords": ["Etramer", " riken structural genomics/proteomics initiative", " rsgi", "", "Structural genomics", "Ligase"], "authors": ["M.goto", "Riken structural genomics/proteomics initiative (rsgi)"], "pmid": "15296735", "doi": "10.1016/J.STR.2004.05.013"}
+{"classification": "Transferase/transferase inhibitor", "pdb": "6C9V", "deposition_date": "2018-01-28", "title": "Mycobacterium tuberculosis adenosine kinase bound to (2r,3s,4r,5r)-2- (hydroxymethyl)-5-(6-(4-phenylpiperazin-1-yl)-9h-purin-9-yl) Tetrahydrofuran-3,4-diol", "Keywords": ["Ucleoside analog", " complex", " inhibitor", " structural genomics", " psi-2", "", "Protein structure initiative", "Tb structural genomics consortium", "", "Tbsgc", "Transferase-transferase inhibitor complex"], "authors": ["R.a.crespo", "Tb structural genomics consortium (tbsgc)"], "pmid": "31002508", "doi": "10.1021/ACS.JMEDCHEM.9B00020"}
+{"classification": "De novo protein", "pdb": "4LPY", "deposition_date": "2013-07-16", "title": "Crystal structure of tencon variant g10", "Keywords": ["Ibronectin type iii fold", " alternate scaffold", " de novo protein"], "authors": ["A.teplyakov", "G.obmolova", "G.l.gilliland"], "pmid": "24375666", "doi": "10.1002/PROT.24502"}
+{"classification": "Isomerase", "pdb": "2Y88", "deposition_date": "2011-02-03", "title": "Crystal structure of mycobacterium tuberculosis phosphoribosyl Isomerase (variant d11n) with bound prfar", "Keywords": ["Romatic amino acid biosynthesis", " isomerase", " tim-barrel", " histidine", "Biosynthesis", "Tryptophan biosynthesis"], "authors": ["J.kuper", "A.v.due", "A.geerlof", "M.wilmanns"], "pmid": "21321225", "doi": "10.1073/PNAS.1015996108"}
+{"classification": "Unknown function", "pdb": "1SR0", "deposition_date": "2004-03-22", "title": "Crystal structure of signalling protein from sheep(sps-40) at 3.0a Resolution using crystal grown in the presence of polysaccharides", "Keywords": ["Ignalling protein", " involution", " unknown function"], "authors": ["D.b.srivastava", "A.s.ethayathulla", "N.singh", "J.kumar", "S.sharma", "T.p.singh"]}
+{"classification": "Dna binding protein", "pdb": "3RH2", "deposition_date": "2011-04-11", "title": "Crystal structure of a tetr-like transcriptional regulator (sama_0099) From shewanella amazonensis sb2b at 2.42 a resolution", "Keywords": ["Na/rna-binding 3-helical bundle", " structural genomics", " joint center", "For structural genomics", "Jcsg", "Protein structure initiative", "Psi-", "Biology", "Dna binding protein"], "authors": ["Joint center for structural genomics (jcsg)"]}
+{"classification": "Transferase", "pdb": "2WK5", "deposition_date": "2009-06-05", "title": "Structural features of native human thymidine phosphorylase And in complex with 5-iodouracil", "Keywords": ["Lycosyltransferase", " developmental protein", " angiogenesis", "", "5-iodouracil", "Growth factor", "Enzyme kinetics", "", "Differentiation", "Disease mutation", "Thymidine", "Phosphorylase", "Chemotaxis", "Transferase", "Mutagenesis", "", "Polymorphism"], "authors": ["E.mitsiki", "A.c.papageorgiou", "S.iyer", "N.thiyagarajan", "S.h.prior", "", "D.sleep", "C.finnis", "K.r.acharya"], "pmid": "19555658", "doi": "10.1016/J.BBRC.2009.06.104"}
+{"classification": "Hydrolase", "pdb": "3P9Y", "deposition_date": "2010-10-18", "title": "Crystal structure of the drosophila melanogaster ssu72-pctd complex", "Keywords": ["Hosphatase", " cis proline", " lmw ptp-like fold", " rna polymerase ii ctd", "", "Hydrolase"], "authors": ["J.w.werner-allen", "P.zhou"], "pmid": "21159777", "doi": "10.1074/JBC.M110.197129"}
+{"classification": "Recombination/dna", "pdb": "6OEO", "deposition_date": "2019-03-27", "title": "Cryo-em structure of mouse rag1/2 nfc complex (dna1)", "Keywords": ["(d)j recombination", " dna transposition", " rag", " scid", " recombination", "", "Recombination-dna complex"], "authors": ["X.chen", "Y.cui", "Z.h.zhou", "W.yang", "M.gellert"], "pmid": "32015552", "doi": "10.1038/S41594-019-0363-2"}
+{"classification": "Hydrolase", "pdb": "4ECA", "deposition_date": "1997-02-21", "title": "Asparaginase from e. coli, mutant t89v with covalently bound aspartate", "Keywords": ["Ydrolase", " acyl-enzyme intermediate", " threonine amidohydrolase"], "authors": ["G.j.palm", "J.lubkowski", "A.wlodawer"], "pmid": "8706862", "doi": "10.1016/0014-5793(96)00660-6"}
+{"classification": "Transcription/protein binding", "pdb": "3UVX", "deposition_date": "2011-11-30", "title": "Crystal structure of the first bromodomain of human brd4 in complex With a diacetylated histone 4 peptide (h4k12ack16ac)", "Keywords": ["Romodomain", " bromodomain containing protein 4", " cap", " hunk1", " mcap", "", "Mitotic chromosome associated protein", "Peptide complex", "Structural", "Genomics consortium", "Sgc", "Transcription-protein binding complex"], "authors": ["P.filippakopoulos", "S.picaud", "T.keates", "E.ugochukwu", "F.von delft", "", "C.h.arrowsmith", "A.m.edwards", "J.weigelt", "C.bountra", "S.knapp", "Structural", "Genomics consortium (sgc)"], "pmid": "22464331", "doi": "10.1016/J.CELL.2012.02.013"}
+{"classification": "Membrane protein", "pdb": "1TLZ", "deposition_date": "2004-06-10", "title": "Tsx structure complexed with uridine", "Keywords": ["Ucleoside transporter", " beta barrel", " uridine", " membrane", "Protein"], "authors": ["J.ye", "B.van den berg"], "pmid": "15272310", "doi": "10.1038/SJ.EMBOJ.7600330"}
+{"classification": "Dna binding protein", "pdb": "7AZD", "deposition_date": "2020-11-16", "title": "Dna polymerase sliding clamp from escherichia coli with peptide 20 Bound", "Keywords": ["Ntibacterial drug", " dna binding protein"], "authors": ["C.monsarrat", "G.compain", "C.andre", "I.martiel", "S.engilberge", "V.olieric", "", "P.wolff", "K.brillet", "M.landolfo", "C.silva da veiga", "J.wagner", "G.guichard", "", "D.y.burnouf"], "pmid": "34806883", "doi": "10.1021/ACS.JMEDCHEM.1C00918"}
+{"classification": "Transferase", "pdb": "5N3K", "deposition_date": "2017-02-08", "title": "Camp-dependent protein kinase a from cricetulus griseus in complex With fragment like molecule o-guanidino-l-homoserine", "Keywords": ["Ragment", " complex", " transferase", " serine threonine kinase", " camp", "", "Kinase", "Pka"], "authors": ["C.siefker", "A.heine", "G.klebe"]}
+{"classification": "Biosynthetic protein", "pdb": "8H52", "deposition_date": "2022-10-11", "title": "Crystal structure of helicobacter pylori carboxyspermidine Dehydrogenase in complex with nadp", "Keywords": ["Arboxyspermidine dehydrogenase", " biosynthetic protein"], "authors": ["K.y.ko", "S.c.park", "S.y.cho", "S.i.yoon"], "pmid": "36283333", "doi": "10.1016/J.BBRC.2022.10.049"}
+{"classification": "Metal binding protein", "pdb": "6DYC", "deposition_date": "2018-07-01", "title": "Co(ii)-bound structure of the engineered cyt cb562 variant, ch3", "Keywords": ["Esigned protein", " 4-helix bundle", " electron transport", " metal binding", "Protein"], "authors": ["F.a.tezcan", "J.rittle"], "pmid": "30778140", "doi": "10.1038/S41557-019-0218-9"}
+{"classification": "Protein fibril", "pdb": "6A6B", "deposition_date": "2018-06-27", "title": "Cryo-em structure of alpha-synuclein fiber", "Keywords": ["Lpha-syn fiber", " parkinson disease", " protein fibril"], "authors": ["Y.w.li", "C.y.zhao", "F.luo", "Z.liu", "X.gui", "Z.luo", "X.zhang", "D.li", "C.liu", "X.li"], "pmid": "30065316", "doi": "10.1038/S41422-018-0075-X"}
+{"classification": "Dna", "pdb": "7D5E", "deposition_date": "2020-09-25", "title": "Left-handed g-quadruplex containing two bulges", "Keywords": ["-quadruplex", " bulge", " dna", " left-handed"], "authors": ["P.das", "A.maity", "K.h.ngo", "F.r.winnerdy", "B.bakalar", "Y.mechulam", "E.schmitt", "", "A.t.phan"], "pmid": "33503265", "doi": "10.1093/NAR/GKAA1259"}
+{"classification": "Transferase", "pdb": "3RSY", "deposition_date": "2011-05-02", "title": "Cellobiose phosphorylase from cellulomonas uda in complex with sulfate And glycerol", "Keywords": ["H94", " alpha barrel", " cellobiose phosphorylase", " disaccharide", "Phosphorylase", "Transferase"], "authors": ["A.van hoorebeke", "J.stout", "W.soetaert", "J.van beeumen", "T.desmet", "S.savvides"]}
+{"classification": "Oxidoreductase", "pdb": "7MCI", "deposition_date": "2021-04-02", "title": "Mofe protein from azotobacter vinelandii with a sulfur-replenished Cofactor", "Keywords": ["Zotobacter vinelandii", " mofe-protein", " nitrogenase", " oxidoreductase"], "authors": ["W.kang", "C.lee", "Y.hu", "M.w.ribbe"], "doi": "10.1038/S41929-022-00782-7"}
+{"classification": "Dna", "pdb": "1XUW", "deposition_date": "2004-10-26", "title": "Structural rationalization of a large difference in rna affinity Despite a small difference in chemistry between two 2'-o-modified Nucleic acid analogs", "Keywords": ["Na mimetic methylcarbamate amide analog", " dna"], "authors": ["R.pattanayek", "L.sethaphong", "C.pan", "M.prhavc", "T.p.prakash", "M.manoharan", "", "M.egli"], "pmid": "15547979", "doi": "10.1021/JA044637K"}
+{"classification": "Lyase", "pdb": "7C0D", "deposition_date": "2020-05-01", "title": "Crystal structure of azospirillum brasilense l-2-keto-3-deoxyarabonate Dehydratase (hydroxypyruvate-bound form)", "Keywords": ["-2-keto-3-deoxyarabonate dehydratase", " lyase"], "authors": ["Y.watanabe", "S.watanabe"], "pmid": "32697085", "doi": "10.1021/ACS.BIOCHEM.0C00515"}
+{"classification": "Signaling protein", "pdb": "5LYK", "deposition_date": "2016-09-28", "title": "Crystal structure of intracellular b30.2 domain of btn3a1 bound to Citrate", "Keywords": ["30.2", " butyrophilin", " signaling protein"], "authors": ["F.mohammed", "A.t.baker", "M.salim", "B.e.willcox"], "pmid": "28862425", "doi": "10.1021/ACSCHEMBIO.7B00694"}
+{"classification": "Toxin", "pdb": "4IZL", "deposition_date": "2013-01-30", "title": "Structure of the n248a mutant of the panton-valentine leucocidin s Component from staphylococcus aureus", "Keywords": ["I-component leucotoxin", " staphylococcus aureus", " s component", "Leucocidin", "Beta-barrel pore forming toxin", "Toxin"], "authors": ["L.maveyraud", "B.j.laventie", "G.prevost", "L.mourey"], "pmid": "24643034", "doi": "10.1371/JOURNAL.PONE.0092094"}
+{"classification": "Dna", "pdb": "6F3C", "deposition_date": "2017-11-28", "title": "The cytotoxic [pt(h2bapbpy)] platinum complex interacting with the Cgtacg hexamer", "Keywords": ["Rug-dna complex", " four-way junction", " dna"], "authors": ["M.ferraroni", "C.bazzicalupi", "P.gratteri", "F.papi"], "pmid": "31046177", "doi": "10.1002/ANIE.201814532"}
+{"classification": "Signaling protein/inhibitor", "pdb": "4L5M", "deposition_date": "2013-06-11", "title": "Complexe of arno sec7 domain with the protein-protein interaction Inhibitor n-(4-hydroxy-2,6-dimethylphenyl)benzenesulfonamide at ph6.5", "Keywords": ["Ec-7domain", " signaling protein-inhibitor complex"], "authors": ["F.hoh", "J.rouhana"], "pmid": "24112024", "doi": "10.1021/JM4009357"}
+{"classification": "Signaling protein", "pdb": "5I6J", "deposition_date": "2016-02-16", "title": "Crystal structure of srgap2 f-barx", "Keywords": ["Rgap2", " f-bar", " fx", " signaling protein"], "authors": ["M.sporny", "J.guez-haddad", "M.n.isupov", "Y.opatowsky"], "pmid": "28333212", "doi": "10.1093/MOLBEV/MSX094"}
+{"classification": "Metal binding protein", "pdb": "1Q80", "deposition_date": "2003-08-20", "title": "Solution structure and dynamics of nereis sarcoplasmic calcium binding Protein", "Keywords": ["Ll-alpha", " metal binding protein"], "authors": ["G.rabah", "R.popescu", "J.a.cox", "Y.engelborghs", "C.t.craescu"], "pmid": "15819893", "doi": "10.1111/J.1742-4658.2005.04629.X"}
+{"classification": "Transferase", "pdb": "1TW1", "deposition_date": "2004-06-30", "title": "Beta-1,4-galactosyltransferase mutant met344his (m344h-gal-t1) complex With udp-galactose and magnesium", "Keywords": ["Et344his mutation; closed conformation; mn binding", " transferase"], "authors": ["B.ramakrishnan", "E.boeggeman", "P.k.qasba"], "pmid": "15449940", "doi": "10.1021/BI049007+"}
+{"classification": "Rna", "pdb": "2PN4", "deposition_date": "2007-04-23", "title": "Crystal structure of hepatitis c virus ires subdomain iia", "Keywords": ["Cv", " ires", " subdoamin iia", " rna", " strontium", " hepatitis"], "authors": ["Q.zhao", "Q.han", "C.r.kissinger", "P.a.thompson"], "pmid": "18391410", "doi": "10.1107/S0907444908002011"}

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/uniprot_dump

@@ -1,6 +1,36 @@
-{"pid": "Q6GZX4", "dates": [{"date": "28-JUN-2011", "date_info": " integrated into UniProtKB/Swiss-Prot."}, {"date": "19-JUL-2004", "date_info": " sequence version 1."}, {"date": "12-AUG-2020", "date_info": " entry version 41."}], "title": "Putative transcription factor 001R;", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3).", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus."], "references": [{"PubMed": "15165820"}, {" DOI": "10.1016/j.virol.2004.02.019"}]}
-{"pid": "Q6GZX3", "dates": [{"date": "28-JUN-2011", "date_info": " integrated into UniProtKB/Swiss-Prot."}, {"date": "19-JUL-2004", "date_info": " sequence version 1."}, {"date": "12-AUG-2020", "date_info": " entry version 42."}], "title": "Uncharacterized protein 002L;", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3).", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus."], "references": [{"PubMed": "15165820"}, {" DOI": "10.1016/j.virol.2004.02.019"}]}
-{"pid": "Q197F8", "dates": [{"date": "16-JUN-2009", "date_info": " integrated into UniProtKB/Swiss-Prot."}, {"date": "11-JUL-2006", "date_info": " sequence version 1."}, {"date": "12-AUG-2020", "date_info": " entry version 27."}], "title": "Uncharacterized protein 002R;", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus).", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus."], "references": [{"PubMed": "16912294"}, {" DOI": "10.1128/jvi.00464-06"}]}
-{"pid": "Q197F7", "dates": [{"date": "16-JUN-2009", "date_info": " integrated into UniProtKB/Swiss-Prot."}, {"date": "11-JUL-2006", "date_info": " sequence version 1."}, {"date": "12-AUG-2020", "date_info": " entry version 23."}], "title": "Uncharacterized protein 003L;", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus).", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus."], "references": [{"PubMed": "16912294"}, {" DOI": "10.1128/jvi.00464-06"}]}
-{"pid": "Q6GZX2", "dates": [{"date": "28-JUN-2011", "date_info": " integrated into UniProtKB/Swiss-Prot."}, {"date": "19-JUL-2004", "date_info": " sequence version 1."}, {"date": "12-AUG-2020", "date_info": " entry version 36."}], "title": "Uncharacterized protein 3R;", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3).", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus."], "references": [{"PubMed": "15165820"}, {" DOI": "10.1016/j.virol.2004.02.019"}]}
-{"pid": "Q6GZX1", "dates": [{"date": "28-JUN-2011", "date_info": " integrated into UniProtKB/Swiss-Prot."}, {"date": "19-JUL-2004", "date_info": " sequence version 1."}, {"date": "12-AUG-2020", "date_info": " entry version 34."}], "title": "Uncharacterized protein 004R;", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3).", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus."], "references": [{"PubMed": "15165820"}, {" DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZX4", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 43"}], "title": "Putative transcription factor 001R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZX3", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 45"}], "title": "Uncharacterized protein 002L", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197F8", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2022-02-23", "date_info": "entry version 29"}], "title": "Uncharacterized protein 002R", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q197F7", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2020-08-12", "date_info": "entry version 23"}], "title": "Uncharacterized protein 003L", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q6GZX2", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 37"}], "title": "Uncharacterized protein 3R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZX1", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 38"}], "title": "Uncharacterized protein 004R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197F5", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2022-10-12", "date_info": "entry version 32"}], "title": "Uncharacterized protein 005L", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q6GZX0", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 47"}], "title": "Uncharacterized protein 005R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q91G88", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2001-12-01", "date_info": "sequence version 1"}, {"date": "2023-06-28", "date_info": "entry version 53"}], "title": "Putative KilA-N domain-containing protein 006L", "organism_species": "Invertebrate iridescent virus 6 (IIV-6) (Chilo iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Iridovirus"], "references": [{"PubMed": "17239238"}, {"DOI": "10.1186/1743-422x-4-11"}]}
+{"pid": " Q6GZW9", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 34"}], "title": "Uncharacterized protein 006R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZW8", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 32"}], "title": "Uncharacterized protein 007R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197F3", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2023-02-22", "date_info": "entry version 28"}], "title": "Uncharacterized protein 007R", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q197F2", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2022-02-23", "date_info": "entry version 22"}], "title": "Uncharacterized protein 008L", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q6GZW6", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 67"}], "title": "Putative helicase 009L", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q91G85", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2001-12-01", "date_info": "sequence version 1"}, {"date": "2023-02-22", "date_info": "entry version 38"}], "title": "Uncharacterized protein 009R", "organism_species": "Invertebrate iridescent virus 6 (IIV-6) (Chilo iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Iridovirus"], "references": [{"PubMed": "17239238"}, {"DOI": "10.1186/1743-422x-4-11"}]}
+{"pid": " Q6GZW5", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 37"}], "title": "Uncharacterized protein 010R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197E9", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2023-02-22", "date_info": "entry version 28"}], "title": "Uncharacterized protein 011L", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q6GZW4", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 37"}], "title": "Uncharacterized protein 011R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZW3", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 35"}], "title": "Uncharacterized protein 012L", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197E7", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2023-02-22", "date_info": "entry version 37"}], "title": "Uncharacterized protein IIV3-013L", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q6GZW2", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 30"}], "title": "Uncharacterized protein 013R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZW1", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 35"}], "title": "Uncharacterized protein 014R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZW0", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 50"}], "title": "Uncharacterized protein 015R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZV8", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 35"}], "title": "Uncharacterized protein 017L", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZV7", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 33"}], "title": "Uncharacterized protein 018L", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZV6", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 87"}], "title": "Putative serine/threonine-protein kinase 019R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZV5", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 40"}], "title": "Uncharacterized protein 020R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZV4", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 35"}], "title": "Uncharacterized protein 021L", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197D8", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2022-12-14", "date_info": "entry version 35"}], "title": "Transmembrane protein 022L", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q6GZV2", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 33"}], "title": "Uncharacterized protein 023R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197D7", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2023-02-22", "date_info": "entry version 25"}], "title": "Uncharacterized protein 023R", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q6GZV1", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 37"}], "title": "Uncharacterized protein 024R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q197D5", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2006-07-11", "date_info": "sequence version 1"}, {"date": "2022-10-12", "date_info": "entry version 24"}], "title": "Uncharacterized protein 025R", "organism_species": "Invertebrate iridescent virus 3 (IIV-3) (Mosquito iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Chloriridovirus"], "references": [{"PubMed": "16912294"}, {"DOI": "10.1128/jvi.00464-06"}]}
+{"pid": " Q91G70", "dates": [{"date": "2009-06-16", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2001-12-01", "date_info": "sequence version 1"}, {"date": "2020-08-12", "date_info": "entry version 32"}], "title": "Uncharacterized protein 026R", "organism_species": "Invertebrate iridescent virus 6 (IIV-6) (Chilo iridescent virus)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Betairidovirinae", "Iridovirus"], "references": [{"PubMed": "17239238"}, {"DOI": "10.1186/1743-422x-4-11"}]}
+{"pid": " Q6GZU9", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 49"}], "title": "Uncharacterized protein 027R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}
+{"pid": " Q6GZU8", "dates": [{"date": "2011-06-28", "date_info": "integrated into UniProtKB/Swiss-Prot"}, {"date": "2004-07-19", "date_info": "sequence version 1"}, {"date": "2023-09-13", "date_info": "entry version 55"}], "title": "Uncharacterized protein 028R", "organism_species": "Frog virus 3 (isolate Goorha) (FV-3)", "subjects": ["Viruses", "Varidnaviria", "Bamfordvirae", "Nucleocytoviricota", "Megaviricetes", "Pimascovirales", "Iridoviridae", "Alphairidovirinae", "Ranavirus"], "references": [{"PubMed": "15165820"}, {"DOI": "10.1016/j.virol.2004.02.019"}]}

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java

@@ -4,7 +4,6 @@ package eu.dnetlib.dhp.oa.dedup;
 import java.util.*;
 import java.util.stream.Stream;
 
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
 import org.apache.commons.beanutils.BeanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.api.java.function.FlatMapFunction;
@@ -15,7 +14,6 @@ import org.apache.spark.sql.*;
 import eu.dnetlib.dhp.oa.dedup.model.Identifier;
 import eu.dnetlib.dhp.oa.merge.AuthorMerger;
 import eu.dnetlib.dhp.schema.common.ModelSupport;
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
 import eu.dnetlib.dhp.schema.oaf.Author;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.OafEntity;
@@ -25,190 +23,190 @@ import scala.Tuple3;
 import scala.collection.JavaConversions;
 
 public class DedupRecordFactory {
-    public static final class DedupRecordReduceState {
-        public final String dedupId;
+	public static final class DedupRecordReduceState {
+		public final String dedupId;
 
-        public final ArrayList<String> aliases = new ArrayList<>();
+		public final ArrayList<String> aliases = new ArrayList<>();
 
-        public final HashSet<String> acceptanceDate = new HashSet<>();
+		public final HashSet<String> acceptanceDate = new HashSet<>();
 
-        public OafEntity entity;
+		public OafEntity entity;
 
-        public DedupRecordReduceState(String dedupId, String id, OafEntity entity) {
-            this.dedupId = dedupId;
-            this.entity = entity;
-            if (entity == null) {
-                aliases.add(id);
-            } else {
-                if (Result.class.isAssignableFrom(entity.getClass())) {
-                    Result result = (Result) entity;
-                    if (result.getDateofacceptance() != null
-                            && StringUtils.isNotBlank(result.getDateofacceptance().getValue())) {
-                        acceptanceDate.add(result.getDateofacceptance().getValue());
-                    }
-                }
-            }
-        }
+		public DedupRecordReduceState(String dedupId, String id, OafEntity entity) {
+			this.dedupId = dedupId;
+			this.entity = entity;
+			if (entity == null) {
+				aliases.add(id);
+			} else {
+				if (Result.class.isAssignableFrom(entity.getClass())) {
+					Result result = (Result) entity;
+					if (result.getDateofacceptance() != null
+						&& StringUtils.isNotBlank(result.getDateofacceptance().getValue())) {
+						acceptanceDate.add(result.getDateofacceptance().getValue());
+					}
+				}
+			}
+		}
 
-        public String getDedupId() {
-            return dedupId;
-        }
-    }
+		public String getDedupId() {
+			return dedupId;
+		}
+	}
 
-    private static final int MAX_ACCEPTANCE_DATE = 20;
+	private static final int MAX_ACCEPTANCE_DATE = 20;
 
-    private DedupRecordFactory() {
-    }
+	private DedupRecordFactory() {
+	}
 
-    public static Dataset<OafEntity> createDedupRecord(
-            final SparkSession spark,
-            final DataInfo dataInfo,
-            final String mergeRelsInputPath,
-            final String entitiesInputPath,
-            final Class<OafEntity> clazz) {
+	public static Dataset<OafEntity> createDedupRecord(
+		final SparkSession spark,
+		final DataInfo dataInfo,
+		final String mergeRelsInputPath,
+		final String entitiesInputPath,
+		final Class<OafEntity> clazz) {
 
-        final long ts = System.currentTimeMillis();
-        final Encoder<OafEntity> beanEncoder = Encoders.bean(clazz);
-        final Encoder<OafEntity> kryoEncoder = Encoders.kryo(clazz);
+		final long ts = System.currentTimeMillis();
+		final Encoder<OafEntity> beanEncoder = Encoders.bean(clazz);
+		final Encoder<OafEntity> kryoEncoder = Encoders.kryo(clazz);
 
-        // <id, json_entity>
-        Dataset<Row> entities = spark
-                .read()
-                .schema(Encoders.bean(clazz).schema())
-                .json(entitiesInputPath)
-                .as(beanEncoder)
-                .map(
-                        (MapFunction<OafEntity, Tuple2<String, OafEntity>>) entity -> {
-                            return new Tuple2<>(entity.getId(), entity);
-                        },
-                        Encoders.tuple(Encoders.STRING(), kryoEncoder))
-                .selectExpr("_1 AS id", "_2 AS kryoObject");
+		// <id, json_entity>
+		Dataset<Row> entities = spark
+			.read()
+			.schema(Encoders.bean(clazz).schema())
+			.json(entitiesInputPath)
+			.as(beanEncoder)
+			.map(
+				(MapFunction<OafEntity, Tuple2<String, OafEntity>>) entity -> {
+					return new Tuple2<>(entity.getId(), entity);
+				},
+				Encoders.tuple(Encoders.STRING(), kryoEncoder))
+			.selectExpr("_1 AS id", "_2 AS kryoObject");
 
-        // <source, target>: source is the dedup_id, target is the id of the mergedIn
-        Dataset<Row> mergeRels = spark
-                .read()
-                .load(mergeRelsInputPath)
-                .where("relClass == 'merges'")
-                .selectExpr("source as dedupId", "target as id");
+		// <source, target>: source is the dedup_id, target is the id of the mergedIn
+		Dataset<Row> mergeRels = spark
+			.read()
+			.load(mergeRelsInputPath)
+			.where("relClass == 'merges'")
+			.selectExpr("source as dedupId", "target as id");
 
-        return mergeRels
-                .join(entities, JavaConversions.asScalaBuffer(Collections.singletonList("id")), "left")
-                .select("dedupId", "id", "kryoObject")
-                .as(Encoders.tuple(Encoders.STRING(), Encoders.STRING(), kryoEncoder))
-                .map(
-                        (MapFunction<Tuple3<String, String, OafEntity>, DedupRecordReduceState>) t -> new DedupRecordReduceState(
-                                t._1(), t._2(), t._3()),
-                        Encoders.kryo(DedupRecordReduceState.class))
-                .groupByKey(
-                        (MapFunction<DedupRecordReduceState, String>) DedupRecordReduceState::getDedupId, Encoders.STRING())
-                .reduceGroups(
-                        (ReduceFunction<DedupRecordReduceState>) (t1, t2) -> {
-                            if (t1.entity == null) {
-                                t2.aliases.addAll(t1.aliases);
-                                return t2;
-                            }
-                            if (t1.acceptanceDate.size() < MAX_ACCEPTANCE_DATE) {
-                                t1.acceptanceDate.addAll(t2.acceptanceDate);
-                            }
-                            t1.aliases.addAll(t2.aliases);
-                            t1.entity = reduceEntity(t1.entity, t2.entity);
+		return mergeRels
+			.join(entities, JavaConversions.asScalaBuffer(Collections.singletonList("id")), "left")
+			.select("dedupId", "id", "kryoObject")
+			.as(Encoders.tuple(Encoders.STRING(), Encoders.STRING(), kryoEncoder))
+			.map(
+				(MapFunction<Tuple3<String, String, OafEntity>, DedupRecordReduceState>) t -> new DedupRecordReduceState(
+					t._1(), t._2(), t._3()),
+				Encoders.kryo(DedupRecordReduceState.class))
+			.groupByKey(
+				(MapFunction<DedupRecordReduceState, String>) DedupRecordReduceState::getDedupId, Encoders.STRING())
+			.reduceGroups(
+				(ReduceFunction<DedupRecordReduceState>) (t1, t2) -> {
+					if (t1.entity == null) {
+						t2.aliases.addAll(t1.aliases);
+						return t2;
+					}
+					if (t1.acceptanceDate.size() < MAX_ACCEPTANCE_DATE) {
+						t1.acceptanceDate.addAll(t2.acceptanceDate);
+					}
+					t1.aliases.addAll(t2.aliases);
+					t1.entity = reduceEntity(t1.entity, t2.entity);
 
-                            return t1;
-                        })
-                .flatMap((FlatMapFunction<Tuple2<String, DedupRecordReduceState>, OafEntity>) t -> {
-                    String dedupId = t._1();
-                    DedupRecordReduceState agg = t._2();
+					return t1;
+				})
+			.flatMap((FlatMapFunction<Tuple2<String, DedupRecordReduceState>, OafEntity>) t -> {
+				String dedupId = t._1();
+				DedupRecordReduceState agg = t._2();
 
-                    if (agg.acceptanceDate.size() >= MAX_ACCEPTANCE_DATE) {
-                        return Collections.emptyIterator();
-                    }
+				if (agg.acceptanceDate.size() >= MAX_ACCEPTANCE_DATE) {
+					return Collections.emptyIterator();
+				}
 
-                    return Stream
-                            .concat(
-                                    Stream
-                                            .of(agg.getDedupId())
-                                            .map(id -> createDedupOafEntity(id, agg.entity, dataInfo, ts)),
-                                    agg.aliases
-                                            .stream()
-                                            .map(id -> createMergedDedupAliasOafEntity(id, agg.entity, dataInfo, ts)))
-                            .iterator();
-                }, beanEncoder);
-    }
+				return Stream
+					.concat(
+						Stream
+							.of(agg.getDedupId())
+							.map(id -> createDedupOafEntity(id, agg.entity, dataInfo, ts)),
+						agg.aliases
+							.stream()
+							.map(id -> createMergedDedupAliasOafEntity(id, agg.entity, dataInfo, ts)))
+					.iterator();
+			}, beanEncoder);
+	}
 
-    private static OafEntity createDedupOafEntity(String id, OafEntity base, DataInfo dataInfo, long ts) {
-        try {
-            OafEntity res = (OafEntity) BeanUtils.cloneBean(base);
-            res.setId(id);
-            res.setDataInfo(dataInfo);
-            res.setLastupdatetimestamp(ts);
-            return res;
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
+	private static OafEntity createDedupOafEntity(String id, OafEntity base, DataInfo dataInfo, long ts) {
+		try {
+			OafEntity res = (OafEntity) BeanUtils.cloneBean(base);
+			res.setId(id);
+			res.setDataInfo(dataInfo);
+			res.setLastupdatetimestamp(ts);
+			return res;
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
 
-    private static OafEntity createMergedDedupAliasOafEntity(String id, OafEntity base, DataInfo dataInfo, long ts) {
-        try {
-            OafEntity res = createDedupOafEntity(id, base, dataInfo, ts);
-            DataInfo ds = (DataInfo) BeanUtils.cloneBean(dataInfo);
-            ds.setDeletedbyinference(true);
-            res.setDataInfo(ds);
-            return res;
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
+	private static OafEntity createMergedDedupAliasOafEntity(String id, OafEntity base, DataInfo dataInfo, long ts) {
+		try {
+			OafEntity res = createDedupOafEntity(id, base, dataInfo, ts);
+			DataInfo ds = (DataInfo) BeanUtils.cloneBean(dataInfo);
+			ds.setDeletedbyinference(true);
+			res.setDataInfo(ds);
+			return res;
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
 
-    private static OafEntity reduceEntity(OafEntity entity, OafEntity duplicate) {
+	private static OafEntity reduceEntity(OafEntity entity, OafEntity duplicate) {
 
-        if (duplicate == null) {
-            return entity;
-        }
+		if (duplicate == null) {
+			return entity;
+		}
 
-        int compare = new IdentifierComparator<>()
-                .compare(Identifier.newInstance(entity), Identifier.newInstance(duplicate));
+		int compare = new IdentifierComparator<>()
+			.compare(Identifier.newInstance(entity), Identifier.newInstance(duplicate));
 
-        if (compare > 0) {
-            OafEntity swap = duplicate;
-            duplicate = entity;
-            entity = swap;
-        }
+		if (compare > 0) {
+			OafEntity swap = duplicate;
+			duplicate = entity;
+			entity = swap;
+		}
 
-        entity = MergeUtils.checkedMerge(entity, duplicate);
+		entity.mergeFrom(duplicate);
 
-        if (ModelSupport.isSubClass(duplicate, Result.class)) {
-            Result re = (Result) entity;
-            Result rd = (Result) duplicate;
+		if (ModelSupport.isSubClass(duplicate, Result.class)) {
+			Result re = (Result) entity;
+			Result rd = (Result) duplicate;
 
-            List<List<Author>> authors = new ArrayList<>();
-            if (re.getAuthor() != null) {
-                authors.add(re.getAuthor());
-            }
-            if (rd.getAuthor() != null) {
-                authors.add(rd.getAuthor());
-            }
+			List<List<Author>> authors = new ArrayList<>();
+			if (re.getAuthor() != null) {
+				authors.add(re.getAuthor());
+			}
+			if (rd.getAuthor() != null) {
+				authors.add(rd.getAuthor());
+			}
 
-            re.setAuthor(AuthorMerger.merge(authors));
-        }
+			re.setAuthor(AuthorMerger.merge(authors));
+		}
 
-        return entity;
-    }
+		return entity;
+	}
 
-    public static <T extends OafEntity> T entityMerger(
-            String id, Iterator<Tuple2<String, T>> entities, long ts, DataInfo dataInfo, Class<T> clazz) {
-        T base = entities.next()._2();
+	public static <T extends OafEntity> T entityMerger(
+		String id, Iterator<Tuple2<String, T>> entities, long ts, DataInfo dataInfo, Class<T> clazz) {
+		T base = entities.next()._2();
 
-        while (entities.hasNext()) {
-            T duplicate = entities.next()._2();
-            if (duplicate != null)
-                base = (T) reduceEntity(base, duplicate);
-        }
+		while (entities.hasNext()) {
+			T duplicate = entities.next()._2();
+			if (duplicate != null)
+				base = (T) reduceEntity(base, duplicate);
+		}
 
-        base.setId(id);
-        base.setDataInfo(dataInfo);
-        base.setLastupdatetimestamp(ts);
+		base.setId(id);
+		base.setDataInfo(dataInfo);
+		base.setLastupdatetimestamp(ts);
 
-        return base;
-    }
+		return base;
+	}
 
 }

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java

@@ -3,7 +3,6 @@ package eu.dnetlib.dhp.oa.dedup;
 
 import static org.apache.spark.sql.functions.col;
 
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.function.MapFunction;
@@ -128,8 +127,10 @@ public class SparkPropagateRelation extends AbstractSparkAction {
 				(MapFunction<Relation, String>) r -> String
 					.join(" ", r.getSource(), r.getTarget(), r.getRelType(), r.getSubRelType(), r.getRelClass()),
 				Encoders.STRING())
-			.reduceGroups((ReduceFunction<Relation>) MergeUtils::mergeRelation
-			)
+			.reduceGroups((ReduceFunction<Relation>) (b, a) -> {
+				b.mergeFrom(a);
+				return b;
+			})
 			.map((MapFunction<Tuple2<String, Relation>, Relation>) Tuple2::_2, REL_BEAN_ENC);
 
 		final String outputRelationPath = graphOutputPath + "/relation";

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/config-default.xml

@@ -15,4 +15,12 @@
         <name>oozie.action.sharelib.for.spark</name>
         <value>spark2</value>
     </property>
+    <property>
+        <name>hiveMetastoreUris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>pivotHistoryDatabase</name>
+        <value>&#x200B;</value>
+    </property>
 </configuration>

dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/openorgs/oozie_app/workflow.xml

@@ -198,6 +198,8 @@
             <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
             <arg>--actionSetId</arg><arg>${actionSetId}</arg>
             <arg>--cutConnectedComponent</arg><arg>${cutConnectedComponent}</arg>
+            <arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
+            <arg>--pivotHistoryDatabase</arg><arg>${pivotHistoryDatabase}</arg>
         </spark>
         <ok to="PrepareOrgRels"/>
         <error to="Kill"/>

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json

@@ -73,12 +73,6 @@
     "name": "Irish Nephrology Society",
     "synonym": []
   },
-  {
-    "id": "100011062",
-    "uri": "http://dx.doi.org/10.13039/100011062",
-    "name": "Asian Spinal Cord Network",
-    "synonym": []
-  },
   {
     "id": "100011096",
     "uri": "http://dx.doi.org/10.13039/100011096",
@@ -223,12 +217,6 @@
     "name": "Global Brain Health Institute",
     "synonym": []
   },
-  {
-    "id": "100015776",
-    "uri": "http://dx.doi.org/10.13039/100015776",
-    "name": "Health and Social Care Board",
-    "synonym": []
-  },
   {
     "id": "100015992",
     "uri": "http://dx.doi.org/10.13039/100015992",
@@ -403,18 +391,6 @@
     "name": "Irish Hospice Foundation",
     "synonym": []
   },
-  {
-    "id": "501100001596",
-    "uri": "http://dx.doi.org/10.13039/501100001596",
-    "name": "Irish Research Council for Science, Engineering and Technology",
-    "synonym": []
-  },
-  {
-    "id": "501100001597",
-    "uri": "http://dx.doi.org/10.13039/501100001597",
-    "name": "Irish Research Council for the Humanities and Social Sciences",
-    "synonym": []
-  },
   {
     "id": "501100001598",
     "uri": "http://dx.doi.org/10.13039/501100001598",
@@ -515,7 +491,7 @@
     "id": "501100002081",
     "uri": "http://dx.doi.org/10.13039/501100002081",
     "name": "Irish Research Council",
-    "synonym": []
+    "synonym": ["501100001596", "501100001597"]
   },
   {
     "id": "501100002736",

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala

@@ -3,18 +3,17 @@ package eu.dnetlib.doiboost
 import eu.dnetlib.dhp.application.ArgumentApplicationParser
 import eu.dnetlib.dhp.oa.merge.AuthorMerger
 import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils
 import eu.dnetlib.dhp.schema.oaf.{Organization, Publication, Relation, Dataset => OafDataset}
 import eu.dnetlib.doiboost.mag.ConversionUtil
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkConf
+import org.apache.spark.sql.expressions.Aggregator
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql._
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST.{JField, JObject, JString}
 import org.json4s.jackson.JsonMethods.parse
 import org.slf4j.{Logger, LoggerFactory}
-
 import scala.collection.JavaConverters._
 
 object SparkGenerateDoiBoost {
@@ -79,10 +78,8 @@ object SparkGenerateDoiBoost {
       if (item._2 != null) {
         val otherPub = item._2._2
         if (otherPub != null) {
-          val mergedAuthor = AuthorMerger.mergeAuthor(crossrefPub.getAuthor, otherPub.getAuthor)
-          val res = MergeUtils.mergePublication(crossrefPub, otherPub)
-          res.setAuthor(mergedAuthor);
-          return res
+          crossrefPub.mergeFrom(otherPub)
+          crossrefPub.setAuthor(AuthorMerger.mergeAuthor(crossrefPub.getAuthor, otherPub.getAuthor))
         }
       }
       crossrefPub
@@ -133,13 +130,14 @@ object SparkGenerateDoiBoost {
             // So we have to merge
             val b1 = left._2
             val b2 = right._2
+            b1.mergeFrom(b2)
+            b1.mergeOAFDataInfo(b2)
             val authors = AuthorMerger.mergeAuthor(b1.getAuthor, b2.getAuthor)
-            val merged = MergeUtils.mergePublication(b1, b2);
-            merged.setAuthor(authors)
+            b1.setAuthor(authors)
             if (b2.getId != null && b2.getId.nonEmpty)
-              merged.setId(b2.getId)
+              b1.setId(b2.getId)
             //Return publication Merged
-            (merged.getId, merged)
+            (b1.getId, b1)
           }
         } else {
           // Left is Null so we return right

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala

@@ -587,7 +587,15 @@ case object Crossref2Oaf {
                 "10.13039/501100000266" | "10.13039/501100006041" | "10.13039/501100000265" | "10.13039/501100000270" |
                 "10.13039/501100013589" | "10.13039/501100000271" =>
               generateSimpleRelationFromAward(funder, "ukri________", a => a)
-
+            //HFRI
+            case "10.13039/501100013209" =>
+              generateSimpleRelationFromAward(funder, "hfri________", a => a)
+              val targetId = getProjectId("hfri________", "1e5e62235d094afd01cd56e65112fc63")
+              queue += generateRelation(sourceId, targetId, ModelConstants.IS_PRODUCED_BY)
+              queue += generateRelation(targetId, sourceId, ModelConstants.PRODUCES)
+            //ERASMUS+
+            case "10.13039/501100010790" =>
+              generateSimpleRelationFromAward(funder, "erasmusplus_", a => a)
             case _ => logger.debug("no match for " + funder.DOI.get)
 
           }

dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/mag/MagDataModel.scala

@@ -1,8 +1,8 @@
 package eu.dnetlib.doiboost.mag
 
 import eu.dnetlib.dhp.schema.common.ModelConstants
-import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, MergeUtils}
-import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, Subject}
+import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory
+import eu.dnetlib.dhp.schema.oaf.{Instance, Journal, Publication, StructuredProperty, Subject}
 import eu.dnetlib.doiboost.DoiBoostMappingUtil
 import eu.dnetlib.doiboost.DoiBoostMappingUtil._
 import org.json4s
@@ -142,7 +142,8 @@ case object ConversionUtil {
 
   def mergePublication(a: Publication, b: Publication): Publication = {
     if ((a != null) && (b != null)) {
-      MergeUtils.mergePublication(a, b)
+      a.mergeFrom(b)
+      a
     } else {
       if (a == null) b else a
     }

dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala

@@ -23,15 +23,10 @@ class CrossrefMappingTest {
   val mapper = new ObjectMapper()
 
   @Test
-  def testMissingAuthorParser(): Unit = {
-    val json: String = Source
-      .fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/s41567-022-01757-y.json"))
-      .mkString
+  def testMissingAuthorParser():Unit = {
+    val json: String = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/doiboost/crossref/s41567-022-01757-y.json")).mkString
     val result = Crossref2Oaf.convert(json)
-    result
-      .filter(o => o.isInstanceOf[Publication])
-      .map(p => p.asInstanceOf[Publication])
-      .foreach(p => assertTrue(p.getAuthor.size() > 0))
+    result.filter(o => o.isInstanceOf[Publication]).map(p=> p.asInstanceOf[Publication]).foreach(p =>assertTrue(p.getAuthor.size()>0))
   }
 
   @Test

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/actions/Action.java

@@ -1,39 +0,0 @@
-
-package eu.dnetlib.dhp.bulktag.actions;
-
-import java.io.Serializable;
-import java.util.List;
-
-/**
- * @author miriam.baglioni
- * @Date 22/01/24
- */
-public class Action implements Serializable {
-	private String clazz;
-	private String method;
-	private List<Parameters> params;
-
-	public String getClazz() {
-		return clazz;
-	}
-
-	public void setClazz(String clazz) {
-		this.clazz = clazz;
-	}
-
-	public String getMethod() {
-		return method;
-	}
-
-	public void setMethod(String method) {
-		this.method = method;
-	}
-
-	public List<Parameters> getParams() {
-		return params;
-	}
-
-	public void setParams(List<Parameters> params) {
-		this.params = params;
-	}
-}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/actions/ExecSubstringAction.java

@@ -1,45 +0,0 @@
-
-package eu.dnetlib.dhp.bulktag.actions;
-
-import java.io.Serializable;
-
-/**
- * @author miriam.baglioni
- * @Date 19/01/24
- */
-public class ExecSubstringAction implements Serializable {
-
-	private String value;
-	private String from;
-	private String to;
-
-	public String getValue() {
-		return value;
-	}
-
-	public void setValue(String value) {
-		this.value = value;
-	}
-
-	public String getFrom() {
-		return from;
-	}
-
-	public void setFrom(String from) {
-		this.from = from;
-	}
-
-	public String getTo() {
-		return to;
-	}
-
-	public void setTo(String to) {
-		this.to = to;
-	}
-
-	public String execSubstring() {
-
-		return this.value.substring(Integer.valueOf(this.from), Integer.valueOf(this.to));
-
-	}
-}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/actions/MapModel.java

@@ -1,30 +0,0 @@
-
-package eu.dnetlib.dhp.bulktag.actions;
-
-import java.io.Serializable;
-
-/**
- * @author miriam.baglioni
- * @Date 22/01/24
- */
-public class MapModel implements Serializable {
-
-	private String path;
-	private Action action;
-
-	public String getPath() {
-		return path;
-	}
-
-	public void setPath(String path) {
-		this.path = path;
-	}
-
-	public Action getAction() {
-		return action;
-	}
-
-	public void setAction(Action action) {
-		this.action = action;
-	}
-}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/actions/Parameters.java

@@ -1,29 +0,0 @@
-
-package eu.dnetlib.dhp.bulktag.actions;
-
-import java.io.Serializable;
-
-/**
- * @author miriam.baglioni
- * @Date 22/01/24
- */
-public class Parameters implements Serializable {
-	private String paramName;
-	private String paramValue;
-
-	public String getParamName() {
-		return paramName;
-	}
-
-	public void setParamName(String paramName) {
-		this.paramName = paramName;
-	}
-
-	public String getParamValue() {
-		return paramValue;
-	}
-
-	public void setParamValue(String paramValue) {
-		this.paramValue = paramValue;
-	}
-}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraints.java

@@ -53,6 +53,8 @@ public class Constraints implements Serializable {
 
 		for (Constraint sc : constraint) {
 			boolean verified = false;
+			if(!param.containsKey(sc.getField()))
+				return false;
 			for (String value : param.get(sc.getField())) {
 				if (sc.verifyCriteria(value.trim())) {
 					verified = true;

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ProtoMap.java

@@ -4,9 +4,7 @@ package eu.dnetlib.dhp.bulktag.community;
 import java.io.Serializable;
 import java.util.HashMap;
 
-import eu.dnetlib.dhp.bulktag.actions.MapModel;
-
-public class ProtoMap extends HashMap<String, MapModel> implements Serializable {
+public class ProtoMap extends HashMap<String, String> implements Serializable {
 
 	public ProtoMap() {
 		super();

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java

@@ -5,8 +5,6 @@ import static eu.dnetlib.dhp.bulktag.community.TaggingConstants.*;
 import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
 
 import java.io.Serializable;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -17,10 +15,7 @@ import org.slf4j.LoggerFactory;
 import com.google.gson.Gson;
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.JsonPath;
-import com.jayway.jsonpath.PathNotFoundException;
 
-import eu.dnetlib.dhp.bulktag.actions.MapModel;
-import eu.dnetlib.dhp.bulktag.actions.Parameters;
 import eu.dnetlib.dhp.bulktag.eosc.EoscIFTag;
 import eu.dnetlib.dhp.schema.oaf.*;
 import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
@@ -40,59 +35,27 @@ public class ResultTagger implements Serializable {
 		return (tmp != clist.size());
 	}
 
-	private Map<String, List<String>> getParamMap(final Result result, Map<String, MapModel> params)
-		throws NoSuchMethodException, InvocationTargetException {
+	private Map<String, List<String>> getParamMap(final Result result, Map<String, String> params) {
 		Map<String, List<String>> param = new HashMap<>();
 		String json = new Gson().toJson(result, Result.class);
 		DocumentContext jsonContext = JsonPath.parse(json);
-
 		if (params == null) {
 			params = new HashMap<>();
 		}
 		for (String key : params.keySet()) {
-			MapModel mapModel = params.get(key);
-
 			try {
-				String path = mapModel.getPath();
-				Object obj = jsonContext.read(path);
-				List<String> pathValue;
-				if (obj instanceof java.lang.String)
-					pathValue = Arrays.asList((String) obj);
-				else
-					pathValue = (List<String>) obj;
-				if (Optional.ofNullable(mapModel.getAction()).isPresent()) {
-					Class<?> c = Class.forName(mapModel.getAction().getClazz());
-					Object class_instance = c.newInstance();
-					Method setField = c.getMethod("setValue", String.class);
-					setField.invoke(class_instance, pathValue.get(0));
-					for (Parameters p : mapModel.getAction().getParams()) {
-						setField = c.getMethod("set" + p.getParamName(), String.class);
-						setField.invoke(class_instance, p.getParamValue());
-					}
-
-					param
-						.put(
-							key, Arrays
-								.asList((String) c.getMethod(mapModel.getAction().getMethod()).invoke(class_instance)));
-
-				}
-
-				else {
-					param.put(key, pathValue);
-				}
-
-			} catch (PathNotFoundException | ClassNotFoundException | InstantiationException
-				| IllegalAccessException e) {
+				param.put(key, jsonContext.read(params.get(key)));
+			} catch (com.jayway.jsonpath.PathNotFoundException e) {
 				param.put(key, new ArrayList<>());
 			}
 		}
 		return param;
-
 	}
 
 	public <R extends Result> R enrichContextCriteria(
-		final R result, final CommunityConfiguration conf, final Map<String, MapModel> criteria)
-		throws InvocationTargetException, NoSuchMethodException {
+		final R result, final CommunityConfiguration conf, final Map<String, String> criteria) {
+
+		final Map<String, List<String>> param = getParamMap(result, criteria);
 
 		// Verify if the entity is deletedbyinference. In case verify if to clean the context list
 		// from all the zenodo communities
@@ -101,8 +64,6 @@ public class ResultTagger implements Serializable {
 			return result;
 		}
 
-		final Map<String, List<String>> param = getParamMap(result, criteria);
-
 		// Execute the EOSCTag for the services
 		switch (result.getResulttype().getClassid()) {
 			case PUBLICATION_RESULTTYPE_CLASSID:

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/criteria/GreatThanVerb.java

@@ -1,34 +0,0 @@
-
-package eu.dnetlib.dhp.bulktag.criteria;
-
-import java.io.Serializable;
-
-/**
- * @author miriam.baglioni
- * @Date 11/11/22
- */
-@VerbClass("greater_than")
-public class GreatThanVerb implements Selection, Serializable {
-
-	private String param;
-
-	public GreatThanVerb() {
-	}
-
-	public GreatThanVerb(final String param) {
-		this.param = param;
-	}
-
-	@Override
-	public boolean apply(String value) {
-		return value.compareTo(param) > 0;
-	}
-
-	public String getParam() {
-		return param;
-	}
-
-	public void setParam(String param) {
-		this.param = param;
-	}
-}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/criteria/LessThanVerb.java

@@ -1,34 +0,0 @@
-
-package eu.dnetlib.dhp.bulktag.criteria;
-
-import java.io.Serializable;
-
-/**
- * @author miriam.baglioni
- * @Date 11/11/22
- */
-@VerbClass("lesser_than")
-public class LessThanVerb implements Selection, Serializable {
-
-	private String param;
-
-	public LessThanVerb() {
-	}
-
-	public LessThanVerb(final String param) {
-		this.param = param;
-	}
-
-	@Override
-	public boolean apply(String value) {
-		return value.compareTo(param) < 0;
-	}
-
-	public String getParam() {
-		return param;
-	}
-
-	public void setParam(String param) {
-		this.param = param;
-	}
-}

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/SparkCountryPropagationJob.java

@@ -14,6 +14,7 @@ import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SaveMode;
 import org.apache.spark.sql.SparkSession;
 import org.slf4j.Logger;
@@ -84,19 +85,26 @@ public class SparkCountryPropagationJob {
 		Dataset<R> res = readPath(spark, sourcePath, resultClazz);
 
 		log.info("Reading prepared info: {}", preparedInfoPath);
-		Dataset<ResultCountrySet> prepared = spark
+		final Dataset<Row> preparedInfoRaw = spark
 			.read()
-			.json(preparedInfoPath)
-			.as(Encoders.bean(ResultCountrySet.class));
-
-		res
-			.joinWith(prepared, res.col("id").equalTo(prepared.col("resultId")), "left_outer")
-			.map(getCountryMergeFn(), Encoders.bean(resultClazz))
-			.write()
-			.option("compression", "gzip")
-			.mode(SaveMode.Overwrite)
-			.json(outputPath);
+			.json(preparedInfoPath);
 
+		if (!preparedInfoRaw.isEmpty()) {
+			final Dataset<ResultCountrySet> prepared = preparedInfoRaw.as(Encoders.bean(ResultCountrySet.class));
+			res
+				.joinWith(prepared, res.col("id").equalTo(prepared.col("resultId")), "left_outer")
+				.map(getCountryMergeFn(), Encoders.bean(resultClazz))
+				.write()
+				.option("compression", "gzip")
+				.mode(SaveMode.Overwrite)
+				.json(outputPath);
+		} else {
+			res
+				.write()
+				.option("compression", "gzip")
+				.mode(SaveMode.Overwrite)
+				.json(outputPath);
+		}
 	}
 
 	private static <R extends Result> MapFunction<Tuple2<R, ResultCountrySet>, R> getCountryMergeFn() {

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java

@@ -149,7 +149,7 @@ public class SparkResultToCommunityFromOrganizationJob {
 					}
 				}
 				// res.setContext(propagatedContexts);
-				// return MergeUtils.mergeResult(ret, res);
+				// ret.mergeFrom(res);
 			}
 			return ret;
 		};

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java

@@ -13,7 +13,6 @@ import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.function.MapFunction;
@@ -25,6 +24,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList;
+import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob;
 import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.schema.oaf.Context;
@@ -161,7 +162,7 @@ public class SparkResultToCommunityFromProject implements Serializable {
 					}
 				}
 				res.setContext(propagatedContexts);
-				return MergeUtils.checkedMerge(ret, res);
+				ret.mergeFrom(res);
 			}
 			return ret;
 		};

dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/SparkResultToCommunityThroughSemRelJob.java

@@ -25,118 +25,118 @@ import scala.Tuple2;
 
 public class SparkResultToCommunityThroughSemRelJob {
 
-    private static final Logger log = LoggerFactory.getLogger(SparkResultToCommunityThroughSemRelJob.class);
+	private static final Logger log = LoggerFactory.getLogger(SparkResultToCommunityThroughSemRelJob.class);
 
-    public static void main(String[] args) throws Exception {
+	public static void main(String[] args) throws Exception {
 
-        String jsonConfiguration = IOUtils
-                .toString(
-                        SparkResultToCommunityThroughSemRelJob.class
-                                .getResourceAsStream(
-                                        "/eu/dnetlib/dhp/wf/subworkflows/resulttocommunityfromsemrel/input_communitytoresult_parameters.json"));
+		String jsonConfiguration = IOUtils
+			.toString(
+				SparkResultToCommunityThroughSemRelJob.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/wf/subworkflows/resulttocommunityfromsemrel/input_communitytoresult_parameters.json"));
 
-        final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
 
-        parser.parseArgument(args);
+		parser.parseArgument(args);
 
-        Boolean isSparkSessionManaged = isSparkSessionManaged(parser);
-        log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+		Boolean isSparkSessionManaged = isSparkSessionManaged(parser);
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
 
-        String inputPath = parser.get("sourcePath");
-        log.info("inputPath: {}", inputPath);
+		String inputPath = parser.get("sourcePath");
+		log.info("inputPath: {}", inputPath);
 
-        final String outputPath = parser.get("outputPath");
-        log.info("outputPath: {}", outputPath);
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath: {}", outputPath);
 
-        final String preparedInfoPath = parser.get("preparedInfoPath");
-        log.info("preparedInfoPath: {}", preparedInfoPath);
+		final String preparedInfoPath = parser.get("preparedInfoPath");
+		log.info("preparedInfoPath: {}", preparedInfoPath);
 
-        SparkConf conf = new SparkConf();
-        conf.set("hive.metastore.uris", parser.get("hive_metastore_uris"));
+		SparkConf conf = new SparkConf();
+		conf.set("hive.metastore.uris", parser.get("hive_metastore_uris"));
 
-        final String resultClassName = parser.get("resultTableName");
-        log.info("resultTableName: {}", resultClassName);
+		final String resultClassName = parser.get("resultTableName");
+		log.info("resultTableName: {}", resultClassName);
 
-        final Boolean saveGraph = Optional
-                .ofNullable(parser.get("saveGraph"))
-                .map(Boolean::valueOf)
-                .orElse(Boolean.TRUE);
-        log.info("saveGraph: {}", saveGraph);
+		final Boolean saveGraph = Optional
+			.ofNullable(parser.get("saveGraph"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+		log.info("saveGraph: {}", saveGraph);
 
-        @SuppressWarnings("unchecked")
-        Class<? extends Result> resultClazz = (Class<? extends Result>) Class.forName(resultClassName);
+		@SuppressWarnings("unchecked")
+		Class<? extends Result> resultClazz = (Class<? extends Result>) Class.forName(resultClassName);
 
-        runWithSparkHiveSession(
-                conf,
-                isSparkSessionManaged,
-                spark -> {
-                    if (isTest(parser)) {
-                        removeOutputDir(spark, outputPath);
-                    }
-                    if (saveGraph) {
-                        execPropagation(
-                                spark, inputPath, outputPath, preparedInfoPath, resultClazz);
-                    }
-                });
-    }
+		runWithSparkHiveSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				if (isTest(parser)) {
+					removeOutputDir(spark, outputPath);
+				}
+				if (saveGraph) {
+					execPropagation(
+						spark, inputPath, outputPath, preparedInfoPath, resultClazz);
+				}
+			});
+	}
 
-    private static <R extends Result> void execPropagation(
-            SparkSession spark,
-            String inputPath,
-            String outputPath,
-            String preparedInfoPath,
-            Class<R> resultClazz) {
+	private static <R extends Result> void execPropagation(
+		SparkSession spark,
+		String inputPath,
+		String outputPath,
+		String preparedInfoPath,
+		Class<R> resultClazz) {
 
-        Dataset<ResultCommunityList> possibleUpdates = readPath(spark, preparedInfoPath, ResultCommunityList.class);
-        Dataset<R> result = readPath(spark, inputPath, resultClazz);
+		Dataset<ResultCommunityList> possibleUpdates = readPath(spark, preparedInfoPath, ResultCommunityList.class);
+		Dataset<R> result = readPath(spark, inputPath, resultClazz);
 
-        result
-                .joinWith(
-                        possibleUpdates,
-                        result.col("id").equalTo(possibleUpdates.col("resultId")),
-                        "left_outer")
-                .map(contextUpdaterFn(), Encoders.bean(resultClazz))
-                .write()
-                .mode(SaveMode.Overwrite)
-                .option("compression", "gzip")
-                .json(outputPath);
+		result
+			.joinWith(
+				possibleUpdates,
+				result.col("id").equalTo(possibleUpdates.col("resultId")),
+				"left_outer")
+			.map(contextUpdaterFn(), Encoders.bean(resultClazz))
+			.write()
+			.mode(SaveMode.Overwrite)
+			.option("compression", "gzip")
+			.json(outputPath);
 
-    }
+	}
 
-    private static <R extends Result> MapFunction<Tuple2<R, ResultCommunityList>, R> contextUpdaterFn() {
-        return value -> {
-            R ret = value._1();
-            Optional<ResultCommunityList> rcl = Optional.ofNullable(value._2());
-            if (rcl.isPresent()) {
-                Set<String> contexts = new HashSet<>();
-                ret.getContext().forEach(c -> contexts.add(c.getId()));
-                rcl
-                        .get()
-                        .getCommunityList()
-                        .stream()
-                        .forEach(
-                                c -> {
-                                    if (!contexts.contains(c)) {
-                                        Context newContext = new Context();
-                                        newContext.setId(c);
-                                        newContext
-                                                .setDataInfo(
-                                                        Arrays
-                                                                .asList(
-                                                                        getDataInfo(
-                                                                                PROPAGATION_DATA_INFO_TYPE,
-                                                                                PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_ID,
-                                                                                PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_NAME,
-                                                                                ModelConstants.DNET_PROVENANCE_ACTIONS)));
-                                        ret.getContext().add(newContext);
-                                    }
+	private static <R extends Result> MapFunction<Tuple2<R, ResultCommunityList>, R> contextUpdaterFn() {
+		return value -> {
+			R ret = value._1();
+			Optional<ResultCommunityList> rcl = Optional.ofNullable(value._2());
+			if (rcl.isPresent()) {
+				Set<String> contexts = new HashSet<>();
+				ret.getContext().forEach(c -> contexts.add(c.getId()));
+				rcl
+					.get()
+					.getCommunityList()
+					.stream()
+					.forEach(
+						c -> {
+							if (!contexts.contains(c)) {
+								Context newContext = new Context();
+								newContext.setId(c);
+								newContext
+									.setDataInfo(
+										Arrays
+											.asList(
+												getDataInfo(
+													PROPAGATION_DATA_INFO_TYPE,
+													PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_ID,
+													PROPAGATION_RESULT_COMMUNITY_SEMREL_CLASS_NAME,
+													ModelConstants.DNET_PROVENANCE_ACTIONS)));
+								ret.getContext().add(newContext);
+							}
 
-                                });
+						});
 
-            }
+			}
 
-            return ret;
-        };
-    }
+			return ret;
+		};
+	}
 
 }

dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java

@@ -13,7 +13,6 @@ import org.apache.commons.io.IOUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.FilterFunction;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SparkSession;
@@ -32,26 +31,18 @@ public class BulkTagJobTest {
 
 	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
-	public static final String pathMap = "{\"author\":{\"path\":\"$['author'][*]['fullname']\"}," +
-		" \"title\":{\"path\":\"$['title'][*]['value']\"}, " +
-		" \"orcid\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid')]['value']\"} , " +
-		" \"orcid_pending\":{\"path\":\"$['author'][*]['pid'][*][?(@['qualifier']['classid']=='orcid_pending')]['value']\"} ,"
-		+
-		"\"contributor\" : {\"path\":\"$['contributor'][*]['value']\"}," +
-		" \"description\" : {\"path\":\"$['description'][*]['value']\"}," +
-		" \"subject\" :{\"path\":\"$['subject'][*]['value']\"}, " +
-		" \"fos\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"} , " +
-		"\"sdg\" : {\"path\":\"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"}," +
-		"\"journal\":{\"path\":\"$['journal'].name\"}," +
-		"\"hostedby\":{\"path\":\"$['instance'][*]['hostedby']['key']\"}," +
-		"\"collectedfrom\":{\"path\":\"$['instance'][*]['collectedfrom']['key']\"}," +
-		"\"publisher\":{\"path\":\"$['publisher'].value\"}," +
-		"\"publicationyear\":{\"path\":\"$['dateofacceptance'].value\", " +
-		" \"action\":{\"clazz\":\"eu.dnetlib.dhp.bulktag.actions.ExecSubstringAction\"," +
-		"\"method\":\"execSubstring\"," +
-		"\"params\":[" +
-		"{\"paramName\":\"From\",  \"paramValue\":0}, " +
-		"{\"paramName\":\"To\",\"paramValue\":4}]}}}";
+	public static final String pathMap = "{ \"author\" : \"$['author'][*]['fullname']\","
+		+ "  \"title\" : \"$['title'][*]['value']\","
+		+ "  \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
+		+ "  \"contributor\" : \"$['contributor'][*]['value']\","
+		+ "  \"description\" : \"$['description'][*]['value']\", "
+		+ " \"subject\" :\"$['subject'][*]['value']\" , " +
+		"\"fos\" : \"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"," +
+		"\"sdg\" : \"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"," +
+		"\"hostedby\" : \"$['instance'][*]['hostedby']['key']\" , " +
+		"\"collectedfrom\" : \"$['instance'][*]['collectedfrom']['key']\"," +
+		"\"publisher\":\"$['publisher'].value\"," +
+		"\"publicationyear\":\"$['dateofacceptance'].value\"} ";
 
 	private static SparkSession spark;
 
@@ -1609,94 +1600,4 @@ public class BulkTagJobTest {
 		Assertions.assertEquals(0, spark.sql(query).count());
 	}
 
-	@Test
-	void pubdateTest() throws Exception {
-
-		final String pathMap = BulkTagJobTest.pathMap;
-		SparkBulkTagJob
-			.main(
-				new String[] {
-					"-isSparkSessionManaged", Boolean.FALSE.toString(),
-					"-sourcePath",
-					getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/publicationyear/").getPath(),
-					"-taggingConf",
-					IOUtils
-						.toString(
-							BulkTagJobTest.class
-								.getResourceAsStream(
-									"/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_publicationdate.xml")),
-					"-outputPath", workingDir.toString() + "/",
-					"-pathMap", pathMap
-				});
-
-		final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
-
-		JavaRDD<Dataset> tmp = sc
-			.textFile(workingDir.toString() + "/dataset")
-			.map(item -> OBJECT_MAPPER.readValue(item, Dataset.class));
-
-		Assertions.assertEquals(10, tmp.count());
-		org.apache.spark.sql.Dataset<Dataset> verificationDataset = spark
-			.createDataset(tmp.rdd(), Encoders.bean(Dataset.class));
-
-		verificationDataset.createOrReplaceTempView("dataset");
-
-		String query = "select id, MyT.id community, MyD.provenanceaction.classid "
-			+ "from dataset "
-			+ "lateral view explode(context) c as MyT "
-			+ "lateral view explode(MyT.datainfo) d as MyD "
-			+ "where MyD.inferenceprovenance = 'bulktagging'";
-
-		org.apache.spark.sql.Dataset<Row> queryResult = spark.sql(query);
-		queryResult.show(false);
-		Assertions.assertEquals(5, queryResult.count());
-
-		Assertions
-			.assertEquals(
-				1,
-				queryResult
-					.filter(
-						(FilterFunction<Row>) r -> r
-							.getAs("id")
-							.equals("50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529"))
-					.count());
-		Assertions
-			.assertEquals(
-				1,
-				queryResult
-					.filter(
-						(FilterFunction<Row>) r -> r
-							.getAs("id")
-							.equals("50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf"))
-					.count());
-		Assertions
-			.assertEquals(
-				1,
-				queryResult
-					.filter(
-						(FilterFunction<Row>) r -> r
-							.getAs("id")
-							.equals("50|od______3989::7fcbe3a03280663cddebfd3cb9203177"))
-					.count());
-		Assertions
-			.assertEquals(
-				1,
-				queryResult
-					.filter(
-						(FilterFunction<Row>) r -> r
-							.getAs("id")
-							.equals("50|od______3989::d791339867bec6d3eb2104deeb4e4961"))
-					.count());
-		Assertions
-			.assertEquals(
-				1,
-				queryResult
-					.filter(
-						(FilterFunction<Row>) r -> r
-							.getAs("id")
-							.equals("50|od______3989::d90d3a1f64ad264b5ebed8a35b280343"))
-					.count());
-
-	}
-
 }