From b5cf505cc6dff654e075e9c1abd70a52397afc66 Mon Sep 17 00:00:00 2001
From: "michele.artini" <michele.artini@isti.cnr.it>
Date: Wed, 28 Apr 2021 16:00:24 +0200
Subject: [PATCH 1/5] partial implementation of the ROR->actionset workflow

---
 .../ror/GenerateRorActionSetJob.java          | 194 ++++++++++++++++++
 .../dhp/actionmanager/ror/model/Address.java  | 100 +++++++++
 .../dhp/actionmanager/ror/model/Country.java  |  28 +++
 .../ror/model/ExternalIdType.java             |  30 +++
 .../actionmanager/ror/model/ExternalIds.java  |  55 +++++
 .../ror/model/GeonamesAdmin.java              |  46 +++++
 .../actionmanager/ror/model/GeonamesCity.java |  82 ++++++++
 .../dhp/actionmanager/ror/model/GridType.java |  32 +++
 .../dhp/actionmanager/ror/model/Label.java    |  28 +++
 .../dhp/actionmanager/ror/model/License.java  |  28 +++
 .../actionmanager/ror/model/NameAndCode.java  |  28 +++
 .../actionmanager/ror/model/Relationship.java |  37 ++++
 .../ror/model/RorOrganization.java            | 156 ++++++++++++++
 .../ror/action_set_parameters.json            |  14 ++
 .../ror/oozie_app/config-default.xml          |  58 ++++++
 .../actionmanager/ror/oozie_app/workflow.xml  |  55 +++++
 16 files changed, 971 insertions(+)
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/config-default.xml
 create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
new file mode 100644
index 000000000..81f0c55f8
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
@@ -0,0 +1,194 @@
+
+package eu.dnetlib.dhp.actionmanager.ror;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+import static eu.dnetlib.dhp.schema.common.ModelConstants.ENTITYREGISTRY_PROVENANCE_ACTION;
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.dataInfo;
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.field;
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.listKeyValues;
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.qualifier;
+import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.structuredProperty;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob;
+import eu.dnetlib.dhp.actionmanager.ror.model.RorOrganization;
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.oaf.DataInfo;
+import eu.dnetlib.dhp.schema.oaf.Field;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Organization;
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
+import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
+import eu.dnetlib.dhp.utils.DHPUtils;
+import scala.Tuple2;
+
+public class GenerateRorActionSetJob {
+
+	private static final String COUNTRIES_VOC = "dnet:countries";
+
+	private static final Logger log = LoggerFactory.getLogger(GenerateRorActionSetJob.class);
+
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	private static final List<KeyValue> ROR_COLLECTED_FROM = listKeyValues("10|openaire____::993a7ae7a863813cf95028b50708e222", "ROR");
+
+	private static final DataInfo ROR_DATA_INFO = dataInfo(false, "", false, false, ENTITYREGISTRY_PROVENANCE_ACTION, "0.92");
+
+	private static final Qualifier ROR_PID_TYPE = qualifier("ROR", "ROR", "dnet:pid_types", "dnet:pid_types");
+	private static final Qualifier GRID_PID_TYPE = qualifier("GRID", "GRID", "dnet:pid_types", "dnet:pid_types");
+	private static final Qualifier WIKIDATA_PID_TYPE = qualifier("Wikidata", "Wikidata", "dnet:pid_types", "dnet:pid_types");
+	private static final Qualifier ORGREF_PID_TYPE = qualifier("OrgRef", "OrgRef", "dnet:pid_types", "dnet:pid_types");
+	private static final Qualifier ISNI_PID_TYPE = qualifier("ISNI", "ISNI", "dnet:pid_types", "dnet:pid_types");
+	private static final Qualifier FUNDREF_PID_TYPE = qualifier("FundRef", "FundRef", "dnet:pid_types", "dnet:pid_types");
+
+	public static void main(final String[] args) throws Exception {
+
+		final String jsonConfiguration = IOUtils
+			.toString(SparkAtomicActionJob.class
+				.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+
+		parser.parseArgument(args);
+
+		final Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		final String inputPath = parser.get("inputPath");
+		log.info("inputPath: {}", inputPath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath {}: ", outputPath);
+
+		final SparkConf conf = new SparkConf();
+
+		runWithSparkSession(conf, isSparkSessionManaged, spark -> {
+			removeOutputDir(spark, outputPath);
+			processRorOrganizations(spark, inputPath, outputPath);
+		});
+	}
+
+	private static void removeOutputDir(final SparkSession spark, final String path) {
+		HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
+	}
+
+	private static void processRorOrganizations(final SparkSession spark,
+		final String inputPath,
+		final String outputPath) {
+
+		readInputPath(spark, inputPath)
+			.map(GenerateRorActionSetJob::convertRorOrg, Encoders.bean(Organization.class))
+			.toJavaRDD()
+			.map(o -> new AtomicAction<>(Organization.class, o))
+			.mapToPair(aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+				new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
+
+	}
+
+	private static Organization convertRorOrg(final RorOrganization r) {
+
+		final Date now = new Date();
+
+		final Organization o = new Organization();
+
+		o.setId(String.format("20|ror_________::%s", DHPUtils.md5(r.getId())));
+		o.setOriginalId(Arrays.asList(r.getId()));
+		o.setCollectedfrom(ROR_COLLECTED_FROM);
+		o.setPid(pids(r));
+		o.setDateofcollection(now.toString());
+		o.setDateoftransformation(now.toString());
+		o.setExtraInfo(new ArrayList<>()); // Values not present in the file
+		o.setOaiprovenance(null); // Values not present in the file
+		o.setLegalshortname(field(r.getAcronyms().stream().findFirst().orElse(r.getName()), ROR_DATA_INFO));
+		o.setLegalname(field(r.getName(), ROR_DATA_INFO));
+		o.setAlternativeNames(alternativeNames(r));
+		o.setWebsiteurl(field(r.getLinks().stream().findFirst().orElse(null), ROR_DATA_INFO));
+		o.setLogourl(null);
+		o.setEclegalbody(null);
+		o.setEclegalperson(null);
+		o.setEcnonprofit(null);
+		o.setEcresearchorganization(null);
+		o.setEchighereducation(null);
+		o.setEcinternationalorganizationeurinterests(null);
+		o.setEcinternationalorganization(null);
+		o.setEcenterprise(null);
+		o.setEcsmevalidated(null);
+		o.setEcnutscode(null);
+		if (r.getCountry() != null) {
+			o.setCountry(qualifier(r.getCountry().getCountryCode(), r.getCountry().getCountryName(), COUNTRIES_VOC, COUNTRIES_VOC));
+		} else {
+			o.setCountry(null);
+		}
+		o.setDataInfo(ROR_DATA_INFO);
+		o.setLastupdatetimestamp(now.getTime());
+
+		return o;
+	}
+
+	private static List<StructuredProperty> pids(final RorOrganization r) {
+		final List<StructuredProperty> pids = new ArrayList<>();
+		pids.add(structuredProperty(r.getId(), ROR_PID_TYPE, ROR_DATA_INFO));
+		pids.add(structuredProperty(r.getExternalIds().getGrid().getAll(), GRID_PID_TYPE, ROR_DATA_INFO));
+		pids.addAll(pids(r.getExternalIds().getFundRef().getAll(), FUNDREF_PID_TYPE));
+		pids.addAll(pids(r.getExternalIds().getIsni().getAll(), ISNI_PID_TYPE));
+		pids.addAll(pids(r.getExternalIds().getOrgRef().getAll(), ORGREF_PID_TYPE));
+		pids.addAll(pids(r.getExternalIds().getWikidata().getAll(), WIKIDATA_PID_TYPE));
+		return pids;
+	}
+
+	private static List<StructuredProperty> pids(final List<String> list, final Qualifier pidType) {
+		if (list == null) { return new ArrayList<>(); }
+		return list.stream()
+			.filter(StringUtils::isNotBlank)
+			.distinct()
+			.map(s -> structuredProperty(s, pidType, ROR_DATA_INFO))
+			.collect(Collectors.toList());
+	}
+
+	private static List<Field<String>> alternativeNames(final RorOrganization r) {
+		final Set<String> names = new LinkedHashSet<>();
+		names.addAll(r.getAliases());
+		names.addAll(r.getAcronyms());
+
+		return names.stream().filter(StringUtils::isNotBlank).map(s -> field(s, ROR_DATA_INFO)).collect(Collectors.toList());
+	}
+
+	private static Dataset<RorOrganization> readInputPath(
+		final SparkSession spark,
+		final String inputPath) {
+		return spark
+			.read()
+			.textFile(inputPath)
+			.map((MapFunction<String, RorOrganization>) value -> OBJECT_MAPPER.readValue(value, RorOrganization.class), Encoders.bean(RorOrganization.class));
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java
new file mode 100644
index 000000000..68dc62e26
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java
@@ -0,0 +1,100 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class Address implements Serializable {
+
+	private Float lat;
+	private String stateCode;
+	private Integer countryGeonamesId;
+	private Float lng;
+	private String state;
+	private String city;
+	private GeonamesCity geonamesCity;
+	private String postcode;
+	private Boolean primary;
+	private String line;
+	private final static long serialVersionUID = 2444635485253443195L;
+
+	public Float getLat() {
+		return lat;
+	}
+
+	public void setLat(final Float lat) {
+		this.lat = lat;
+	}
+
+	public String getStateCode() {
+		return stateCode;
+	}
+
+	public void setStateCode(final String stateCode) {
+		this.stateCode = stateCode;
+	}
+
+	public Integer getCountryGeonamesId() {
+		return countryGeonamesId;
+	}
+
+	public void setCountryGeonamesId(final Integer countryGeonamesId) {
+		this.countryGeonamesId = countryGeonamesId;
+	}
+
+	public Float getLng() {
+		return lng;
+	}
+
+	public void setLng(final Float lng) {
+		this.lng = lng;
+	}
+
+	public String getState() {
+		return state;
+	}
+
+	public void setState(final String state) {
+		this.state = state;
+	}
+
+	public String getCity() {
+		return city;
+	}
+
+	public void setCity(final String city) {
+		this.city = city;
+	}
+
+	public GeonamesCity getGeonamesCity() {
+		return geonamesCity;
+	}
+
+	public void setGeonamesCity(final GeonamesCity geonamesCity) {
+		this.geonamesCity = geonamesCity;
+	}
+
+	public String getPostcode() {
+		return postcode;
+	}
+
+	public void setPostcode(final String postcode) {
+		this.postcode = postcode;
+	}
+
+	public Boolean getPrimary() {
+		return primary;
+	}
+
+	public void setPrimary(final Boolean primary) {
+		this.primary = primary;
+	}
+
+	public String getLine() {
+		return line;
+	}
+
+	public void setLine(final String line) {
+		this.line = line;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java
new file mode 100644
index 000000000..08ca939f2
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java
@@ -0,0 +1,28 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class Country implements Serializable {
+
+	private String countryCode;
+	private String countryName;
+	private final static long serialVersionUID = 4357848706229493627L;
+
+	public String getCountryCode() {
+		return countryCode;
+	}
+
+	public void setCountryCode(final String countryCode) {
+		this.countryCode = countryCode;
+	}
+
+	public String getCountryName() {
+		return countryName;
+	}
+
+	public void setCountryName(final String countryName) {
+		this.countryName = countryName;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
new file mode 100644
index 000000000..42406f097
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ExternalIdType implements Serializable {
+
+	private List<String> all = new ArrayList<>();
+	private String preferred;
+	private final static long serialVersionUID = 2616688352998387611L;
+
+	public List<String> getAll() {
+		return all;
+	}
+
+	public void setAll(final List<String> all) {
+		this.all = all;
+	}
+
+	public String getPreferred() {
+		return preferred;
+	}
+
+	public void setPreferred(final String preferred) {
+		this.preferred = preferred;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
new file mode 100644
index 000000000..9ea1ccdb8
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
@@ -0,0 +1,55 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class ExternalIds implements Serializable {
+
+	private ExternalIdType wikidata;
+	private ExternalIdType orgRef;
+	private ExternalIdType isni;
+	private ExternalIdType fundRef;
+	private GridType grid;
+	private final static long serialVersionUID = 686536347353680869L;
+
+	public ExternalIdType getWikidata() {
+		return wikidata;
+	}
+
+	public void setWikidata(final ExternalIdType wikidata) {
+		this.wikidata = wikidata;
+	}
+
+	public ExternalIdType getOrgRef() {
+		return orgRef;
+	}
+
+	public void setOrgRef(final ExternalIdType orgRef) {
+		this.orgRef = orgRef;
+	}
+
+	public ExternalIdType getIsni() {
+		return isni;
+	}
+
+	public void setIsni(final ExternalIdType isni) {
+		this.isni = isni;
+	}
+
+	public ExternalIdType getFundRef() {
+		return fundRef;
+	}
+
+	public void setFundRef(final ExternalIdType fundRef) {
+		this.fundRef = fundRef;
+	}
+
+	public GridType getGrid() {
+		return grid;
+	}
+
+	public void setGrid(final GridType grid) {
+		this.grid = grid;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java
new file mode 100644
index 000000000..a446c092f
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java
@@ -0,0 +1,46 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class GeonamesAdmin implements Serializable {
+
+	private String asciiName;
+	private Integer id;
+	private String name;
+	private String code;
+	private final static long serialVersionUID = 7294958526269195673L;
+
+	public String getAsciiName() {
+		return asciiName;
+	}
+
+	public void setAsciiName(final String asciiName) {
+		this.asciiName = asciiName;
+	}
+
+	public Integer getId() {
+		return id;
+	}
+
+	public void setId(final Integer id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getCode() {
+		return code;
+	}
+
+	public void setCode(final String code) {
+		this.code = code;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java
new file mode 100644
index 000000000..9c98bebd1
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java
@@ -0,0 +1,82 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class GeonamesCity implements Serializable {
+
+	private GeonamesAdmin geonamesAdmin1;
+	private GeonamesAdmin geonamesAdmin2;
+	private String city;
+	private Integer id;
+	private NameAndCode nutsLevel1;
+	private NameAndCode nutsLevel2;
+	private NameAndCode nutsLevel3;
+	private License license;
+	private final static long serialVersionUID = -8389480201526252955L;
+
+	public NameAndCode getNutsLevel2() {
+		return nutsLevel2;
+	}
+
+	public void setNutsLevel2(final NameAndCode nutsLevel2) {
+		this.nutsLevel2 = nutsLevel2;
+	}
+
+	public GeonamesAdmin getGeonamesAdmin2() {
+		return geonamesAdmin2;
+	}
+
+	public void setGeonamesAdmin2(final GeonamesAdmin geonamesAdmin2) {
+		this.geonamesAdmin2 = geonamesAdmin2;
+	}
+
+	public GeonamesAdmin getGeonamesAdmin1() {
+		return geonamesAdmin1;
+	}
+
+	public void setGeonamesAdmin1(final GeonamesAdmin geonamesAdmin1) {
+		this.geonamesAdmin1 = geonamesAdmin1;
+	}
+
+	public String getCity() {
+		return city;
+	}
+
+	public void setCity(final String city) {
+		this.city = city;
+	}
+
+	public Integer getId() {
+		return id;
+	}
+
+	public void setId(final Integer id) {
+		this.id = id;
+	}
+
+	public NameAndCode getNutsLevel1() {
+		return nutsLevel1;
+	}
+
+	public void setNutsLevel1(final NameAndCode nutsLevel1) {
+		this.nutsLevel1 = nutsLevel1;
+	}
+
+	public NameAndCode getNutsLevel3() {
+		return nutsLevel3;
+	}
+
+	public void setNutsLevel3(final NameAndCode nutsLevel3) {
+		this.nutsLevel3 = nutsLevel3;
+	}
+
+	public License getLicense() {
+		return license;
+	}
+
+	public void setLicense(final License license) {
+		this.license = license;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
new file mode 100644
index 000000000..739ea22e0
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
@@ -0,0 +1,32 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class GridType implements Serializable {
+
+	/**
+	 *
+	 */
+	private static final long serialVersionUID = -5605887658267581353L;
+
+	private String all;
+	private String preferred;
+
+	public String getAll() {
+		return all;
+	}
+
+	public void setAll(final String all) {
+		this.all = all;
+	}
+
+	public String getPreferred() {
+		return preferred;
+	}
+
+	public void setPreferred(final String preferred) {
+		this.preferred = preferred;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java
new file mode 100644
index 000000000..39816da6b
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java
@@ -0,0 +1,28 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class Label implements Serializable {
+
+	private String iso639;
+	private String label;
+	private final static long serialVersionUID = -6576156103297850809L;
+
+	public String getIso639() {
+		return iso639;
+	}
+
+	public void setIso639(final String iso639) {
+		this.iso639 = iso639;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(final String label) {
+		this.label = label;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java
new file mode 100644
index 000000000..1c07bbc2b
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java
@@ -0,0 +1,28 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class License implements Serializable {
+
+	private String attribution;
+	private String license;
+	private final static long serialVersionUID = -194308261058176439L;
+
+	public String getAttribution() {
+		return attribution;
+	}
+
+	public void setAttribution(final String attribution) {
+		this.attribution = attribution;
+	}
+
+	public String getLicense() {
+		return license;
+	}
+
+	public void setLicense(final String license) {
+		this.license = license;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java
new file mode 100644
index 000000000..b2d728f7d
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java
@@ -0,0 +1,28 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class NameAndCode implements Serializable {
+
+	private String name;
+	private String code;
+	private final static long serialVersionUID = 5459836979206140843L;
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getCode() {
+		return code;
+	}
+
+	public void setCode(final String code) {
+		this.code = code;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java
new file mode 100644
index 000000000..1d0e03f42
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java
@@ -0,0 +1,37 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+
+public class Relationship implements Serializable {
+
+	private String type;
+	private String id;
+	private String label;
+	private final static long serialVersionUID = 7847399503395576960L;
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(final String label) {
+		this.label = label;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
new file mode 100644
index 000000000..fae8b3ead
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
@@ -0,0 +1,156 @@
+
+package eu.dnetlib.dhp.actionmanager.ror.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class RorOrganization implements Serializable {
+
+	private List<String> ipAddresses = new ArrayList<>();
+	private List<String> aliases = new ArrayList<>();
+	private List<String> acronyms = new ArrayList<>();
+	private List<String> links = new ArrayList<>();
+	private Country country;
+	private String name;
+	private String wikipediaUrl;
+	private List<Address> addresses = new ArrayList<>();
+	private List<String> types = new ArrayList<>();
+	private Integer established;
+	private List<Relationship> relationships = new ArrayList<>();
+	private String emailAddress;
+	private ExternalIds externalIds;
+	private String id;
+	private List<Label> labels = new ArrayList<>();
+	private String status;
+	private final static long serialVersionUID = -2658312087616043225L;
+
+	public List<String> getIpAddresses() {
+		return ipAddresses;
+	}
+
+	public void setIpAddresses(final List<String> ipAddresses) {
+		this.ipAddresses = ipAddresses;
+	}
+
+	public List<String> getAliases() {
+		return aliases;
+	}
+
+	public void setAliases(final List<String> aliases) {
+		this.aliases = aliases;
+	}
+
+	public List<String> getAcronyms() {
+		return acronyms;
+	}
+
+	public void setAcronyms(final List<String> acronyms) {
+		this.acronyms = acronyms;
+	}
+
+	public List<String> getLinks() {
+		return links;
+	}
+
+	public void setLinks(final List<String> links) {
+		this.links = links;
+	}
+
+	public Country getCountry() {
+		return country;
+	}
+
+	public void setCountry(final Country country) {
+		this.country = country;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getWikipediaUrl() {
+		return wikipediaUrl;
+	}
+
+	public void setWikipediaUrl(final String wikipediaUrl) {
+		this.wikipediaUrl = wikipediaUrl;
+	}
+
+	public List<Address> getAddresses() {
+		return addresses;
+	}
+
+	public void setAddresses(final List<Address> addresses) {
+		this.addresses = addresses;
+	}
+
+	public List<String> getTypes() {
+		return types;
+	}
+
+	public void setTypes(final List<String> types) {
+		this.types = types;
+	}
+
+	public Integer getEstablished() {
+		return established;
+	}
+
+	public void setEstablished(final Integer established) {
+		this.established = established;
+	}
+
+	public List<Relationship> getRelationships() {
+		return relationships;
+	}
+
+	public void setRelationships(final List<Relationship> relationships) {
+		this.relationships = relationships;
+	}
+
+	public String getEmailAddress() {
+		return emailAddress;
+	}
+
+	public void setEmailAddress(final String emailAddress) {
+		this.emailAddress = emailAddress;
+	}
+
+	public ExternalIds getExternalIds() {
+		return externalIds;
+	}
+
+	public void setExternalIds(final ExternalIds externalIds) {
+		this.externalIds = externalIds;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public List<Label> getLabels() {
+		return labels;
+	}
+
+	public void setLabels(final List<Label> labels) {
+		this.labels = labels;
+	}
+
+	public String getStatus() {
+		return status;
+	}
+
+	public void setStatus(final String status) {
+		this.status = status;
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json
new file mode 100644
index 000000000..765e9c0af
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json
@@ -0,0 +1,14 @@
+[
+	{
+		"paramName": "i",
+		"paramLongName": "inputPath",
+		"paramDescription": "the path of the input json",
+		"paramRequired": true
+	},
+	{
+		"paramName": "o",
+		"paramLongName": "outputPath",
+		"paramDescription": "the path of the new ActionSet",
+		"paramRequired": true
+	}
+]
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/config-default.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/config-default.xml
new file mode 100644
index 000000000..a1755f329
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/config-default.xml
@@ -0,0 +1,58 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>oozie.action.sharelib.for.spark</name>
+        <value>spark2</value>
+    </property>
+    <property>
+        <name>hive_metastore_uris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>spark2YarnHistoryServerAddress</name>
+        <value>http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089</value>
+    </property>
+    <property>
+        <name>spark2ExtraListeners</name>
+        <value>com.cloudera.spark.lineage.NavigatorAppListener</value>
+    </property>
+    <property>
+        <name>spark2SqlQueryExecutionListeners</name>
+        <value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
+    </property>
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>sparkExecutorNumber</name>
+        <value>4</value>
+    </property>
+    <property>
+        <name>spark2EventLogDir</name>
+        <value>/user/spark/spark2ApplicationHistory</value>
+    </property>
+    <property>
+        <name>sparkDriverMemory</name>
+        <value>15G</value>
+    </property>
+    <property>
+        <name>sparkExecutorMemory</name>
+        <value>6G</value>
+    </property>
+    <property>
+        <name>sparkExecutorCores</name>
+        <value>1</value>
+    </property>
+</configuration>
\ No newline at end of file
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml
new file mode 100644
index 000000000..844f9ebb2
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml
@@ -0,0 +1,55 @@
+<workflow-app name="Update_ROR_action_set" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>inputPath</name>
+            <description>the path of the json</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>path where to store the action set</description>
+        </property>
+    </parameters>
+
+    <start to="deleteoutputpath"/>
+    
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+    
+    <action name="deleteoutputpath">
+        <fs>
+            <delete path='${outputPath}'/>
+            <mkdir path='${outputPath}'/>
+            <delete path='${workingDir}'/>
+            <mkdir path='${workingDir}'/>
+        </fs>
+        <ok to="processRorFile"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="processRorFile">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn</master>
+            <mode>cluster</mode>
+            <name>ProcessRorFile</name>
+            <class>eu.dnetlib.dhp.actionmanager.ror.GenerateRorActionSetJob</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-cores=${sparkExecutorCores}
+                --executor-memory=${sparkExecutorMemory}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+                --conf spark.sql.shuffle.partitions=3840
+            </spark-opts>
+            <arg>--inputPath</arg><arg>${inputPath}</arg>
+            <arg>--outputPath</arg><arg>${outputPath}</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <end name="End"/>
+</workflow-app>

From 7c5cd86927cd449b96bd131834658a094f5d52de Mon Sep 17 00:00:00 2001
From: "michele.artini" <michele.artini@isti.cnr.it>
Date: Thu, 29 Apr 2021 09:29:19 +0200
Subject: [PATCH 2/5] annotations and tests

---
 .../ror/GenerateRorActionSetJob.java          |   3 +-
 .../dhp/actionmanager/ror/model/Address.java  |  22 ++++
 .../dhp/actionmanager/ror/model/Country.java  |   6 +
 .../ror/model/ExternalIdType.java             |   6 +
 .../actionmanager/ror/model/ExternalIds.java  |  12 ++
 .../ror/model/GeonamesAdmin.java              |  10 ++
 .../actionmanager/ror/model/GeonamesCity.java |  18 +++
 .../dhp/actionmanager/ror/model/GridType.java |   5 +
 .../dhp/actionmanager/ror/model/Label.java    |   6 +
 .../dhp/actionmanager/ror/model/License.java  |   6 +
 .../actionmanager/ror/model/NameAndCode.java  |   6 +
 .../actionmanager/ror/model/Relationship.java |   8 ++
 .../ror/model/RorOrganization.java            |  34 +++++
 .../ror/GenerateRorActionSetJobTest.java      |  40 ++++++
 .../dhp/actionmanager/ror/ror_org.json        | 123 ++++++++++++++++++
 15 files changed, 304 insertions(+), 1 deletion(-)
 create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
 create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/ror/ror_org.json

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
index 81f0c55f8..78ce56f03 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
@@ -114,7 +114,7 @@ public class GenerateRorActionSetJob {
 
 	}
 
-	private static Organization convertRorOrg(final RorOrganization r) {
+	protected static Organization convertRorOrg(final RorOrganization r) {
 
 		final Date now = new Date();
 
@@ -178,6 +178,7 @@ public class GenerateRorActionSetJob {
 		final Set<String> names = new LinkedHashSet<>();
 		names.addAll(r.getAliases());
 		names.addAll(r.getAcronyms());
+		r.getLabels().forEach(l -> names.add(l.getLabel()));
 
 		return names.stream().filter(StringUtils::isNotBlank).map(s -> field(s, ROR_DATA_INFO)).collect(Collectors.toList());
 	}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java
index 68dc62e26..b566a5501 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Address.java
@@ -3,18 +3,40 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class Address implements Serializable {
 
+	@JsonProperty("lat")
 	private Float lat;
+
+	@JsonProperty("state_code")
 	private String stateCode;
+
+	@JsonProperty("country_geonames_id")
 	private Integer countryGeonamesId;
+
+	@JsonProperty("lng")
 	private Float lng;
+
+	@JsonProperty("state")
 	private String state;
+
+	@JsonProperty("city")
 	private String city;
+
+	@JsonProperty("geonames_city")
 	private GeonamesCity geonamesCity;
+
+	@JsonProperty("postcode")
 	private String postcode;
+
+	@JsonProperty("primary")
 	private Boolean primary;
+
+	@JsonProperty("line")
 	private String line;
+
 	private final static long serialVersionUID = 2444635485253443195L;
 
 	public Float getLat() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java
index 08ca939f2..3dab60a9f 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Country.java
@@ -3,10 +3,16 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class Country implements Serializable {
 
+	@JsonProperty("country_code")
 	private String countryCode;
+
+	@JsonProperty("country_name")
 	private String countryName;
+
 	private final static long serialVersionUID = 4357848706229493627L;
 
 	public String getCountryCode() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
index 42406f097..f36c25ef0 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
@@ -5,10 +5,16 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class ExternalIdType implements Serializable {
 
+	@JsonProperty("all")
 	private List<String> all = new ArrayList<>();
+
+	@JsonProperty("preferred")
 	private String preferred;
+
 	private final static long serialVersionUID = 2616688352998387611L;
 
 	public List<String> getAll() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
index 9ea1ccdb8..bbf92f6ab 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
@@ -3,13 +3,25 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class ExternalIds implements Serializable {
 
+	@JsonProperty("Wikidata")
 	private ExternalIdType wikidata;
+
+	@JsonProperty("OrgRef")
 	private ExternalIdType orgRef;
+
+	@JsonProperty("ISNI")
 	private ExternalIdType isni;
+
+	@JsonProperty("FundRef")
 	private ExternalIdType fundRef;
+
+	@JsonProperty("GRID")
 	private GridType grid;
+
 	private final static long serialVersionUID = 686536347353680869L;
 
 	public ExternalIdType getWikidata() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java
index a446c092f..9616db447 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesAdmin.java
@@ -3,12 +3,22 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class GeonamesAdmin implements Serializable {
 
+	@JsonProperty("ascii_name")
 	private String asciiName;
+
+	@JsonProperty("id")
 	private Integer id;
+
+	@JsonProperty("name")
 	private String name;
+
+	@JsonProperty("code")
 	private String code;
+
 	private final static long serialVersionUID = 7294958526269195673L;
 
 	public String getAsciiName() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java
index 9c98bebd1..3fd539066 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GeonamesCity.java
@@ -3,16 +3,34 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class GeonamesCity implements Serializable {
 
+	@JsonProperty("geonames_admin1")
 	private GeonamesAdmin geonamesAdmin1;
+
+	@JsonProperty("geonames_admin2")
 	private GeonamesAdmin geonamesAdmin2;
+
+	@JsonProperty("city")
 	private String city;
+
+	@JsonProperty("id")
 	private Integer id;
+
+	@JsonProperty("nuts_level1")
 	private NameAndCode nutsLevel1;
+
+	@JsonProperty("nuts_level2")
 	private NameAndCode nutsLevel2;
+
+	@JsonProperty("nuts_level3")
 	private NameAndCode nutsLevel3;
+
+	@JsonProperty("")
 	private License license;
+
 	private final static long serialVersionUID = -8389480201526252955L;
 
 	public NameAndCode getNutsLevel2() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
index 739ea22e0..4a7f7ca77 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
@@ -3,6 +3,8 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class GridType implements Serializable {
 
 	/**
@@ -10,7 +12,10 @@ public class GridType implements Serializable {
 	 */
 	private static final long serialVersionUID = -5605887658267581353L;
 
+	@JsonProperty("all")
 	private String all;
+
+	@JsonProperty("preferred")
 	private String preferred;
 
 	public String getAll() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java
index 39816da6b..61eb0339d 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Label.java
@@ -3,10 +3,16 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class Label implements Serializable {
 
+	@JsonProperty("iso639")
 	private String iso639;
+
+	@JsonProperty("label")
 	private String label;
+
 	private final static long serialVersionUID = -6576156103297850809L;
 
 	public String getIso639() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java
index 1c07bbc2b..bdc8f4c42 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/License.java
@@ -3,10 +3,16 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class License implements Serializable {
 
+	@JsonProperty("attribution")
 	private String attribution;
+
+	@JsonProperty("license")
 	private String license;
+
 	private final static long serialVersionUID = -194308261058176439L;
 
 	public String getAttribution() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java
index b2d728f7d..61d7eb8e6 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/NameAndCode.java
@@ -3,10 +3,16 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class NameAndCode implements Serializable {
 
+	@JsonProperty("name")
 	private String name;
+
+	@JsonProperty("code")
 	private String code;
+
 	private final static long serialVersionUID = 5459836979206140843L;
 
 	public String getName() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java
index 1d0e03f42..8b73db98f 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/Relationship.java
@@ -3,11 +3,19 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class Relationship implements Serializable {
 
+	@JsonProperty("type")
 	private String type;
+
+	@JsonProperty("id")
 	private String id;
+
+	@JsonProperty("label")
 	private String label;
+
 	private final static long serialVersionUID = 7847399503395576960L;
 
 	public String getType() {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
index fae8b3ead..7712ef4f2 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
@@ -5,24 +5,58 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class RorOrganization implements Serializable {
 
+	@JsonProperty("ip_addresses")
 	private List<String> ipAddresses = new ArrayList<>();
+
+	@JsonProperty("aliases")
 	private List<String> aliases = new ArrayList<>();
+
+	@JsonProperty("acronyms")
 	private List<String> acronyms = new ArrayList<>();
+
+	@JsonProperty("links")
 	private List<String> links = new ArrayList<>();
+
+	@JsonProperty("country")
 	private Country country;
+
+	@JsonProperty("name")
 	private String name;
+
+	@JsonProperty("wikipedia_url")
 	private String wikipediaUrl;
+
+	@JsonProperty("addresses")
 	private List<Address> addresses = new ArrayList<>();
+
+	@JsonProperty("types")
 	private List<String> types = new ArrayList<>();
+
+	@JsonProperty("established")
 	private Integer established;
+
+	@JsonProperty("relationships")
 	private List<Relationship> relationships = new ArrayList<>();
+
+	@JsonProperty("email_address")
 	private String emailAddress;
+
+	@JsonProperty("external_ids")
 	private ExternalIds externalIds;
+
+	@JsonProperty("id")
 	private String id;
+
+	@JsonProperty("labels")
 	private List<Label> labels = new ArrayList<>();
+
+	@JsonProperty("status")
 	private String status;
+
 	private final static long serialVersionUID = -2658312087616043225L;
 
 	public List<String> getIpAddresses() {
diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
new file mode 100644
index 000000000..7948e8e5d
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
@@ -0,0 +1,40 @@
+package eu.dnetlib.dhp.actionmanager.ror;
+
+import java.io.FileInputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.actionmanager.ror.model.RorOrganization;
+import eu.dnetlib.dhp.schema.oaf.Organization;
+
+class GenerateRorActionSetJobTest {
+
+	private static final ObjectMapper mapper = new ObjectMapper();
+
+	private static final String local_file_path = "/Users/michele/Downloads/ror-data-2021-04-06.json";
+
+	@BeforeEach
+	void setUp() throws Exception {}
+
+	@Test
+	void testConvertRorOrg() throws Exception {
+		final RorOrganization r = mapper.readValue(IOUtils.toString(getClass().getResourceAsStream("ror_org.json")), RorOrganization.class);
+		final Organization org = GenerateRorActionSetJob.convertRorOrg(r);
+
+		System.out.println(mapper.writeValueAsString(org));
+	}
+
+	@Test
+	void testConvertAllRorOrg() throws Exception {
+		final RorOrganization[] arr = mapper.readValue(IOUtils.toString(new FileInputStream(local_file_path)), RorOrganization[].class);
+
+		for (final RorOrganization r : arr) {
+			GenerateRorActionSetJob.convertRorOrg(r);
+		}
+	}
+
+}
diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/ror/ror_org.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/ror/ror_org.json
new file mode 100644
index 000000000..d2b4fa64b
--- /dev/null
+++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/ror/ror_org.json
@@ -0,0 +1,123 @@
+{
+  "ip_addresses": [],
+  "aliases": [],
+  "acronyms": [
+    "ANU"
+  ],
+  "links": [
+    "http://www.anu.edu.au/"
+  ],
+  "country": {
+    "country_code": "AU",
+    "country_name": "Australia"
+  },
+  "name": "Australian National University",
+  "wikipedia_url": "http://en.wikipedia.org/wiki/Australian_National_University",
+  "addresses": [
+    {
+      "lat": -35.2778,
+      "state_code": "AU-ACT",
+      "country_geonames_id": 2077456,
+      "lng": 149.1205,
+      "state": "Australian Capital Territory",
+      "city": "Canberra",
+      "geonames_city": {
+        "nuts_level2": {
+          "name": null,
+          "code": null
+        },
+        "geonames_admin2": {
+          "ascii_name": null,
+          "id": null,
+          "name": null,
+          "code": null
+        },
+        "geonames_admin1": {
+          "ascii_name": "ACT",
+          "id": 2177478,
+          "name": "ACT",
+          "code": "AU.01"
+        },
+        "city": "Canberra",
+        "id": 2172517,
+        "nuts_level1": {
+          "name": null,
+          "code": null
+        },
+        "nuts_level3": {
+          "name": null,
+          "code": null
+        },
+        "license": {
+          "attribution": "Data from geonames.org under a CC-BY 3.0 license",
+          "license": "http://creativecommons.org/licenses/by/3.0/"
+        }
+      },
+      "postcode": null,
+      "primary": false,
+      "line": null
+    }
+  ],
+  "types": [
+    "Education"
+  ],
+  "established": 1946,
+  "relationships": [
+    {
+      "type": "Related",
+      "id": "https://ror.org/041c7s516",
+      "label": "Calvary Hospital"
+    },
+    {
+      "type": "Related",
+      "id": "https://ror.org/04h7nbn38",
+      "label": "Canberra Hospital"
+    },
+    {
+      "type": "Related",
+      "id": "https://ror.org/030jpqj15",
+      "label": "Goulburn Base Hospital"
+    },
+    {
+      "type": "Child",
+      "id": "https://ror.org/006a4jj40",
+      "label": "Mount Stromlo Observatory"
+    }
+  ],
+  "email_address": null,
+  "external_ids": {
+    "Wikidata": {
+      "all": [
+        "Q127990"
+      ],
+      "preferred": null
+    },
+    "OrgRef": {
+      "all": [
+        "285106"
+      ],
+      "preferred": null
+    },
+    "ISNI": {
+      "all": [
+        "0000 0001 2180 7477"
+      ],
+      "preferred": null
+    },
+    "FundRef": {
+      "all": [
+        "501100000995",
+        "501100001151",
+        "100009020"
+      ],
+      "preferred": "501100000995"
+    },
+    "GRID": {
+      "all": "grid.1001.0",
+      "preferred": "grid.1001.0"
+    }
+  },
+  "id": "https://ror.org/019wvm592",
+  "labels": [],
+  "status": "active"
+}
\ No newline at end of file

From f77ba341265fadd75689aed49a4f3b5a3592ed0a Mon Sep 17 00:00:00 2001
From: "michele.artini" <michele.artini@isti.cnr.it>
Date: Thu, 29 Apr 2021 09:50:05 +0200
Subject: [PATCH 3/5] pid types

---
 .../ror/GenerateRorActionSetJob.java          | 43 ++++++------
 .../ror/model/ExternalIdType.java             |  8 +--
 .../actionmanager/ror/model/ExternalIds.java  | 67 -------------------
 .../dhp/actionmanager/ror/model/GridType.java | 37 ----------
 .../ror/model/RorOrganization.java            |  8 ++-
 .../ror/GenerateRorActionSetJobTest.java      |  2 +
 6 files changed, 34 insertions(+), 131 deletions(-)
 delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
 delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
index 78ce56f03..d306afef0 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
@@ -11,9 +11,11 @@ import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.structuredProperty;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Date;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -33,6 +35,7 @@ import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import eu.dnetlib.dhp.actionmanager.project.SparkAtomicActionJob;
+import eu.dnetlib.dhp.actionmanager.ror.model.ExternalIdType;
 import eu.dnetlib.dhp.actionmanager.ror.model.RorOrganization;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
@@ -59,11 +62,6 @@ public class GenerateRorActionSetJob {
 	private static final DataInfo ROR_DATA_INFO = dataInfo(false, "", false, false, ENTITYREGISTRY_PROVENANCE_ACTION, "0.92");
 
 	private static final Qualifier ROR_PID_TYPE = qualifier("ROR", "ROR", "dnet:pid_types", "dnet:pid_types");
-	private static final Qualifier GRID_PID_TYPE = qualifier("GRID", "GRID", "dnet:pid_types", "dnet:pid_types");
-	private static final Qualifier WIKIDATA_PID_TYPE = qualifier("Wikidata", "Wikidata", "dnet:pid_types", "dnet:pid_types");
-	private static final Qualifier ORGREF_PID_TYPE = qualifier("OrgRef", "OrgRef", "dnet:pid_types", "dnet:pid_types");
-	private static final Qualifier ISNI_PID_TYPE = qualifier("ISNI", "ISNI", "dnet:pid_types", "dnet:pid_types");
-	private static final Qualifier FUNDREF_PID_TYPE = qualifier("FundRef", "FundRef", "dnet:pid_types", "dnet:pid_types");
 
 	public static void main(final String[] args) throws Exception {
 
@@ -157,21 +155,28 @@ public class GenerateRorActionSetJob {
 	private static List<StructuredProperty> pids(final RorOrganization r) {
 		final List<StructuredProperty> pids = new ArrayList<>();
 		pids.add(structuredProperty(r.getId(), ROR_PID_TYPE, ROR_DATA_INFO));
-		pids.add(structuredProperty(r.getExternalIds().getGrid().getAll(), GRID_PID_TYPE, ROR_DATA_INFO));
-		pids.addAll(pids(r.getExternalIds().getFundRef().getAll(), FUNDREF_PID_TYPE));
-		pids.addAll(pids(r.getExternalIds().getIsni().getAll(), ISNI_PID_TYPE));
-		pids.addAll(pids(r.getExternalIds().getOrgRef().getAll(), ORGREF_PID_TYPE));
-		pids.addAll(pids(r.getExternalIds().getWikidata().getAll(), WIKIDATA_PID_TYPE));
-		return pids;
-	}
 
-	private static List<StructuredProperty> pids(final List<String> list, final Qualifier pidType) {
-		if (list == null) { return new ArrayList<>(); }
-		return list.stream()
-			.filter(StringUtils::isNotBlank)
-			.distinct()
-			.map(s -> structuredProperty(s, pidType, ROR_DATA_INFO))
-			.collect(Collectors.toList());
+		for (final Map.Entry<String, ExternalIdType> e : r.getExternalIds().entrySet()) {
+			final String type = e.getKey();
+			final Object all = e.getValue().getAll();
+			if (all == null) {
+				// skip
+			} else if (all instanceof String) {
+				pids.add(structuredProperty(all.toString(), qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
+			} else if (all instanceof Collection) {
+				for (final Object pid : (Collection<?>) all) {
+					pids.add(structuredProperty(pid.toString(), qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
+				}
+			} else if (all instanceof String[]) {
+				for (final String pid : (String[]) all) {
+					pids.add(structuredProperty(pid, qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
+				}
+			} else {
+				log.warn("Invalid type for pid list: " + all.getClass());
+			}
+		}
+
+		return pids;
 	}
 
 	private static List<Field<String>> alternativeNames(final RorOrganization r) {
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
index f36c25ef0..68c2d429b 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIdType.java
@@ -2,26 +2,24 @@
 package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 public class ExternalIdType implements Serializable {
 
 	@JsonProperty("all")
-	private List<String> all = new ArrayList<>();
+	private Object all;
 
 	@JsonProperty("preferred")
 	private String preferred;
 
 	private final static long serialVersionUID = 2616688352998387611L;
 
-	public List<String> getAll() {
+	public Object getAll() {
 		return all;
 	}
 
-	public void setAll(final List<String> all) {
+	public void setAll(final Object all) {
 		this.all = all;
 	}
 
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
deleted file mode 100644
index bbf92f6ab..000000000
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/ExternalIds.java
+++ /dev/null
@@ -1,67 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.ror.model;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class ExternalIds implements Serializable {
-
-	@JsonProperty("Wikidata")
-	private ExternalIdType wikidata;
-
-	@JsonProperty("OrgRef")
-	private ExternalIdType orgRef;
-
-	@JsonProperty("ISNI")
-	private ExternalIdType isni;
-
-	@JsonProperty("FundRef")
-	private ExternalIdType fundRef;
-
-	@JsonProperty("GRID")
-	private GridType grid;
-
-	private final static long serialVersionUID = 686536347353680869L;
-
-	public ExternalIdType getWikidata() {
-		return wikidata;
-	}
-
-	public void setWikidata(final ExternalIdType wikidata) {
-		this.wikidata = wikidata;
-	}
-
-	public ExternalIdType getOrgRef() {
-		return orgRef;
-	}
-
-	public void setOrgRef(final ExternalIdType orgRef) {
-		this.orgRef = orgRef;
-	}
-
-	public ExternalIdType getIsni() {
-		return isni;
-	}
-
-	public void setIsni(final ExternalIdType isni) {
-		this.isni = isni;
-	}
-
-	public ExternalIdType getFundRef() {
-		return fundRef;
-	}
-
-	public void setFundRef(final ExternalIdType fundRef) {
-		this.fundRef = fundRef;
-	}
-
-	public GridType getGrid() {
-		return grid;
-	}
-
-	public void setGrid(final GridType grid) {
-		this.grid = grid;
-	}
-
-}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
deleted file mode 100644
index 4a7f7ca77..000000000
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/GridType.java
+++ /dev/null
@@ -1,37 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.ror.model;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class GridType implements Serializable {
-
-	/**
-	 *
-	 */
-	private static final long serialVersionUID = -5605887658267581353L;
-
-	@JsonProperty("all")
-	private String all;
-
-	@JsonProperty("preferred")
-	private String preferred;
-
-	public String getAll() {
-		return all;
-	}
-
-	public void setAll(final String all) {
-		this.all = all;
-	}
-
-	public String getPreferred() {
-		return preferred;
-	}
-
-	public void setPreferred(final String preferred) {
-		this.preferred = preferred;
-	}
-
-}
diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
index 7712ef4f2..94de34fee 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/model/RorOrganization.java
@@ -3,7 +3,9 @@ package eu.dnetlib.dhp.actionmanager.ror.model;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 
@@ -46,7 +48,7 @@ public class RorOrganization implements Serializable {
 	private String emailAddress;
 
 	@JsonProperty("external_ids")
-	private ExternalIds externalIds;
+	private Map<String, ExternalIdType> externalIds = new LinkedHashMap<>();
 
 	@JsonProperty("id")
 	private String id;
@@ -155,11 +157,11 @@ public class RorOrganization implements Serializable {
 		this.emailAddress = emailAddress;
 	}
 
-	public ExternalIds getExternalIds() {
+	public Map<String, ExternalIdType> getExternalIds() {
 		return externalIds;
 	}
 
-	public void setExternalIds(final ExternalIds externalIds) {
+	public void setExternalIds(final Map<String, ExternalIdType> externalIds) {
 		this.externalIds = externalIds;
 	}
 
diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
index 7948e8e5d..af2bd1c9d 100644
--- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
+++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
@@ -4,6 +4,7 @@ import java.io.FileInputStream;
 
 import org.apache.commons.io.IOUtils;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -11,6 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.dnetlib.dhp.actionmanager.ror.model.RorOrganization;
 import eu.dnetlib.dhp.schema.oaf.Organization;
 
+@Disabled
 class GenerateRorActionSetJobTest {
 
 	private static final ObjectMapper mapper = new ObjectMapper();

From a278d67175a76b620129a441325b182c4247ed29 Mon Sep 17 00:00:00 2001
From: "michele.artini" <michele.artini@isti.cnr.it>
Date: Thu, 29 Apr 2021 11:34:47 +0200
Subject: [PATCH 4/5] parse input file

---
 .../ror/GenerateRorActionSetJob.java          |  64 +++--
 .../actionmanager/ror/oozie_app/workflow.xml  |  12 +-
 .../ror/GenerateRorActionSetJobTest.java      |  10 +-
 .../dnetlib/dhp/oa/dedup/SparkDedupTest.java  | 250 +++++++++---------
 4 files changed, 177 insertions(+), 159 deletions(-)

diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
index d306afef0..c8876a415 100644
--- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
+++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJob.java
@@ -9,6 +9,7 @@ import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.listKeyValues;
 import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.qualifier;
 import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.structuredProperty;
 
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -22,10 +23,12 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SparkSession;
@@ -57,17 +60,20 @@ public class GenerateRorActionSetJob {
 
 	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
-	private static final List<KeyValue> ROR_COLLECTED_FROM = listKeyValues("10|openaire____::993a7ae7a863813cf95028b50708e222", "ROR");
+	private static final List<KeyValue> ROR_COLLECTED_FROM = listKeyValues(
+		"10|openaire____::993a7ae7a863813cf95028b50708e222", "ROR");
 
-	private static final DataInfo ROR_DATA_INFO = dataInfo(false, "", false, false, ENTITYREGISTRY_PROVENANCE_ACTION, "0.92");
+	private static final DataInfo ROR_DATA_INFO = dataInfo(
+		false, "", false, false, ENTITYREGISTRY_PROVENANCE_ACTION, "0.92");
 
 	private static final Qualifier ROR_PID_TYPE = qualifier("ROR", "ROR", "dnet:pid_types", "dnet:pid_types");
 
 	public static void main(final String[] args) throws Exception {
 
 		final String jsonConfiguration = IOUtils
-			.toString(SparkAtomicActionJob.class
-				.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json"));
+			.toString(
+				SparkAtomicActionJob.class
+					.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/ror/action_set_parameters.json"));
 
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
 
@@ -100,16 +106,16 @@ public class GenerateRorActionSetJob {
 
 	private static void processRorOrganizations(final SparkSession spark,
 		final String inputPath,
-		final String outputPath) {
+		final String outputPath) throws Exception {
 
 		readInputPath(spark, inputPath)
 			.map(GenerateRorActionSetJob::convertRorOrg, Encoders.bean(Organization.class))
 			.toJavaRDD()
 			.map(o -> new AtomicAction<>(Organization.class, o))
-			.mapToPair(aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
-				new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+			.mapToPair(
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
 			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
-
 	}
 
 	protected static Organization convertRorOrg(final RorOrganization r) {
@@ -142,7 +148,11 @@ public class GenerateRorActionSetJob {
 		o.setEcsmevalidated(null);
 		o.setEcnutscode(null);
 		if (r.getCountry() != null) {
-			o.setCountry(qualifier(r.getCountry().getCountryCode(), r.getCountry().getCountryName(), COUNTRIES_VOC, COUNTRIES_VOC));
+			o
+				.setCountry(
+					qualifier(
+						r.getCountry().getCountryCode(), r.getCountry().getCountryName(), COUNTRIES_VOC,
+						COUNTRIES_VOC));
 		} else {
 			o.setCountry(null);
 		}
@@ -162,14 +172,24 @@ public class GenerateRorActionSetJob {
 			if (all == null) {
 				// skip
 			} else if (all instanceof String) {
-				pids.add(structuredProperty(all.toString(), qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
+				pids
+					.add(
+						structuredProperty(
+							all.toString(), qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
 			} else if (all instanceof Collection) {
 				for (final Object pid : (Collection<?>) all) {
-					pids.add(structuredProperty(pid.toString(), qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
+					pids
+						.add(
+							structuredProperty(
+								pid.toString(), qualifier(type, type, "dnet:pid_types", "dnet:pid_types"),
+								ROR_DATA_INFO));
 				}
 			} else if (all instanceof String[]) {
 				for (final String pid : (String[]) all) {
-					pids.add(structuredProperty(pid, qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
+					pids
+						.add(
+							structuredProperty(
+								pid, qualifier(type, type, "dnet:pid_types", "dnet:pid_types"), ROR_DATA_INFO));
 				}
 			} else {
 				log.warn("Invalid type for pid list: " + all.getClass());
@@ -185,16 +205,22 @@ public class GenerateRorActionSetJob {
 		names.addAll(r.getAcronyms());
 		r.getLabels().forEach(l -> names.add(l.getLabel()));
 
-		return names.stream().filter(StringUtils::isNotBlank).map(s -> field(s, ROR_DATA_INFO)).collect(Collectors.toList());
+		return names
+			.stream()
+			.filter(StringUtils::isNotBlank)
+			.map(s -> field(s, ROR_DATA_INFO))
+			.collect(Collectors.toList());
 	}
 
 	private static Dataset<RorOrganization> readInputPath(
 		final SparkSession spark,
-		final String inputPath) {
-		return spark
-			.read()
-			.textFile(inputPath)
-			.map((MapFunction<String, RorOrganization>) value -> OBJECT_MAPPER.readValue(value, RorOrganization.class), Encoders.bean(RorOrganization.class));
+		final String path) throws Exception {
+
+		try (final FileSystem fileSystem = FileSystem.get(new Configuration());
+			final InputStream is = fileSystem.open(new Path(path))) {
+			final RorOrganization[] arr = OBJECT_MAPPER.readValue(is, RorOrganization[].class);
+			return spark.createDataset(Arrays.asList(arr), Encoders.bean(RorOrganization.class));
+		}
 	}
 
 }
diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml
index 844f9ebb2..3d00b80a8 100644
--- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/ror/oozie_app/workflow.xml
@@ -1,11 +1,11 @@
 <workflow-app name="Update_ROR_action_set" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
-            <name>inputPath</name>
+            <name>rorJsonInputPath</name>
             <description>the path of the json</description>
         </property>
         <property>
-            <name>outputPath</name>
+            <name>rorActionSetPath</name>
             <description>path where to store the action set</description>
         </property>
     </parameters>
@@ -18,8 +18,8 @@
     
     <action name="deleteoutputpath">
         <fs>
-            <delete path='${outputPath}'/>
-            <mkdir path='${outputPath}'/>
+            <delete path='${rorActionSetPath}'/>
+            <mkdir path='${rorActionSetPath}'/>
             <delete path='${workingDir}'/>
             <mkdir path='${workingDir}'/>
         </fs>
@@ -44,8 +44,8 @@
                 --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
                 --conf spark.sql.shuffle.partitions=3840
             </spark-opts>
-            <arg>--inputPath</arg><arg>${inputPath}</arg>
-            <arg>--outputPath</arg><arg>${outputPath}</arg>
+            <arg>--inputPath</arg><arg>${rorJsonInputPath}</arg>
+            <arg>--outputPath</arg><arg>${rorActionSetPath}</arg>
         </spark>
         <ok to="End"/>
         <error to="Kill"/>
diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
index af2bd1c9d..f16901cb4 100644
--- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
+++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/ror/GenerateRorActionSetJobTest.java
@@ -1,3 +1,4 @@
+
 package eu.dnetlib.dhp.actionmanager.ror;
 
 import java.io.FileInputStream;
@@ -20,11 +21,13 @@ class GenerateRorActionSetJobTest {
 	private static final String local_file_path = "/Users/michele/Downloads/ror-data-2021-04-06.json";
 
 	@BeforeEach
-	void setUp() throws Exception {}
+	void setUp() throws Exception {
+	}
 
 	@Test
 	void testConvertRorOrg() throws Exception {
-		final RorOrganization r = mapper.readValue(IOUtils.toString(getClass().getResourceAsStream("ror_org.json")), RorOrganization.class);
+		final RorOrganization r = mapper
+			.readValue(IOUtils.toString(getClass().getResourceAsStream("ror_org.json")), RorOrganization.class);
 		final Organization org = GenerateRorActionSetJob.convertRorOrg(r);
 
 		System.out.println(mapper.writeValueAsString(org));
@@ -32,7 +35,8 @@ class GenerateRorActionSetJobTest {
 
 	@Test
 	void testConvertAllRorOrg() throws Exception {
-		final RorOrganization[] arr = mapper.readValue(IOUtils.toString(new FileInputStream(local_file_path)), RorOrganization[].class);
+		final RorOrganization[] arr = mapper
+			.readValue(IOUtils.toString(new FileInputStream(local_file_path)), RorOrganization[].class);
 
 		for (final RorOrganization r : arr) {
 			GenerateRorActionSetJob.convertRorOrg(r);
diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
index bf4913056..a95e28c43 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
+++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
@@ -12,7 +12,6 @@ import java.io.IOException;
 import java.io.Serializable;
 import java.net.URISyntaxException;
 import java.nio.file.Paths;
-import java.util.List;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
@@ -27,7 +26,13 @@ import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SparkSession;
-import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.Mockito;
@@ -97,8 +102,7 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream(
-								"/eu/dnetlib/dhp/dedup/profiles/mock_orchestrator.xml")));
+							.getResourceAsStream("/eu/dnetlib/dhp/dedup/profiles/mock_orchestrator.xml")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("organization")))
@@ -106,8 +110,7 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream(
-								"/eu/dnetlib/dhp/dedup/conf/org.curr.conf.json")));
+							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/org.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("publication")))
@@ -115,8 +118,7 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream(
-								"/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json")));
+							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("software")))
@@ -124,8 +126,7 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream(
-								"/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json")));
+							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("dataset")))
@@ -133,8 +134,7 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream(
-								"/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json")));
+							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("otherresearchproduct")))
@@ -142,54 +142,51 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream(
-								"/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json")));
+							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json")));
 	}
 
 	@Test
 	@Order(1)
 	public void createSimRelsTest() throws Exception {
 
-		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateSimRels.class
-						.getResourceAsStream(
-							"/eu/dnetlib/dhp/oa/dedup/createSimRels_parameters.json")));
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createSimRels_parameters.json")));
 
 		parser
-			.parseArgument(
-				new String[] {
-					"-i", testGraphBasePath,
-					"-asi", testActionSetId,
-					"-la", "lookupurl",
-					"-w", testOutputBasePath,
-					"-np", "50"
-				});
+			.parseArgument(new String[] {
+				"-i", testGraphBasePath,
+				"-asi", testActionSetId,
+				"-la", "lookupurl",
+				"-w", testOutputBasePath,
+				"-np", "50"
+			});
 
 		new SparkCreateSimRels(parser, spark).run(isLookUpService);
 
-		long orgs_simrel = spark
+		final long orgs_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization"))
 			.count();
 
-		long pubs_simrel = spark
+		final long pubs_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "publication"))
 			.count();
 
-		long sw_simrel = spark
+		final long sw_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"))
 			.count();
 
-		long ds_simrel = spark
+		final long ds_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "dataset"))
 			.count();
 
-		long orp_simrel = spark
+		final long orp_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
 			.count();
@@ -205,31 +202,29 @@ public class SparkDedupTest implements Serializable {
 	@Order(2)
 	public void cutMergeRelsTest() throws Exception {
 
-		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateMergeRels.class
-						.getResourceAsStream(
-							"/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
 
 		parser
-			.parseArgument(
-				new String[] {
-					"-i",
-					testGraphBasePath,
-					"-asi",
-					testActionSetId,
-					"-la",
-					"lookupurl",
-					"-w",
-					testOutputBasePath,
-					"-cc",
-					"3"
-				});
+			.parseArgument(new String[] {
+				"-i",
+				testGraphBasePath,
+				"-asi",
+				testActionSetId,
+				"-la",
+				"lookupurl",
+				"-w",
+				testOutputBasePath,
+				"-cc",
+				"3"
+			});
 
 		new SparkCreateMergeRels(parser, spark).run(isLookUpService);
 
-		long orgs_mergerel = spark
+		final long orgs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -240,7 +235,7 @@ public class SparkDedupTest implements Serializable {
 			.where("cnt > 3")
 			.count();
 
-		long pubs_mergerel = spark
+		final long pubs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -250,7 +245,7 @@ public class SparkDedupTest implements Serializable {
 			.select("source", "cnt")
 			.where("cnt > 3")
 			.count();
-		long sw_mergerel = spark
+		final long sw_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -261,7 +256,7 @@ public class SparkDedupTest implements Serializable {
 			.where("cnt > 3")
 			.count();
 
-		long ds_mergerel = spark
+		final long ds_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -272,7 +267,7 @@ public class SparkDedupTest implements Serializable {
 			.where("cnt > 3")
 			.count();
 
-		long orp_mergerel = spark
+		final long orp_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -301,46 +296,44 @@ public class SparkDedupTest implements Serializable {
 	@Order(3)
 	public void createMergeRelsTest() throws Exception {
 
-		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateMergeRels.class
-						.getResourceAsStream(
-							"/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
 
 		parser
-			.parseArgument(
-				new String[] {
-					"-i",
-					testGraphBasePath,
-					"-asi",
-					testActionSetId,
-					"-la",
-					"lookupurl",
-					"-w",
-					testOutputBasePath
-				});
+			.parseArgument(new String[] {
+				"-i",
+				testGraphBasePath,
+				"-asi",
+				testActionSetId,
+				"-la",
+				"lookupurl",
+				"-w",
+				testOutputBasePath
+			});
 
 		new SparkCreateMergeRels(parser, spark).run(isLookUpService);
 
-		long orgs_mergerel = spark
+		final long orgs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
 			.count();
-		long pubs_mergerel = spark
+		final long pubs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
 			.count();
-		long sw_mergerel = spark
+		final long sw_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
 			.count();
-		long ds_mergerel = spark
+		final long ds_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
 			.count();
 
-		long orp_mergerel = spark
+		final long orp_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
 			.count();
@@ -357,40 +350,39 @@ public class SparkDedupTest implements Serializable {
 	@Order(4)
 	public void createDedupRecordTest() throws Exception {
 
-		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateDedupRecord.class
-						.getResourceAsStream(
-							"/eu/dnetlib/dhp/oa/dedup/createDedupRecord_parameters.json")));
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createDedupRecord_parameters.json")));
 		parser
-			.parseArgument(
-				new String[] {
-					"-i",
-					testGraphBasePath,
-					"-asi",
-					testActionSetId,
-					"-la",
-					"lookupurl",
-					"-w",
-					testOutputBasePath
-				});
+			.parseArgument(new String[] {
+				"-i",
+				testGraphBasePath,
+				"-asi",
+				testActionSetId,
+				"-la",
+				"lookupurl",
+				"-w",
+				testOutputBasePath
+			});
 
 		new SparkCreateDedupRecord(parser, spark).run(isLookUpService);
 
-		long orgs_deduprecord = jsc
+		final long orgs_deduprecord = jsc
 			.textFile(testOutputBasePath + "/" + testActionSetId + "/organization_deduprecord")
 			.count();
-		long pubs_deduprecord = jsc
+		final long pubs_deduprecord = jsc
 			.textFile(testOutputBasePath + "/" + testActionSetId + "/publication_deduprecord")
 			.count();
-		long sw_deduprecord = jsc
+		final long sw_deduprecord = jsc
 			.textFile(testOutputBasePath + "/" + testActionSetId + "/software_deduprecord")
 			.count();
-		long ds_deduprecord = jsc.textFile(testOutputBasePath + "/" + testActionSetId + "/dataset_deduprecord").count();
-		long orp_deduprecord = jsc
-			.textFile(
-				testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_deduprecord")
+		final long ds_deduprecord = jsc
+			.textFile(testOutputBasePath + "/" + testActionSetId + "/dataset_deduprecord")
+			.count();
+		final long orp_deduprecord = jsc
+			.textFile(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_deduprecord")
 			.count();
 
 		assertEquals(85, orgs_deduprecord);
@@ -404,29 +396,27 @@ public class SparkDedupTest implements Serializable {
 	@Order(5)
 	public void updateEntityTest() throws Exception {
 
-		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkUpdateEntity.class
-						.getResourceAsStream(
-							"/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json")));
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json")));
 		parser
-			.parseArgument(
-				new String[] {
-					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
-				});
+			.parseArgument(new String[] {
+				"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
+			});
 
 		new SparkUpdateEntity(parser, spark).run(isLookUpService);
 
-		long organizations = jsc.textFile(testDedupGraphBasePath + "/organization").count();
-		long publications = jsc.textFile(testDedupGraphBasePath + "/publication").count();
-		long projects = jsc.textFile(testDedupGraphBasePath + "/project").count();
-		long datasource = jsc.textFile(testDedupGraphBasePath + "/datasource").count();
-		long softwares = jsc.textFile(testDedupGraphBasePath + "/software").count();
-		long dataset = jsc.textFile(testDedupGraphBasePath + "/dataset").count();
-		long otherresearchproduct = jsc.textFile(testDedupGraphBasePath + "/otherresearchproduct").count();
+		final long organizations = jsc.textFile(testDedupGraphBasePath + "/organization").count();
+		final long publications = jsc.textFile(testDedupGraphBasePath + "/publication").count();
+		final long projects = jsc.textFile(testDedupGraphBasePath + "/project").count();
+		final long datasource = jsc.textFile(testDedupGraphBasePath + "/datasource").count();
+		final long softwares = jsc.textFile(testDedupGraphBasePath + "/software").count();
+		final long dataset = jsc.textFile(testDedupGraphBasePath + "/dataset").count();
+		final long otherresearchproduct = jsc.textFile(testDedupGraphBasePath + "/otherresearchproduct").count();
 
-		long mergedOrgs = spark
+		final long mergedOrgs = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -436,7 +426,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		long mergedPubs = spark
+		final long mergedPubs = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -446,7 +436,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		long mergedSw = spark
+		final long mergedSw = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -456,7 +446,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		long mergedDs = spark
+		final long mergedDs = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -466,7 +456,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		long mergedOrp = spark
+		final long mergedOrp = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -484,27 +474,27 @@ public class SparkDedupTest implements Serializable {
 		assertEquals(389, dataset);
 		assertEquals(517, otherresearchproduct);
 
-		long deletedOrgs = jsc
+		final long deletedOrgs = jsc
 			.textFile(testDedupGraphBasePath + "/organization")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		long deletedPubs = jsc
+		final long deletedPubs = jsc
 			.textFile(testDedupGraphBasePath + "/publication")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		long deletedSw = jsc
+		final long deletedSw = jsc
 			.textFile(testDedupGraphBasePath + "/software")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		long deletedDs = jsc
+		final long deletedDs = jsc
 			.textFile(testDedupGraphBasePath + "/dataset")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		long deletedOrp = jsc
+		final long deletedOrp = jsc
 			.textFile(testDedupGraphBasePath + "/otherresearchproduct")
 			.filter(this::isDeletedByInference)
 			.count();
@@ -520,21 +510,19 @@ public class SparkDedupTest implements Serializable {
 	@Order(6)
 	public void propagateRelationTest() throws Exception {
 
-		ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkPropagateRelation.class
-						.getResourceAsStream(
-							"/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")));
+						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")));
 		parser
-			.parseArgument(
-				new String[] {
-					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
-				});
+			.parseArgument(new String[] {
+				"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
+			});
 
 		new SparkPropagateRelation(parser, spark).run(isLookUpService);
 
-		long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
+		final long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
 
 		assertEquals(4862, relations);
 
@@ -548,10 +536,9 @@ public class SparkDedupTest implements Serializable {
 			.select(mergeRels.col("target"))
 			.distinct()
 			.toJavaRDD()
-			.mapToPair(
-				(PairFunction<Row, String, String>) r -> new Tuple2<String, String>(r.getString(0), "d"));
+			.mapToPair((PairFunction<Row, String, String>) r -> new Tuple2<>(r.getString(0), "d"));
 
-		JavaRDD<String> toCheck = jsc
+		final JavaRDD<String> toCheck = jsc
 			.textFile(testDedupGraphBasePath + "/relation")
 			.mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.source", json), json))
 			.join(mergedIds)
@@ -560,8 +547,8 @@ public class SparkDedupTest implements Serializable {
 			.join(mergedIds)
 			.map(t -> t._2()._1());
 
-		long deletedbyinference = toCheck.filter(this::isDeletedByInference).count();
-		long updated = toCheck.count();
+		final long deletedbyinference = toCheck.filter(this::isDeletedByInference).count();
+		final long updated = toCheck.count();
 
 		assertEquals(updated, deletedbyinference);
 	}
@@ -573,8 +560,8 @@ public class SparkDedupTest implements Serializable {
 		testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_2.json", 10, 2);
 	}
 
-	private void testUniqueness(String path, int expected_total, int expected_unique) {
-		Dataset<Relation> rel = spark
+	private void testUniqueness(final String path, final int expected_total, final int expected_unique) {
+		final Dataset<Relation> rel = spark
 			.read()
 			.textFile(getClass().getResource(path).getPath())
 			.map(
@@ -591,7 +578,8 @@ public class SparkDedupTest implements Serializable {
 		FileUtils.deleteDirectory(new File(testDedupGraphBasePath));
 	}
 
-	public boolean isDeletedByInference(String s) {
+	public boolean isDeletedByInference(final String s) {
 		return s.contains("\"deletedbyinference\":true");
 	}
+
 }

From f4bd2b5619a513baa476787fa10f559ccbc341f3 Mon Sep 17 00:00:00 2001
From: "michele.artini" <michele.artini@isti.cnr.it>
Date: Tue, 4 May 2021 10:26:14 +0200
Subject: [PATCH 5/5] recert file SparkDedupTest.java

---
 .../dnetlib/dhp/oa/dedup/SparkDedupTest.java  | 250 +++++++++---------
 1 file changed, 131 insertions(+), 119 deletions(-)

diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
index a95e28c43..bf4913056 100644
--- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
+++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java
@@ -12,6 +12,7 @@ import java.io.IOException;
 import java.io.Serializable;
 import java.net.URISyntaxException;
 import java.nio.file.Paths;
+import java.util.List;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
@@ -26,13 +27,7 @@ import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SparkSession;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.MethodOrderer;
-import org.junit.jupiter.api.Order;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.Mockito;
@@ -102,7 +97,8 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream("/eu/dnetlib/dhp/dedup/profiles/mock_orchestrator.xml")));
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/profiles/mock_orchestrator.xml")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("organization")))
@@ -110,7 +106,8 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/org.curr.conf.json")));
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/org.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("publication")))
@@ -118,7 +115,8 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json")));
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/pub.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("software")))
@@ -126,7 +124,8 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json")));
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/sw.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("dataset")))
@@ -134,7 +133,8 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json")));
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/ds.curr.conf.json")));
 
 		lenient()
 			.when(isLookUpService.getResourceProfileByQuery(Mockito.contains("otherresearchproduct")))
@@ -142,51 +142,54 @@ public class SparkDedupTest implements Serializable {
 				IOUtils
 					.toString(
 						SparkDedupTest.class
-							.getResourceAsStream("/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json")));
+							.getResourceAsStream(
+								"/eu/dnetlib/dhp/dedup/conf/orp.curr.conf.json")));
 	}
 
 	@Test
 	@Order(1)
 	public void createSimRelsTest() throws Exception {
 
-		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateSimRels.class
-						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createSimRels_parameters.json")));
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createSimRels_parameters.json")));
 
 		parser
-			.parseArgument(new String[] {
-				"-i", testGraphBasePath,
-				"-asi", testActionSetId,
-				"-la", "lookupurl",
-				"-w", testOutputBasePath,
-				"-np", "50"
-			});
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath,
+					"-asi", testActionSetId,
+					"-la", "lookupurl",
+					"-w", testOutputBasePath,
+					"-np", "50"
+				});
 
 		new SparkCreateSimRels(parser, spark).run(isLookUpService);
 
-		final long orgs_simrel = spark
+		long orgs_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization"))
 			.count();
 
-		final long pubs_simrel = spark
+		long pubs_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "publication"))
 			.count();
 
-		final long sw_simrel = spark
+		long sw_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "software"))
 			.count();
 
-		final long ds_simrel = spark
+		long ds_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "dataset"))
 			.count();
 
-		final long orp_simrel = spark
+		long orp_simrel = spark
 			.read()
 			.load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "otherresearchproduct"))
 			.count();
@@ -202,29 +205,31 @@ public class SparkDedupTest implements Serializable {
 	@Order(2)
 	public void cutMergeRelsTest() throws Exception {
 
-		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateMergeRels.class
-						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
 
 		parser
-			.parseArgument(new String[] {
-				"-i",
-				testGraphBasePath,
-				"-asi",
-				testActionSetId,
-				"-la",
-				"lookupurl",
-				"-w",
-				testOutputBasePath,
-				"-cc",
-				"3"
-			});
+			.parseArgument(
+				new String[] {
+					"-i",
+					testGraphBasePath,
+					"-asi",
+					testActionSetId,
+					"-la",
+					"lookupurl",
+					"-w",
+					testOutputBasePath,
+					"-cc",
+					"3"
+				});
 
 		new SparkCreateMergeRels(parser, spark).run(isLookUpService);
 
-		final long orgs_mergerel = spark
+		long orgs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -235,7 +240,7 @@ public class SparkDedupTest implements Serializable {
 			.where("cnt > 3")
 			.count();
 
-		final long pubs_mergerel = spark
+		long pubs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -245,7 +250,7 @@ public class SparkDedupTest implements Serializable {
 			.select("source", "cnt")
 			.where("cnt > 3")
 			.count();
-		final long sw_mergerel = spark
+		long sw_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -256,7 +261,7 @@ public class SparkDedupTest implements Serializable {
 			.where("cnt > 3")
 			.count();
 
-		final long ds_mergerel = spark
+		long ds_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -267,7 +272,7 @@ public class SparkDedupTest implements Serializable {
 			.where("cnt > 3")
 			.count();
 
-		final long orp_mergerel = spark
+		long orp_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -296,44 +301,46 @@ public class SparkDedupTest implements Serializable {
 	@Order(3)
 	public void createMergeRelsTest() throws Exception {
 
-		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateMergeRels.class
-						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createCC_parameters.json")));
 
 		parser
-			.parseArgument(new String[] {
-				"-i",
-				testGraphBasePath,
-				"-asi",
-				testActionSetId,
-				"-la",
-				"lookupurl",
-				"-w",
-				testOutputBasePath
-			});
+			.parseArgument(
+				new String[] {
+					"-i",
+					testGraphBasePath,
+					"-asi",
+					testActionSetId,
+					"-la",
+					"lookupurl",
+					"-w",
+					testOutputBasePath
+				});
 
 		new SparkCreateMergeRels(parser, spark).run(isLookUpService);
 
-		final long orgs_mergerel = spark
+		long orgs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
 			.count();
-		final long pubs_mergerel = spark
+		long pubs_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
 			.count();
-		final long sw_mergerel = spark
+		long sw_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
 			.count();
-		final long ds_mergerel = spark
+		long ds_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
 			.count();
 
-		final long orp_mergerel = spark
+		long orp_mergerel = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
 			.count();
@@ -350,39 +357,40 @@ public class SparkDedupTest implements Serializable {
 	@Order(4)
 	public void createDedupRecordTest() throws Exception {
 
-		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkCreateDedupRecord.class
-						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/createDedupRecord_parameters.json")));
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/createDedupRecord_parameters.json")));
 		parser
-			.parseArgument(new String[] {
-				"-i",
-				testGraphBasePath,
-				"-asi",
-				testActionSetId,
-				"-la",
-				"lookupurl",
-				"-w",
-				testOutputBasePath
-			});
+			.parseArgument(
+				new String[] {
+					"-i",
+					testGraphBasePath,
+					"-asi",
+					testActionSetId,
+					"-la",
+					"lookupurl",
+					"-w",
+					testOutputBasePath
+				});
 
 		new SparkCreateDedupRecord(parser, spark).run(isLookUpService);
 
-		final long orgs_deduprecord = jsc
+		long orgs_deduprecord = jsc
 			.textFile(testOutputBasePath + "/" + testActionSetId + "/organization_deduprecord")
 			.count();
-		final long pubs_deduprecord = jsc
+		long pubs_deduprecord = jsc
 			.textFile(testOutputBasePath + "/" + testActionSetId + "/publication_deduprecord")
 			.count();
-		final long sw_deduprecord = jsc
+		long sw_deduprecord = jsc
 			.textFile(testOutputBasePath + "/" + testActionSetId + "/software_deduprecord")
 			.count();
-		final long ds_deduprecord = jsc
-			.textFile(testOutputBasePath + "/" + testActionSetId + "/dataset_deduprecord")
-			.count();
-		final long orp_deduprecord = jsc
-			.textFile(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_deduprecord")
+		long ds_deduprecord = jsc.textFile(testOutputBasePath + "/" + testActionSetId + "/dataset_deduprecord").count();
+		long orp_deduprecord = jsc
+			.textFile(
+				testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_deduprecord")
 			.count();
 
 		assertEquals(85, orgs_deduprecord);
@@ -396,27 +404,29 @@ public class SparkDedupTest implements Serializable {
 	@Order(5)
 	public void updateEntityTest() throws Exception {
 
-		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkUpdateEntity.class
-						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json")));
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json")));
 		parser
-			.parseArgument(new String[] {
-				"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
-			});
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
+				});
 
 		new SparkUpdateEntity(parser, spark).run(isLookUpService);
 
-		final long organizations = jsc.textFile(testDedupGraphBasePath + "/organization").count();
-		final long publications = jsc.textFile(testDedupGraphBasePath + "/publication").count();
-		final long projects = jsc.textFile(testDedupGraphBasePath + "/project").count();
-		final long datasource = jsc.textFile(testDedupGraphBasePath + "/datasource").count();
-		final long softwares = jsc.textFile(testDedupGraphBasePath + "/software").count();
-		final long dataset = jsc.textFile(testDedupGraphBasePath + "/dataset").count();
-		final long otherresearchproduct = jsc.textFile(testDedupGraphBasePath + "/otherresearchproduct").count();
+		long organizations = jsc.textFile(testDedupGraphBasePath + "/organization").count();
+		long publications = jsc.textFile(testDedupGraphBasePath + "/publication").count();
+		long projects = jsc.textFile(testDedupGraphBasePath + "/project").count();
+		long datasource = jsc.textFile(testDedupGraphBasePath + "/datasource").count();
+		long softwares = jsc.textFile(testDedupGraphBasePath + "/software").count();
+		long dataset = jsc.textFile(testDedupGraphBasePath + "/dataset").count();
+		long otherresearchproduct = jsc.textFile(testDedupGraphBasePath + "/otherresearchproduct").count();
 
-		final long mergedOrgs = spark
+		long mergedOrgs = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -426,7 +436,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		final long mergedPubs = spark
+		long mergedPubs = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/publication_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -436,7 +446,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		final long mergedSw = spark
+		long mergedSw = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/software_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -446,7 +456,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		final long mergedDs = spark
+		long mergedDs = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/dataset_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -456,7 +466,7 @@ public class SparkDedupTest implements Serializable {
 			.distinct()
 			.count();
 
-		final long mergedOrp = spark
+		long mergedOrp = spark
 			.read()
 			.load(testOutputBasePath + "/" + testActionSetId + "/otherresearchproduct_mergerel")
 			.as(Encoders.bean(Relation.class))
@@ -474,27 +484,27 @@ public class SparkDedupTest implements Serializable {
 		assertEquals(389, dataset);
 		assertEquals(517, otherresearchproduct);
 
-		final long deletedOrgs = jsc
+		long deletedOrgs = jsc
 			.textFile(testDedupGraphBasePath + "/organization")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		final long deletedPubs = jsc
+		long deletedPubs = jsc
 			.textFile(testDedupGraphBasePath + "/publication")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		final long deletedSw = jsc
+		long deletedSw = jsc
 			.textFile(testDedupGraphBasePath + "/software")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		final long deletedDs = jsc
+		long deletedDs = jsc
 			.textFile(testDedupGraphBasePath + "/dataset")
 			.filter(this::isDeletedByInference)
 			.count();
 
-		final long deletedOrp = jsc
+		long deletedOrp = jsc
 			.textFile(testDedupGraphBasePath + "/otherresearchproduct")
 			.filter(this::isDeletedByInference)
 			.count();
@@ -510,19 +520,21 @@ public class SparkDedupTest implements Serializable {
 	@Order(6)
 	public void propagateRelationTest() throws Exception {
 
-		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+		ArgumentApplicationParser parser = new ArgumentApplicationParser(
 			IOUtils
 				.toString(
 					SparkPropagateRelation.class
-						.getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")));
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")));
 		parser
-			.parseArgument(new String[] {
-				"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
-			});
+			.parseArgument(
+				new String[] {
+					"-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath
+				});
 
 		new SparkPropagateRelation(parser, spark).run(isLookUpService);
 
-		final long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
+		long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count();
 
 		assertEquals(4862, relations);
 
@@ -536,9 +548,10 @@ public class SparkDedupTest implements Serializable {
 			.select(mergeRels.col("target"))
 			.distinct()
 			.toJavaRDD()
-			.mapToPair((PairFunction<Row, String, String>) r -> new Tuple2<>(r.getString(0), "d"));
+			.mapToPair(
+				(PairFunction<Row, String, String>) r -> new Tuple2<String, String>(r.getString(0), "d"));
 
-		final JavaRDD<String> toCheck = jsc
+		JavaRDD<String> toCheck = jsc
 			.textFile(testDedupGraphBasePath + "/relation")
 			.mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.source", json), json))
 			.join(mergedIds)
@@ -547,8 +560,8 @@ public class SparkDedupTest implements Serializable {
 			.join(mergedIds)
 			.map(t -> t._2()._1());
 
-		final long deletedbyinference = toCheck.filter(this::isDeletedByInference).count();
-		final long updated = toCheck.count();
+		long deletedbyinference = toCheck.filter(this::isDeletedByInference).count();
+		long updated = toCheck.count();
 
 		assertEquals(updated, deletedbyinference);
 	}
@@ -560,8 +573,8 @@ public class SparkDedupTest implements Serializable {
 		testUniqueness("/eu/dnetlib/dhp/dedup/test/relation_2.json", 10, 2);
 	}
 
-	private void testUniqueness(final String path, final int expected_total, final int expected_unique) {
-		final Dataset<Relation> rel = spark
+	private void testUniqueness(String path, int expected_total, int expected_unique) {
+		Dataset<Relation> rel = spark
 			.read()
 			.textFile(getClass().getResource(path).getPath())
 			.map(
@@ -578,8 +591,7 @@ public class SparkDedupTest implements Serializable {
 		FileUtils.deleteDirectory(new File(testDedupGraphBasePath));
 	}
 
-	public boolean isDeletedByInference(final String s) {
+	public boolean isDeletedByInference(String s) {
 		return s.contains("\"deletedbyinference\":true");
 	}
-
 }