Aggiornare 'dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java'

fix in SQL query: while writing the blacklist constraint it used d.id to indicate the datasource id, but no alias for the datasource was defined. So I removed the alias

dhp-build/dhp-code-style/pom.xml

@@ -15,12 +15,12 @@
         <snapshotRepository>
             <id>dnet45-snapshots</id>
             <name>DNet45 Snapshots</name>
-            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots</url>
+            <url>https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots</url>
             <layout>default</layout>
         </snapshotRepository>
         <repository>
             <id>dnet45-releases</id>
-            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases</url>
+            <url>https://maven.d4science.org/nexus/content/repositories/dnet45-releases</url>
         </repository>
     </distributionManagement>
 

dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_dnet.xml

@@ -19,7 +19,7 @@
         <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
-        <setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
+        <setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="false"/>
         <setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>

dhp-common/pom.xml

@@ -6,7 +6,7 @@
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dhp</artifactId>
 		<version>1.2.4-SNAPSHOT</version>
-		<relativePath>../</relativePath>
+		<relativePath>../pom.xml</relativePath>
 	</parent>
 
 	<artifactId>dhp-common</artifactId>
@@ -87,6 +87,21 @@
 			<groupId>org.postgresql</groupId>
 			<artifactId>postgresql</artifactId>
 		</dependency>
+
+		<dependency>
+			<groupId>com.squareup.okhttp3</groupId>
+			<artifactId>okhttp</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>dnet-pace-core</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>eu.dnetlib.dhp</groupId>
+			<artifactId>dhp-schemas</artifactId>
+		</dependency>
 	</dependencies>
 
 </project>

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStore.java

@@ -1,119 +0,0 @@
-
-package eu.dnetlib.data.mdstore.manager.common.model;
-
-import java.io.Serializable;
-import java.util.UUID;
-
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.Table;
-
-@Entity
-@Table(name = "mdstores")
-public class MDStore implements Serializable {
-
-	/** */
-	private static final long serialVersionUID = 3160530489149700055L;
-
-	@Id
-	@Column(name = "id")
-	private String id;
-
-	@Column(name = "format")
-	private String format;
-
-	@Column(name = "layout")
-	private String layout;
-
-	@Column(name = "interpretation")
-	private String interpretation;
-
-	@Column(name = "datasource_name")
-	private String datasourceName;
-
-	@Column(name = "datasource_id")
-	private String datasourceId;
-
-	@Column(name = "api_id")
-	private String apiId;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(final String id) {
-		this.id = id;
-	}
-
-	public String getFormat() {
-		return format;
-	}
-
-	public void setFormat(final String format) {
-		this.format = format;
-	}
-
-	public String getLayout() {
-		return layout;
-	}
-
-	public void setLayout(final String layout) {
-		this.layout = layout;
-	}
-
-	public String getInterpretation() {
-		return interpretation;
-	}
-
-	public void setInterpretation(final String interpretation) {
-		this.interpretation = interpretation;
-	}
-
-	public String getDatasourceName() {
-		return datasourceName;
-	}
-
-	public void setDatasourceName(final String datasourceName) {
-		this.datasourceName = datasourceName;
-	}
-
-	public String getDatasourceId() {
-		return datasourceId;
-	}
-
-	public void setDatasourceId(final String datasourceId) {
-		this.datasourceId = datasourceId;
-	}
-
-	public String getApiId() {
-		return apiId;
-	}
-
-	public void setApiId(final String apiId) {
-		this.apiId = apiId;
-	}
-
-	public static MDStore newInstance(
-		final String format, final String layout, final String interpretation) {
-		return newInstance(format, layout, interpretation, null, null, null);
-	}
-
-	public static MDStore newInstance(
-		final String format,
-		final String layout,
-		final String interpretation,
-		final String dsName,
-		final String dsId,
-		final String apiId) {
-		final MDStore md = new MDStore();
-		md.setId("md-" + UUID.randomUUID());
-		md.setFormat(format);
-		md.setLayout(layout);
-		md.setInterpretation(interpretation);
-		md.setDatasourceName(dsName);
-		md.setDatasourceId(dsId);
-		md.setApiId(apiId);
-		return md;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreCurrentVersion.java

@@ -1,51 +0,0 @@
-
-package eu.dnetlib.data.mdstore.manager.common.model;
-
-import java.io.Serializable;
-
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.Table;
-
-@Entity
-@Table(name = "mdstore_current_versions")
-public class MDStoreCurrentVersion implements Serializable {
-
-	/** */
-	private static final long serialVersionUID = -4757725888593745773L;
-
-	@Id
-	@Column(name = "mdstore")
-	private String mdstore;
-
-	@Column(name = "current_version")
-	private String currentVersion;
-
-	public String getMdstore() {
-		return mdstore;
-	}
-
-	public void setMdstore(final String mdstore) {
-		this.mdstore = mdstore;
-	}
-
-	public String getCurrentVersion() {
-		return currentVersion;
-	}
-
-	public void setCurrentVersion(final String currentVersion) {
-		this.currentVersion = currentVersion;
-	}
-
-	public static MDStoreCurrentVersion newInstance(final String mdId, final String versionId) {
-		final MDStoreCurrentVersion cv = new MDStoreCurrentVersion();
-		cv.setMdstore(mdId);
-		cv.setCurrentVersion(versionId);
-		return cv;
-	}
-
-	public static MDStoreCurrentVersion newInstance(final MDStoreVersion v) {
-		return newInstance(v.getMdstore(), v.getId());
-	}
-}

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreVersion.java

@@ -1,99 +0,0 @@
-
-package eu.dnetlib.data.mdstore.manager.common.model;
-
-import java.io.Serializable;
-import java.util.Date;
-
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.Table;
-import javax.persistence.Temporal;
-import javax.persistence.TemporalType;
-
-@Entity
-@Table(name = "mdstore_versions")
-public class MDStoreVersion implements Serializable {
-
-	/** */
-	private static final long serialVersionUID = -4763494442274298339L;
-
-	@Id
-	@Column(name = "id")
-	private String id;
-
-	@Column(name = "mdstore")
-	private String mdstore;
-
-	@Column(name = "writing")
-	private boolean writing;
-
-	@Column(name = "readcount")
-	private int readCount = 0;
-
-	@Column(name = "lastupdate")
-	@Temporal(TemporalType.TIMESTAMP)
-	private Date lastUpdate;
-
-	@Column(name = "size")
-	private long size = 0;
-
-	public static MDStoreVersion newInstance(final String mdId, final boolean writing) {
-		final MDStoreVersion t = new MDStoreVersion();
-		t.setId(mdId + "-" + new Date().getTime());
-		t.setMdstore(mdId);
-		t.setLastUpdate(null);
-		t.setWriting(writing);
-		t.setReadCount(0);
-		t.setSize(0);
-		return t;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(final String id) {
-		this.id = id;
-	}
-
-	public String getMdstore() {
-		return mdstore;
-	}
-
-	public void setMdstore(final String mdstore) {
-		this.mdstore = mdstore;
-	}
-
-	public boolean isWriting() {
-		return writing;
-	}
-
-	public void setWriting(final boolean writing) {
-		this.writing = writing;
-	}
-
-	public int getReadCount() {
-		return readCount;
-	}
-
-	public void setReadCount(final int readCount) {
-		this.readCount = readCount;
-	}
-
-	public Date getLastUpdate() {
-		return lastUpdate;
-	}
-
-	public void setLastUpdate(final Date lastUpdate) {
-		this.lastUpdate = lastUpdate;
-	}
-
-	public long getSize() {
-		return size;
-	}
-
-	public void setSize(final long size) {
-		this.size = size;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreWithInfo.java

@@ -1,143 +0,0 @@
-
-package eu.dnetlib.data.mdstore.manager.common.model;
-
-import java.io.Serializable;
-import java.util.Date;
-
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.Table;
-import javax.persistence.Temporal;
-import javax.persistence.TemporalType;
-
-@Entity
-@Table(name = "mdstores_with_info")
-public class MDStoreWithInfo implements Serializable {
-
-	/** */
-	private static final long serialVersionUID = -8445784770687571492L;
-
-	@Id
-	@Column(name = "id")
-	private String id;
-
-	@Column(name = "format")
-	private String format;
-
-	@Column(name = "layout")
-	private String layout;
-
-	@Column(name = "interpretation")
-	private String interpretation;
-
-	@Column(name = "datasource_name")
-	private String datasourceName;
-
-	@Column(name = "datasource_id")
-	private String datasourceId;
-
-	@Column(name = "api_id")
-	private String apiId;
-
-	@Column(name = "current_version")
-	private String currentVersion;
-
-	@Column(name = "lastupdate")
-	@Temporal(TemporalType.TIMESTAMP)
-	private Date lastUpdate;
-
-	@Column(name = "size")
-	private long size = 0;
-
-	@Column(name = "n_versions")
-	private long numberOfVersions = 0;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(final String id) {
-		this.id = id;
-	}
-
-	public String getFormat() {
-		return format;
-	}
-
-	public void setFormat(final String format) {
-		this.format = format;
-	}
-
-	public String getLayout() {
-		return layout;
-	}
-
-	public void setLayout(final String layout) {
-		this.layout = layout;
-	}
-
-	public String getInterpretation() {
-		return interpretation;
-	}
-
-	public void setInterpretation(final String interpretation) {
-		this.interpretation = interpretation;
-	}
-
-	public String getDatasourceName() {
-		return datasourceName;
-	}
-
-	public void setDatasourceName(final String datasourceName) {
-		this.datasourceName = datasourceName;
-	}
-
-	public String getDatasourceId() {
-		return datasourceId;
-	}
-
-	public void setDatasourceId(final String datasourceId) {
-		this.datasourceId = datasourceId;
-	}
-
-	public String getApiId() {
-		return apiId;
-	}
-
-	public void setApiId(final String apiId) {
-		this.apiId = apiId;
-	}
-
-	public String getCurrentVersion() {
-		return currentVersion;
-	}
-
-	public void setCurrentVersion(final String currentVersion) {
-		this.currentVersion = currentVersion;
-	}
-
-	public Date getLastUpdate() {
-		return lastUpdate;
-	}
-
-	public void setLastUpdate(final Date lastUpdate) {
-		this.lastUpdate = lastUpdate;
-	}
-
-	public long getSize() {
-		return size;
-	}
-
-	public void setSize(final long size) {
-		this.size = size;
-	}
-
-	public long getNumberOfVersions() {
-		return numberOfVersions;
-	}
-
-	public void setNumberOfVersions(final long numberOfVersions) {
-		this.numberOfVersions = numberOfVersions;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/Constants.java

@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.common;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+public class Constants {
+
+	public static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
+	public static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
+
+	public static String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
+
+	static {
+		accessRightsCoarMap.put("OPEN", "c_abf2");
+		accessRightsCoarMap.put("RESTRICTED", "c_16ec");
+		accessRightsCoarMap.put("OPEN SOURCE", "c_abf2");
+		accessRightsCoarMap.put("CLOSED", "c_14cb");
+		accessRightsCoarMap.put("EMBARGO", "c_f1cf");
+	}
+
+	static {
+		coarCodeLabelMap.put("c_abf2", "OPEN");
+		coarCodeLabelMap.put("c_16ec", "RESTRICTED");
+		coarCodeLabelMap.put("c_14cb", "CLOSED");
+		coarCodeLabelMap.put("c_f1cf", "EMBARGO");
+	}
+
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/GraphResultMapper.java

@@ -0,0 +1,412 @@
+
+package eu.dnetlib.dhp.common;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+import eu.dnetlib.dhp.schema.dump.oaf.*;
+import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityInstance;
+import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
+import eu.dnetlib.dhp.schema.oaf.DataInfo;
+import eu.dnetlib.dhp.schema.oaf.Field;
+import eu.dnetlib.dhp.schema.oaf.Journal;
+import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
+
+public class GraphResultMapper implements Serializable {
+
+	public static <E extends eu.dnetlib.dhp.schema.oaf.OafEntity> Result map(
+		E in) {
+
+		CommunityResult out = new CommunityResult();
+
+		eu.dnetlib.dhp.schema.oaf.Result input = (eu.dnetlib.dhp.schema.oaf.Result) in;
+		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> ort = Optional.ofNullable(input.getResulttype());
+		if (ort.isPresent()) {
+			switch (ort.get().getClassid()) {
+				case "publication":
+					Optional<Journal> journal = Optional
+						.ofNullable(((eu.dnetlib.dhp.schema.oaf.Publication) input).getJournal());
+					if (journal.isPresent()) {
+						Journal j = journal.get();
+						Container c = new Container();
+						c.setConferencedate(j.getConferencedate());
+						c.setConferenceplace(j.getConferenceplace());
+						c.setEdition(j.getEdition());
+						c.setEp(j.getEp());
+						c.setIss(j.getIss());
+						c.setIssnLinking(j.getIssnLinking());
+						c.setIssnOnline(j.getIssnOnline());
+						c.setIssnPrinted(j.getIssnPrinted());
+						c.setName(j.getName());
+						c.setSp(j.getSp());
+						c.setVol(j.getVol());
+						out.setContainer(c);
+						out.setType(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE.getClassname());
+					}
+					break;
+				case "dataset":
+					eu.dnetlib.dhp.schema.oaf.Dataset id = (eu.dnetlib.dhp.schema.oaf.Dataset) input;
+					Optional.ofNullable(id.getSize()).ifPresent(v -> out.setSize(v.getValue()));
+					Optional.ofNullable(id.getVersion()).ifPresent(v -> out.setVersion(v.getValue()));
+
+					out
+						.setGeolocation(
+							Optional
+								.ofNullable(id.getGeolocation())
+								.map(
+									igl -> igl
+										.stream()
+										.filter(Objects::nonNull)
+										.map(gli -> {
+											GeoLocation gl = new GeoLocation();
+											gl.setBox(gli.getBox());
+											gl.setPlace(gli.getPlace());
+											gl.setPoint(gli.getPoint());
+											return gl;
+										})
+										.collect(Collectors.toList()))
+								.orElse(null));
+
+					out.setType(ModelConstants.DATASET_DEFAULT_RESULTTYPE.getClassname());
+					break;
+				case "software":
+
+					eu.dnetlib.dhp.schema.oaf.Software is = (eu.dnetlib.dhp.schema.oaf.Software) input;
+					Optional
+						.ofNullable(is.getCodeRepositoryUrl())
+						.ifPresent(value -> out.setCodeRepositoryUrl(value.getValue()));
+					Optional
+						.ofNullable(is.getDocumentationUrl())
+						.ifPresent(
+							value -> out
+								.setDocumentationUrl(
+									value
+										.stream()
+										.map(v -> v.getValue())
+										.collect(Collectors.toList())));
+
+					Optional
+						.ofNullable(is.getProgrammingLanguage())
+						.ifPresent(value -> out.setProgrammingLanguage(value.getClassid()));
+
+					out.setType(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE.getClassname());
+					break;
+				case "other":
+
+					eu.dnetlib.dhp.schema.oaf.OtherResearchProduct ir = (eu.dnetlib.dhp.schema.oaf.OtherResearchProduct) input;
+					out
+						.setContactgroup(
+							Optional
+								.ofNullable(ir.getContactgroup())
+								.map(value -> value.stream().map(cg -> cg.getValue()).collect(Collectors.toList()))
+								.orElse(null));
+
+					out
+						.setContactperson(
+							Optional
+								.ofNullable(ir.getContactperson())
+								.map(value -> value.stream().map(cp -> cp.getValue()).collect(Collectors.toList()))
+								.orElse(null));
+					out
+						.setTool(
+							Optional
+								.ofNullable(ir.getTool())
+								.map(value -> value.stream().map(t -> t.getValue()).collect(Collectors.toList()))
+								.orElse(null));
+
+					out.setType(ModelConstants.ORP_DEFAULT_RESULTTYPE.getClassname());
+
+					break;
+			}
+
+			Optional
+				.ofNullable(input.getAuthor())
+				.ifPresent(ats -> out.setAuthor(ats.stream().map(at -> getAuthor(at)).collect(Collectors.toList())));
+
+			// I do not map Access Right UNKNOWN or OTHER
+
+			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oar = Optional.ofNullable(input.getBestaccessright());
+			if (oar.isPresent()) {
+				if (Constants.accessRightsCoarMap.containsKey(oar.get().getClassid())) {
+					String code = Constants.accessRightsCoarMap.get(oar.get().getClassid());
+					out
+						.setBestaccessright(
+							AccessRight
+								.newInstance(
+									code,
+									Constants.coarCodeLabelMap.get(code),
+									Constants.COAR_ACCESS_RIGHT_SCHEMA));
+				}
+			}
+
+			final List<String> contributorList = new ArrayList<>();
+			Optional
+				.ofNullable(input.getContributor())
+				.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
+			out.setContributor(contributorList);
+
+			Optional
+				.ofNullable(input.getCountry())
+				.ifPresent(
+					value -> out
+						.setCountry(
+							value
+								.stream()
+								.map(
+									c -> {
+										if (c.getClassid().equals((ModelConstants.UNKNOWN))) {
+											return null;
+										}
+										Country country = new Country();
+										country.setCode(c.getClassid());
+										country.setLabel(c.getClassname());
+										Optional
+											.ofNullable(c.getDataInfo())
+											.ifPresent(
+												provenance -> country
+													.setProvenance(
+														Provenance
+															.newInstance(
+																provenance
+																	.getProvenanceaction()
+																	.getClassname(),
+																c.getDataInfo().getTrust())));
+										return country;
+									})
+								.filter(Objects::nonNull)
+								.collect(Collectors.toList())));
+
+			final List<String> coverageList = new ArrayList<>();
+			Optional
+				.ofNullable(input.getCoverage())
+				.ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue())));
+			out.setCoverage(coverageList);
+
+			out.setDateofcollection(input.getDateofcollection());
+
+			final List<String> descriptionList = new ArrayList<>();
+			Optional
+				.ofNullable(input.getDescription())
+				.ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue())));
+			out.setDescription(descriptionList);
+			Optional<Field<String>> oStr = Optional.ofNullable(input.getEmbargoenddate());
+			if (oStr.isPresent()) {
+				out.setEmbargoenddate(oStr.get().getValue());
+			}
+
+			final List<String> formatList = new ArrayList<>();
+			Optional
+				.ofNullable(input.getFormat())
+				.ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue())));
+			out.setFormat(formatList);
+			out.setId(input.getId());
+			out.setOriginalId(input.getOriginalId());
+
+			Optional<List<eu.dnetlib.dhp.schema.oaf.Instance>> oInst = Optional
+				.ofNullable(input.getInstance());
+
+			if (oInst.isPresent()) {
+				out
+					.setInstance(
+						oInst.get().stream().map(i -> getInstance(i)).collect(Collectors.toList()));
+
+			}
+
+			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oL = Optional.ofNullable(input.getLanguage());
+			if (oL.isPresent()) {
+				eu.dnetlib.dhp.schema.oaf.Qualifier language = oL.get();
+				out.setLanguage(Qualifier.newInstance(language.getClassid(), language.getClassname()));
+			}
+			Optional<Long> oLong = Optional.ofNullable(input.getLastupdatetimestamp());
+			if (oLong.isPresent()) {
+				out.setLastupdatetimestamp(oLong.get());
+			}
+			Optional<List<StructuredProperty>> otitle = Optional.ofNullable(input.getTitle());
+			if (otitle.isPresent()) {
+				List<StructuredProperty> iTitle = otitle
+					.get()
+					.stream()
+					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title"))
+					.collect(Collectors.toList());
+				if (iTitle.size() > 0) {
+					out.setMaintitle(iTitle.get(0).getValue());
+				}
+
+				iTitle = otitle
+					.get()
+					.stream()
+					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle"))
+					.collect(Collectors.toList());
+				if (iTitle.size() > 0) {
+					out.setSubtitle(iTitle.get(0).getValue());
+				}
+
+			}
+
+			List<ControlledField> pids = new ArrayList<>();
+			Optional
+				.ofNullable(input.getPid())
+				.ifPresent(
+					value -> value
+						.stream()
+						.forEach(
+							p -> pids
+								.add(
+									ControlledField
+										.newInstance(p.getQualifier().getClassid(), p.getValue()))));
+			out.setPid(pids);
+			oStr = Optional.ofNullable(input.getDateofacceptance());
+			if (oStr.isPresent()) {
+				out.setPublicationdate(oStr.get().getValue());
+			}
+			oStr = Optional.ofNullable(input.getPublisher());
+			if (oStr.isPresent()) {
+				out.setPublisher(oStr.get().getValue());
+			}
+
+			List<String> sourceList = new ArrayList<>();
+			Optional
+				.ofNullable(input.getSource())
+				.ifPresent(value -> value.stream().forEach(s -> sourceList.add(s.getValue())));
+			// out.setSource(input.getSource().stream().map(s -> s.getValue()).collect(Collectors.toList()));
+			List<Subject> subjectList = new ArrayList<>();
+			Optional
+				.ofNullable(input.getSubject())
+				.ifPresent(
+					value -> value
+						.forEach(s -> subjectList.add(getSubject(s))));
+
+			out.setSubjects(subjectList);
+
+			out.setType(input.getResulttype().getClassid());
+		}
+
+		out
+			.setCollectedfrom(
+				input
+					.getCollectedfrom()
+					.stream()
+					.map(cf -> KeyValue.newInstance(cf.getKey(), cf.getValue()))
+					.collect(Collectors.toList()));
+
+		return out;
+
+	}
+
+	private static CommunityInstance getInstance(eu.dnetlib.dhp.schema.oaf.Instance i) {
+		CommunityInstance instance = new CommunityInstance();
+
+		setCommonValue(i, instance);
+
+		instance
+			.setCollectedfrom(
+				KeyValue
+					.newInstance(i.getCollectedfrom().getKey(), i.getCollectedfrom().getValue()));
+
+		instance
+			.setHostedby(
+				KeyValue.newInstance(i.getHostedby().getKey(), i.getHostedby().getValue()));
+
+		return instance;
+
+	}
+
+	private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {
+		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
+			.ofNullable(i.getAccessright());
+		if (opAr.isPresent()) {
+			if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
+				String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
+				instance
+					.setAccessright(
+						AccessRight
+							.newInstance(
+								code,
+								Constants.coarCodeLabelMap.get(code),
+								Constants.COAR_ACCESS_RIGHT_SCHEMA));
+			}
+		}
+
+		Optional
+			.ofNullable(i.getLicense())
+			.ifPresent(value -> instance.setLicense(value.getValue()));
+		Optional
+			.ofNullable(i.getDateofacceptance())
+			.ifPresent(value -> instance.setPublicationdate(value.getValue()));
+		Optional
+			.ofNullable(i.getRefereed())
+			.ifPresent(value -> instance.setRefereed(value.getClassname()));
+		Optional
+			.ofNullable(i.getInstancetype())
+			.ifPresent(value -> instance.setType(value.getClassname()));
+		Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
+
+	}
+
+	private static Subject getSubject(StructuredProperty s) {
+		Subject subject = new Subject();
+		subject.setSubject(ControlledField.newInstance(s.getQualifier().getClassid(), s.getValue()));
+		Optional<DataInfo> di = Optional.ofNullable(s.getDataInfo());
+		if (di.isPresent()) {
+			Provenance p = new Provenance();
+			p.setProvenance(di.get().getProvenanceaction().getClassname());
+			p.setTrust(di.get().getTrust());
+			subject.setProvenance(p);
+		}
+
+		return subject;
+	}
+
+	private static Author getAuthor(eu.dnetlib.dhp.schema.oaf.Author oa) {
+		Author a = new Author();
+		a.setFullname(oa.getFullname());
+		a.setName(oa.getName());
+		a.setSurname(oa.getSurname());
+		a.setRank(oa.getRank());
+
+		Optional<List<StructuredProperty>> oPids = Optional
+			.ofNullable(oa.getPid());
+		if (oPids.isPresent()) {
+			Pid pid = getOrcid(oPids.get());
+			if (pid != null) {
+				a.setPid(pid);
+			}
+		}
+
+		return a;
+	}
+
+	private static Pid getOrcid(List<StructuredProperty> p) {
+		for (StructuredProperty pid : p) {
+			if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID)) {
+				Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
+				if (di.isPresent()) {
+					return Pid
+						.newInstance(
+							ControlledField
+								.newInstance(
+									pid.getQualifier().getClassid(),
+									pid.getValue()),
+							Provenance
+								.newInstance(
+									di.get().getProvenanceaction().getClassname(),
+									di.get().getTrust()));
+				} else {
+					return Pid
+						.newInstance(
+							ControlledField
+								.newInstance(
+									pid.getQualifier().getClassid(),
+									pid.getValue())
+
+						);
+				}
+
+			}
+		}
+		return null;
+	}
+
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java

@@ -0,0 +1,114 @@
+
+package eu.dnetlib.dhp.common;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.hadoop.fs.*;
+
+public class MakeTarArchive implements Serializable {
+
+	private static TarArchiveOutputStream getTar(FileSystem fileSystem, String outputPath) throws IOException {
+		Path hdfsWritePath = new Path(outputPath);
+		FSDataOutputStream fsDataOutputStream = null;
+		if (fileSystem.exists(hdfsWritePath)) {
+			fileSystem.delete(hdfsWritePath, true);
+
+		}
+		fsDataOutputStream = fileSystem.create(hdfsWritePath);
+
+		return new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
+	}
+
+	private static void write(FileSystem fileSystem, String inputPath, String outputPath, String dir_name)
+		throws IOException {
+
+		Path hdfsWritePath = new Path(outputPath);
+		FSDataOutputStream fsDataOutputStream = null;
+		if (fileSystem.exists(hdfsWritePath)) {
+			fileSystem.delete(hdfsWritePath, true);
+
+		}
+		fsDataOutputStream = fileSystem.create(hdfsWritePath);
+
+		TarArchiveOutputStream ar = new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
+
+		RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
+			.listFiles(
+				new Path(inputPath), true);
+
+		while (fileStatusListIterator.hasNext()) {
+			writeCurrentFile(fileSystem, dir_name, fileStatusListIterator, ar, 0);
+		}
+
+		ar.close();
+	}
+
+	public static void tarMaxSize(FileSystem fileSystem, String inputPath, String outputPath, String dir_name,
+		int gBperSplit) throws IOException {
+		final long bytesPerSplit = 1024L * 1024L * 1024L * gBperSplit;
+
+		long sourceSize = fileSystem.getContentSummary(new Path(inputPath)).getSpaceConsumed();
+
+		if (sourceSize < bytesPerSplit) {
+			write(fileSystem, inputPath, outputPath + ".tar", dir_name);
+		} else {
+			int partNum = 0;
+
+			RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
+				.listFiles(
+					new Path(inputPath), true);
+			boolean next = fileStatusListIterator.hasNext();
+			while (next) {
+				TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar");
+
+				long current_size = 0;
+				while (next && current_size < bytesPerSplit) {
+					current_size = writeCurrentFile(fileSystem, dir_name, fileStatusListIterator, ar, current_size);
+					next = fileStatusListIterator.hasNext();
+
+				}
+
+				partNum += 1;
+				ar.close();
+			}
+
+		}
+
+	}
+
+	private static long writeCurrentFile(FileSystem fileSystem, String dir_name,
+		RemoteIterator<LocatedFileStatus> fileStatusListIterator,
+		TarArchiveOutputStream ar, long current_size) throws IOException {
+		LocatedFileStatus fileStatus = fileStatusListIterator.next();
+
+		Path p = fileStatus.getPath();
+		String p_string = p.toString();
+		if (!p_string.endsWith("_SUCCESS")) {
+			String name = p_string.substring(p_string.lastIndexOf("/") + 1);
+			TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
+			entry.setSize(fileStatus.getLen());
+			current_size += fileStatus.getLen();
+			ar.putArchiveEntry(entry);
+
+			InputStream is = fileSystem.open(fileStatus.getPath());
+
+			BufferedInputStream bis = new BufferedInputStream(is);
+
+			int count;
+			byte data[] = new byte[1024];
+			while ((count = bis.read(data, 0, data.length)) != -1) {
+				ar.write(data, 0, count);
+			}
+			bis.close();
+			ar.closeArchiveEntry();
+
+		}
+		return current_size;
+	}
+
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java

@@ -0,0 +1,53 @@
+
+package eu.dnetlib.dhp.common.api;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import okhttp3.MediaType;
+import okhttp3.RequestBody;
+import okhttp3.internal.Util;
+import okio.BufferedSink;
+import okio.Okio;
+import okio.Source;
+
+public class InputStreamRequestBody extends RequestBody {
+
+	private InputStream inputStream;
+	private MediaType mediaType;
+	private long lenght;
+
+	public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
+
+		return new InputStreamRequestBody(inputStream, mediaType, len);
+	}
+
+	private InputStreamRequestBody(InputStream inputStream, MediaType mediaType, long len) {
+		this.inputStream = inputStream;
+		this.mediaType = mediaType;
+		this.lenght = len;
+	}
+
+	@Override
+	public MediaType contentType() {
+		return mediaType;
+	}
+
+	@Override
+	public long contentLength() {
+
+		return lenght;
+
+	}
+
+	@Override
+	public void writeTo(BufferedSink sink) throws IOException {
+		Source source = null;
+		try {
+			source = Okio.source(inputStream);
+			sink.writeAll(source);
+		} finally {
+			Util.closeQuietly(source);
+		}
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java

@@ -0,0 +1,8 @@
+
+package eu.dnetlib.dhp.common.api;
+
+public class MissingConceptDoiException extends Throwable {
+	public MissingConceptDoiException(String message) {
+		super(message);
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java

@@ -0,0 +1,315 @@
+
+package eu.dnetlib.dhp.common.api;
+
+import java.io.*;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import com.google.gson.Gson;
+
+import eu.dnetlib.dhp.common.api.zenodo.ZenodoModel;
+import eu.dnetlib.dhp.common.api.zenodo.ZenodoModelList;
+import okhttp3.*;
+
+public class ZenodoAPIClient implements Serializable {
+
+	String urlString;
+	String bucket;
+
+	String deposition_id;
+	String access_token;
+
+	public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
+
+	private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip");
+
+	public String getUrlString() {
+		return urlString;
+	}
+
+	public void setUrlString(String urlString) {
+		this.urlString = urlString;
+	}
+
+	public String getBucket() {
+		return bucket;
+	}
+
+	public void setBucket(String bucket) {
+		this.bucket = bucket;
+	}
+
+	public void setDeposition_id(String deposition_id) {
+		this.deposition_id = deposition_id;
+	}
+
+	public ZenodoAPIClient(String urlString, String access_token) throws IOException {
+
+		this.urlString = urlString;
+		this.access_token = access_token;
+	}
+
+	/**
+	 * Brand new deposition in Zenodo. It sets the deposition_id and the bucket where to store the files to upload
+	 *
+	 * @return response code
+	 * @throws IOException
+	 */
+	public int newDeposition() throws IOException {
+		String json = "{}";
+		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
+
+		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
+
+		Request request = new Request.Builder()
+			.url(urlString)
+			.addHeader("Content-Type", "application/json") // add request headers
+			.addHeader("Authorization", "Bearer " + access_token)
+			.post(body)
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+
+			// Get response body
+			json = response.body().string();
+
+			ZenodoModel newSubmission = new Gson().fromJson(json, ZenodoModel.class);
+			this.bucket = newSubmission.getLinks().getBucket();
+			this.deposition_id = newSubmission.getId();
+
+			return response.code();
+
+		}
+
+	}
+
+	/**
+	 * Upload files in Zenodo.
+	 *
+	 * @param is the inputStream for the file to upload
+	 * @param file_name the name of the file as it will appear on Zenodo
+	 * @param len the size of the file
+	 * @return the response code
+	 */
+	public int uploadIS(InputStream is, String file_name, long len) throws IOException {
+		OkHttpClient httpClient = new OkHttpClient.Builder()
+			.writeTimeout(600, TimeUnit.SECONDS)
+			.readTimeout(600, TimeUnit.SECONDS)
+			.connectTimeout(600, TimeUnit.SECONDS)
+			.build();
+
+		Request request = new Request.Builder()
+			.url(bucket + "/" + file_name)
+			.addHeader("Content-Type", "application/zip") // add request headers
+			.addHeader("Authorization", "Bearer " + access_token)
+			.put(InputStreamRequestBody.create(MEDIA_TYPE_ZIP, is, len))
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+			return response.code();
+		}
+	}
+
+	/**
+	 * Associates metadata information to the current deposition
+	 *
+	 * @param metadata the metadata
+	 * @return response code
+	 * @throws IOException
+	 */
+	public int sendMretadata(String metadata) throws IOException {
+
+		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
+
+		RequestBody body = RequestBody.create(metadata, MEDIA_TYPE_JSON);
+
+		Request request = new Request.Builder()
+			.url(urlString + "/" + deposition_id)
+			.addHeader("Content-Type", "application/json") // add request headers
+			.addHeader("Authorization", "Bearer " + access_token)
+			.put(body)
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+
+			return response.code();
+
+		}
+
+	}
+
+	/**
+	 * To publish the current deposition. It works for both new deposition or new version of an old deposition
+	 *
+	 * @return response code
+	 * @throws IOException
+	 */
+	public int publish() throws IOException {
+
+		String json = "{}";
+
+		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
+
+		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
+
+		Request request = new Request.Builder()
+			.url(urlString + "/" + deposition_id + "/actions/publish")
+			.addHeader("Authorization", "Bearer " + access_token)
+			.post(body)
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+
+			return response.code();
+
+		}
+	}
+
+	/**
+	 * To create a new version of an already published deposition. It sets the deposition_id and the bucket to be used
+	 * for the new version.
+	 *
+	 * @param concept_rec_id the concept record id of the deposition for which to create a new version. It is the last
+	 *            part of the url for the DOI Zenodo suggests to use to cite all versions: DOI: 10.xxx/zenodo.656930
+	 *            concept_rec_id = 656930
+	 * @return response code
+	 * @throws IOException
+	 * @throws MissingConceptDoiException
+	 */
+	public int newVersion(String concept_rec_id) throws IOException, MissingConceptDoiException {
+		setDepositionId(concept_rec_id);
+		String json = "{}";
+
+		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
+
+		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
+
+		Request request = new Request.Builder()
+			.url(urlString + "/" + deposition_id + "/actions/newversion")
+			.addHeader("Authorization", "Bearer " + access_token)
+			.post(body)
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+
+			ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
+			String latest_draft = zenodoModel.getLinks().getLatest_draft();
+			deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
+			bucket = getBucket(latest_draft);
+			return response.code();
+
+		}
+	}
+
+	/**
+	 * To finish uploading a version or new deposition not published
+	 * It sets the deposition_id and the bucket to be used
+	 *
+	 *
+	 * @param deposition_id the deposition id of the not yet published upload
+	 *            concept_rec_id = 656930
+	 * @return response code
+	 * @throws IOException
+	 * @throws MissingConceptDoiException
+	 */
+	public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
+
+		this.deposition_id = deposition_id;
+
+		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
+
+		Request request = new Request.Builder()
+			.url(urlString + "/" + deposition_id)
+			.addHeader("Authorization", "Bearer " + access_token)
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+
+			ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
+			bucket = zenodoModel.getLinks().getBucket();
+			return response.code();
+
+		}
+
+	}
+
+	private void setDepositionId(String concept_rec_id) throws IOException, MissingConceptDoiException {
+
+		ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class);
+
+		for (ZenodoModel zm : zenodoModelList) {
+			if (zm.getConceptrecid().equals(concept_rec_id)) {
+				deposition_id = zm.getId();
+				return;
+			}
+		}
+
+		throw new MissingConceptDoiException("The concept record id specified was missing in the list of depositions");
+
+	}
+
+	private String getPrevDepositions() throws IOException {
+		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
+
+		Request request = new Request.Builder()
+			.url(urlString)
+			.addHeader("Content-Type", "application/json") // add request headers
+			.addHeader("Authorization", "Bearer " + access_token)
+			.get()
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+
+			return response.body().string();
+
+		}
+
+	}
+
+	private String getBucket(String url) throws IOException {
+		OkHttpClient httpClient = new OkHttpClient.Builder()
+			.connectTimeout(600, TimeUnit.SECONDS)
+			.build();
+
+		Request request = new Request.Builder()
+			.url(url)
+			.addHeader("Content-Type", "application/json") // add request headers
+			.addHeader("Authorization", "Bearer " + access_token)
+			.get()
+			.build();
+
+		try (Response response = httpClient.newCall(request).execute()) {
+
+			if (!response.isSuccessful())
+				throw new IOException("Unexpected code " + response + response.body().string());
+
+			// Get response body
+			ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
+
+			return zenodoModel.getLinks().getBucket();
+
+		}
+
+	}
+
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java

@@ -0,0 +1,14 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+public class Community {
+	private String identifier;
+
+	public String getIdentifier() {
+		return identifier;
+	}
+
+	public void setIdentifier(String identifier) {
+		this.identifier = identifier;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java

@@ -0,0 +1,47 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+public class Creator {
+	private String affiliation;
+	private String name;
+	private String orcid;
+
+	public String getAffiliation() {
+		return affiliation;
+	}
+
+	public void setAffiliation(String affiliation) {
+		this.affiliation = affiliation;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getOrcid() {
+		return orcid;
+	}
+
+	public void setOrcid(String orcid) {
+		this.orcid = orcid;
+	}
+
+	public static Creator newInstance(String name, String affiliation, String orcid) {
+		Creator c = new Creator();
+		if (!(name == null)) {
+			c.name = name;
+		}
+		if (!(affiliation == null)) {
+			c.affiliation = affiliation;
+		}
+		if (!(orcid == null)) {
+			c.orcid = orcid;
+		}
+
+		return c;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java

@@ -0,0 +1,58 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.io.Serializable;
+
+import net.minidev.json.annotate.JsonIgnore;
+
+public class File implements Serializable {
+	private String checksum;
+	private String filename;
+	private long filesize;
+	private String id;
+
+	@JsonIgnore
+	// private Links links;
+
+	public String getChecksum() {
+		return checksum;
+	}
+
+	public void setChecksum(String checksum) {
+		this.checksum = checksum;
+	}
+
+	public String getFilename() {
+		return filename;
+	}
+
+	public void setFilename(String filename) {
+		this.filename = filename;
+	}
+
+	public long getFilesize() {
+		return filesize;
+	}
+
+	public void setFilesize(long filesize) {
+		this.filesize = filesize;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+//    @JsonIgnore
+//    public Links getLinks() {
+//        return links;
+//    }
+//
+//    @JsonIgnore
+//    public void setLinks(Links links) {
+//        this.links = links;
+//    }
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java

@@ -0,0 +1,23 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.io.Serializable;
+
+public class Grant implements Serializable {
+	private String id;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public static Grant newInstance(String id) {
+		Grant g = new Grant();
+		g.id = id;
+
+		return g;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java

@@ -0,0 +1,92 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.io.Serializable;
+
+public class Links implements Serializable {
+
+	private String bucket;
+
+	private String discard;
+
+	private String edit;
+	private String files;
+	private String html;
+	private String latest_draft;
+	private String latest_draft_html;
+	private String publish;
+
+	private String self;
+
+	public String getBucket() {
+		return bucket;
+	}
+
+	public void setBucket(String bucket) {
+		this.bucket = bucket;
+	}
+
+	public String getDiscard() {
+		return discard;
+	}
+
+	public void setDiscard(String discard) {
+		this.discard = discard;
+	}
+
+	public String getEdit() {
+		return edit;
+	}
+
+	public void setEdit(String edit) {
+		this.edit = edit;
+	}
+
+	public String getFiles() {
+		return files;
+	}
+
+	public void setFiles(String files) {
+		this.files = files;
+	}
+
+	public String getHtml() {
+		return html;
+	}
+
+	public void setHtml(String html) {
+		this.html = html;
+	}
+
+	public String getLatest_draft() {
+		return latest_draft;
+	}
+
+	public void setLatest_draft(String latest_draft) {
+		this.latest_draft = latest_draft;
+	}
+
+	public String getLatest_draft_html() {
+		return latest_draft_html;
+	}
+
+	public void setLatest_draft_html(String latest_draft_html) {
+		this.latest_draft_html = latest_draft_html;
+	}
+
+	public String getPublish() {
+		return publish;
+	}
+
+	public void setPublish(String publish) {
+		this.publish = publish;
+	}
+
+	public String getSelf() {
+		return self;
+	}
+
+	public void setSelf(String self) {
+		this.self = self;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java

@@ -0,0 +1,153 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class Metadata implements Serializable {
+
+	private String access_right;
+	private List<Community> communities;
+	private List<Creator> creators;
+	private String description;
+	private String doi;
+	private List<Grant> grants;
+	private List<String> keywords;
+	private String language;
+	private String license;
+	private PrereserveDoi prereserve_doi;
+	private String publication_date;
+	private List<String> references;
+	private List<RelatedIdentifier> related_identifiers;
+	private String title;
+	private String upload_type;
+	private String version;
+
+	public String getUpload_type() {
+		return upload_type;
+	}
+
+	public void setUpload_type(String upload_type) {
+		this.upload_type = upload_type;
+	}
+
+	public String getVersion() {
+		return version;
+	}
+
+	public void setVersion(String version) {
+		this.version = version;
+	}
+
+	public String getAccess_right() {
+		return access_right;
+	}
+
+	public void setAccess_right(String access_right) {
+		this.access_right = access_right;
+	}
+
+	public List<Community> getCommunities() {
+		return communities;
+	}
+
+	public void setCommunities(List<Community> communities) {
+		this.communities = communities;
+	}
+
+	public List<Creator> getCreators() {
+		return creators;
+	}
+
+	public void setCreators(List<Creator> creators) {
+		this.creators = creators;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(String description) {
+		this.description = description;
+	}
+
+	public String getDoi() {
+		return doi;
+	}
+
+	public void setDoi(String doi) {
+		this.doi = doi;
+	}
+
+	public List<Grant> getGrants() {
+		return grants;
+	}
+
+	public void setGrants(List<Grant> grants) {
+		this.grants = grants;
+	}
+
+	public List<String> getKeywords() {
+		return keywords;
+	}
+
+	public void setKeywords(List<String> keywords) {
+		this.keywords = keywords;
+	}
+
+	public String getLanguage() {
+		return language;
+	}
+
+	public void setLanguage(String language) {
+		this.language = language;
+	}
+
+	public String getLicense() {
+		return license;
+	}
+
+	public void setLicense(String license) {
+		this.license = license;
+	}
+
+	public PrereserveDoi getPrereserve_doi() {
+		return prereserve_doi;
+	}
+
+	public void setPrereserve_doi(PrereserveDoi prereserve_doi) {
+		this.prereserve_doi = prereserve_doi;
+	}
+
+	public String getPublication_date() {
+		return publication_date;
+	}
+
+	public void setPublication_date(String publication_date) {
+		this.publication_date = publication_date;
+	}
+
+	public List<String> getReferences() {
+		return references;
+	}
+
+	public void setReferences(List<String> references) {
+		this.references = references;
+	}
+
+	public List<RelatedIdentifier> getRelated_identifiers() {
+		return related_identifiers;
+	}
+
+	public void setRelated_identifiers(List<RelatedIdentifier> related_identifiers) {
+		this.related_identifiers = related_identifiers;
+	}
+
+	public String getTitle() {
+		return title;
+	}
+
+	public void setTitle(String title) {
+		this.title = title;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java

@@ -0,0 +1,25 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.io.Serializable;
+
+public class PrereserveDoi implements Serializable {
+	private String doi;
+	private String recid;
+
+	public String getDoi() {
+		return doi;
+	}
+
+	public void setDoi(String doi) {
+		this.doi = doi;
+	}
+
+	public String getRecid() {
+		return recid;
+	}
+
+	public void setRecid(String recid) {
+		this.recid = recid;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java

@@ -0,0 +1,43 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.io.Serializable;
+
+public class RelatedIdentifier implements Serializable {
+	private String identifier;
+	private String relation;
+	private String resource_type;
+	private String scheme;
+
+	public String getIdentifier() {
+		return identifier;
+	}
+
+	public void setIdentifier(String identifier) {
+		this.identifier = identifier;
+	}
+
+	public String getRelation() {
+		return relation;
+	}
+
+	public void setRelation(String relation) {
+		this.relation = relation;
+	}
+
+	public String getResource_type() {
+		return resource_type;
+	}
+
+	public void setResource_type(String resource_type) {
+		this.resource_type = resource_type;
+	}
+
+	public String getScheme() {
+		return scheme;
+	}
+
+	public void setScheme(String scheme) {
+		this.scheme = scheme;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java

@@ -0,0 +1,118 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class ZenodoModel implements Serializable {
+
+	private String conceptrecid;
+	private String created;
+
+	private List<File> files;
+	private String id;
+	private Links links;
+	private Metadata metadata;
+	private String modified;
+	private String owner;
+	private String record_id;
+	private String state;
+	private boolean submitted;
+	private String title;
+
+	public String getConceptrecid() {
+		return conceptrecid;
+	}
+
+	public void setConceptrecid(String conceptrecid) {
+		this.conceptrecid = conceptrecid;
+	}
+
+	public String getCreated() {
+		return created;
+	}
+
+	public void setCreated(String created) {
+		this.created = created;
+	}
+
+	public List<File> getFiles() {
+		return files;
+	}
+
+	public void setFiles(List<File> files) {
+		this.files = files;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public Links getLinks() {
+		return links;
+	}
+
+	public void setLinks(Links links) {
+		this.links = links;
+	}
+
+	public Metadata getMetadata() {
+		return metadata;
+	}
+
+	public void setMetadata(Metadata metadata) {
+		this.metadata = metadata;
+	}
+
+	public String getModified() {
+		return modified;
+	}
+
+	public void setModified(String modified) {
+		this.modified = modified;
+	}
+
+	public String getOwner() {
+		return owner;
+	}
+
+	public void setOwner(String owner) {
+		this.owner = owner;
+	}
+
+	public String getRecord_id() {
+		return record_id;
+	}
+
+	public void setRecord_id(String record_id) {
+		this.record_id = record_id;
+	}
+
+	public String getState() {
+		return state;
+	}
+
+	public void setState(String state) {
+		this.state = state;
+	}
+
+	public boolean isSubmitted() {
+		return submitted;
+	}
+
+	public void setSubmitted(boolean submitted) {
+		this.submitted = submitted;
+	}
+
+	public String getTitle() {
+		return title;
+	}
+
+	public void setTitle(String title) {
+		this.title = title;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java

@@ -0,0 +1,7 @@
+
+package eu.dnetlib.dhp.common.api.zenodo;
+
+import java.util.ArrayList;
+
+public class ZenodoModelList extends ArrayList<ZenodoModel> {
+}

dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java

@@ -1,9 +1,10 @@
 
-package eu.dnetlib.dhp.oa.dedup;
+package eu.dnetlib.dhp.oa.merge;
 
 import java.text.Normalizer;
 import java.util.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -32,27 +33,33 @@ public class AuthorMerger {
 
 	}
 
-	public static List<Author> mergeAuthor(final List<Author> a, final List<Author> b) {
+	public static List<Author> mergeAuthor(final List<Author> a, final List<Author> b, Double threshold) {
 		int pa = countAuthorsPids(a);
 		int pb = countAuthorsPids(b);
 		List<Author> base, enrich;
 		int sa = authorsSize(a);
 		int sb = authorsSize(b);
 
-		if (pa == pb) {
-			base = sa > sb ? a : b;
-			enrich = sa > sb ? b : a;
-		} else {
+		if (sa == sb) {
 			base = pa > pb ? a : b;
 			enrich = pa > pb ? b : a;
+		} else {
+			base = sa > sb ? a : b;
+			enrich = sa > sb ? b : a;
 		}
-		enrichPidFromList(base, enrich);
+		enrichPidFromList(base, enrich, threshold);
 		return base;
 	}
 
-	private static void enrichPidFromList(List<Author> base, List<Author> enrich) {
+	public static List<Author> mergeAuthor(final List<Author> a, final List<Author> b) {
+		return mergeAuthor(a, b, THRESHOLD);
+	}
+
+	private static void enrichPidFromList(List<Author> base, List<Author> enrich, Double threshold) {
 		if (base == null || enrich == null)
 			return;
+
+		// <pidComparableString, Author> (if an Author has more than 1 pid, it appears 2 times in the list)
 		final Map<String, Author> basePidAuthorMap = base
 			.stream()
 			.filter(a -> a.getPid() != null && a.getPid().size() > 0)
@@ -63,6 +70,7 @@ public class AuthorMerger {
 					.map(p -> new Tuple2<>(pidToComparableString(p), a)))
 			.collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1));
 
+		// <pid, Author> (list of pid that are missing in the other list)
 		final List<Tuple2<StructuredProperty, Author>> pidToEnrich = enrich
 			.stream()
 			.filter(a -> a.getPid() != null && a.getPid().size() > 0)
@@ -83,10 +91,10 @@ public class AuthorMerger {
 						.max(Comparator.comparing(Tuple2::_1));
 
 					if (simAuthor.isPresent()) {
-						double th = THRESHOLD;
+						double th = threshold;
 						// increase the threshold if the surname is too short
 						if (simAuthor.get()._2().getSurname() != null
-							&& simAuthor.get()._2().getSurname().length() <= 3)
+							&& simAuthor.get()._2().getSurname().length() <= 3 && threshold > 0.0)
 							th = 0.99;
 
 						if (simAuthor.get()._1() > th) {
@@ -94,7 +102,13 @@ public class AuthorMerger {
 							if (r.getPid() == null) {
 								r.setPid(new ArrayList<>());
 							}
-							r.getPid().add(a._1());
+
+							// TERRIBLE HACK but for some reason when we create and Array with Arrays.asList,
+							// it creates of fixed size, and the add method raise UnsupportedOperationException at
+							// java.util.AbstractList.add
+							final List<StructuredProperty> tmp = new ArrayList<>(r.getPid());
+							tmp.add(a._1());
+							r.setPid(tmp);
 						}
 					}
 				});
@@ -150,7 +164,7 @@ public class AuthorMerger {
 	}
 
 	private static String normalize(final String s) {
-		return nfd(s)
+		String[] normalized = nfd(s)
 			.toLowerCase()
 			// do not compact the regexes in a single expression, would cause StackOverflowError
 			// in case
@@ -160,7 +174,12 @@ public class AuthorMerger {
 			.replaceAll("(\\p{Punct})+", " ")
 			.replaceAll("(\\d)+", " ")
 			.replaceAll("(\\n)+", " ")
-			.trim();
+			.trim()
+			.split(" ");
+
+		Arrays.sort(normalized);
+
+		return String.join(" ", normalized);
 	}
 
 	private static String nfd(final String s) {

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/OafMapperUtils.java

@@ -1,19 +1,57 @@
 
-package eu.dnetlib.dhp.oa.graph.raw.common;
+package eu.dnetlib.dhp.schema.oaf;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
-import eu.dnetlib.dhp.schema.oaf.*;
+import com.google.common.base.Joiner;
+
+import eu.dnetlib.dhp.schema.common.ModelSupport;
 import eu.dnetlib.dhp.utils.DHPUtils;
 
 public class OafMapperUtils {
 
+	public static Oaf merge(final Oaf o1, final Oaf o2) {
+		if (ModelSupport.isSubClass(o1, OafEntity.class)) {
+			if (ModelSupport.isSubClass(o1, Result.class)) {
+
+				return mergeResults((Result) o1, (Result) o2);
+			} else if (ModelSupport.isSubClass(o1, Datasource.class)) {
+				((Datasource) o1).mergeFrom((Datasource) o2);
+			} else if (ModelSupport.isSubClass(o1, Organization.class)) {
+				((Organization) o1).mergeFrom((Organization) o2);
+			} else if (ModelSupport.isSubClass(o1, Project.class)) {
+				((Project) o1).mergeFrom((Project) o2);
+			} else {
+				throw new RuntimeException("invalid OafEntity subtype:" + o1.getClass().getCanonicalName());
+			}
+		} else if (ModelSupport.isSubClass(o1, Relation.class)) {
+			((Relation) o1).mergeFrom((Relation) o2);
+		} else {
+			throw new RuntimeException("invalid Oaf type:" + o1.getClass().getCanonicalName());
+		}
+		return o1;
+	}
+
+	public static Result mergeResults(Result r1, Result r2) {
+		if (new ResultTypeComparator().compare(r1, r2) < 0) {
+			r1.mergeFrom(r2);
+			return r1;
+		} else {
+			r2.mergeFrom(r1);
+			return r2;
+		}
+	}
+
 	public static KeyValue keyValue(final String k, final String v) {
 		final KeyValue kv = new KeyValue();
 		kv.setKey(k);
@@ -49,6 +87,7 @@ public class OafMapperUtils {
 			.stream(values)
 			.map(v -> field(v, info))
 			.filter(Objects::nonNull)
+			.filter(distinctByKey(f -> f.getValue()))
 			.collect(Collectors.toList());
 	}
 
@@ -57,6 +96,7 @@ public class OafMapperUtils {
 			.stream()
 			.map(v -> field(v, info))
 			.filter(Objects::nonNull)
+			.filter(distinctByKey(f -> f.getValue()))
 			.collect(Collectors.toList());
 	}
 
@@ -89,7 +129,9 @@ public class OafMapperUtils {
 	}
 
 	public static StructuredProperty structuredProperty(
-		final String value, final Qualifier qualifier, final DataInfo dataInfo) {
+		final String value,
+		final Qualifier qualifier,
+		final DataInfo dataInfo) {
 		if (value == null) {
 			return null;
 		}
@@ -137,6 +179,28 @@ public class OafMapperUtils {
 		return p;
 	}
 
+	public static Journal journal(
+		final String name,
+		final String issnPrinted,
+		final String issnOnline,
+		final String issnLinking,
+		final DataInfo dataInfo) {
+
+		return hasIssn(issnPrinted, issnOnline, issnLinking) ? journal(
+			name,
+			issnPrinted,
+			issnOnline,
+			issnLinking,
+			null,
+			null,
+			null,
+			null,
+			null,
+			null,
+			null,
+			dataInfo) : null;
+	}
+
 	public static Journal journal(
 		final String name,
 		final String issnPrinted,
@@ -151,10 +215,7 @@ public class OafMapperUtils {
 		final String conferencedate,
 		final DataInfo dataInfo) {
 
-		if (StringUtils.isNotBlank(name)
-			|| StringUtils.isNotBlank(issnPrinted)
-			|| StringUtils.isNotBlank(issnOnline)
-			|| StringUtils.isNotBlank(issnLinking)) {
+		if (StringUtils.isNotBlank(name) || hasIssn(issnPrinted, issnOnline, issnLinking)) {
 			final Journal j = new Journal();
 			j.setName(name);
 			j.setIssnPrinted(issnPrinted);
@@ -174,6 +235,12 @@ public class OafMapperUtils {
 		}
 	}
 
+	private static boolean hasIssn(String issnPrinted, String issnOnline, String issnLinking) {
+		return StringUtils.isNotBlank(issnPrinted)
+			|| StringUtils.isNotBlank(issnOnline)
+			|| StringUtils.isNotBlank(issnLinking);
+	}
+
 	public static DataInfo dataInfo(
 		final Boolean deletedbyinference,
 		final String inferenceprovenance,
@@ -192,8 +259,12 @@ public class OafMapperUtils {
 	}
 
 	public static String createOpenaireId(
-		final int prefix, final String originalId, final boolean to_md5) {
-		if (to_md5) {
+		final int prefix,
+		final String originalId,
+		final boolean to_md5) {
+		if (StringUtils.isBlank(originalId)) {
+			return null;
+		} else if (to_md5) {
 			final String nsPrefix = StringUtils.substringBefore(originalId, "::");
 			final String rest = StringUtils.substringAfter(originalId, "::");
 			return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(rest));
@@ -203,7 +274,9 @@ public class OafMapperUtils {
 	}
 
 	public static String createOpenaireId(
-		final String type, final String originalId, final boolean to_md5) {
+		final String type,
+		final String originalId,
+		final boolean to_md5) {
 		switch (type) {
 			case "datasource":
 				return createOpenaireId(10, originalId, to_md5);
@@ -221,4 +294,10 @@ public class OafMapperUtils {
 	public static String asString(final Object o) {
 		return o == null ? "" : o.toString();
 	}
+
+	public static <T> Predicate<T> distinctByKey(
+		final Function<? super T, ?> keyExtractor) {
+		final Map<Object, Boolean> seen = new ConcurrentHashMap<>();
+		return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
+	}
 }

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/ResultTypeComparator.java

@@ -0,0 +1,49 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.util.Comparator;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+
+public class ResultTypeComparator implements Comparator<Result> {
+
+	@Override
+	public int compare(Result left, Result right) {
+
+		if (left == null && right == null)
+			return 0;
+		if (left == null)
+			return 1;
+		if (right == null)
+			return -1;
+
+		String lClass = left.getResulttype().getClassid();
+		String rClass = right.getResulttype().getClassid();
+
+		if (lClass.equals(rClass))
+			return 0;
+
+		if (lClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
+			return -1;
+		if (rClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
+			return 1;
+
+		if (lClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
+			return -1;
+		if (rClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
+			return 1;
+
+		if (lClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
+			return -1;
+		if (rClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
+			return 1;
+
+		if (lClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
+			return -1;
+		if (rClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
+			return 1;
+
+		// Else (but unlikely), lexicographical ordering will do.
+		return lClass.compareTo(rClass);
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java

@@ -5,6 +5,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
+import java.util.List;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -15,9 +16,15 @@ import org.apache.commons.codec.binary.Hex;
 import com.jayway.jsonpath.JsonPath;
 
 import net.minidev.json.JSONArray;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
 
 public class DHPUtils {
 
+	public static Seq<String> toSeq(List<String> list) {
+		return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq();
+	}
+
 	public static String md5(final String s) {
 		try {
 			final MessageDigest md = MessageDigest.getInstance("MD5");

dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java

@@ -1,15 +1,22 @@
 
 package eu.dnetlib.dhp.utils;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.cxf.endpoint.Client;
+import org.apache.cxf.frontend.ClientProxy;
 import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
+import org.apache.cxf.transport.http.HTTPConduit;
+import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
 
 public class ISLookupClientFactory {
 
-	private static final Log log = LogFactory.getLog(ISLookupClientFactory.class);
+	private static final Logger log = LoggerFactory.getLogger(ISLookupClientFactory.class);
+
+	private static int requestTimeout = 60000 * 10;
+	private static int connectTimeout = 60000 * 10;
 
 	public static ISLookUpService getLookUpService(final String isLookupUrl) {
 		return getServiceStub(ISLookUpService.class, isLookupUrl);
@@ -21,6 +28,28 @@ public class ISLookupClientFactory {
 		final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
 		jaxWsProxyFactory.setServiceClass(clazz);
 		jaxWsProxyFactory.setAddress(endpoint);
-		return (T) jaxWsProxyFactory.create();
+
+		final T service = (T) jaxWsProxyFactory.create();
+
+		Client client = ClientProxy.getClient(service);
+		if (client != null) {
+			HTTPConduit conduit = (HTTPConduit) client.getConduit();
+			HTTPClientPolicy policy = new HTTPClientPolicy();
+
+			log
+				.info(
+					String
+						.format(
+							"setting connectTimeout to %s, requestTimeout to %s for service %s",
+							connectTimeout,
+							requestTimeout,
+							clazz.getCanonicalName()));
+
+			policy.setConnectionTimeout(connectTimeout);
+			policy.setReceiveTimeout(requestTimeout);
+			conduit.setClient(policy);
+		}
+
+		return service;
 	}
 }

dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java

@@ -5,6 +5,8 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
+import org.apache.commons.lang3.StringUtils;
+
 import net.sf.saxon.expr.XPathContext;
 import net.sf.saxon.om.Sequence;
 import net.sf.saxon.trans.XPathException;
@@ -19,6 +21,8 @@ public class NormalizeDate extends AbstractExtensionFunction {
 
 	private static final String normalizeOutFormat = "yyyy-MM-dd'T'hh:mm:ss'Z'";
 
+	public static final String BLANK = "";
+
 	@Override
 	public String getName() {
 		return "normalizeDate";
@@ -27,10 +31,10 @@ public class NormalizeDate extends AbstractExtensionFunction {
 	@Override
 	public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
 		if (arguments == null | arguments.length == 0) {
-			return new StringValue("");
+			return new StringValue(BLANK);
 		}
 		String s = arguments[0].head().getStringValue();
-		return new StringValue(_year(s));
+		return new StringValue(_normalizeDate(s));
 	}
 
 	@Override
@@ -55,8 +59,8 @@ public class NormalizeDate extends AbstractExtensionFunction {
 		return SequenceType.SINGLE_STRING;
 	}
 
-	private String _year(String s) {
-		final String date = s != null ? s.trim() : "";
+	private String _normalizeDate(String s) {
+		final String date = StringUtils.isNotBlank(s) ? s.trim() : BLANK;
 
 		for (String format : normalizeDateFormats) {
 			try {
@@ -66,6 +70,6 @@ public class NormalizeDate extends AbstractExtensionFunction {
 			} catch (ParseException e) {
 			}
 		}
-		return "";
+		return BLANK;
 	}
 }

dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java

@@ -0,0 +1,109 @@
+
+package eu.dnetlib.dhp.common.api;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+@Disabled
+public class ZenodoAPIClientTest {
+
+	private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
+	private final String ACCESS_TOKEN = "";
+
+	private final String CONCEPT_REC_ID = "657113";
+
+	private final String depositionId = "674915";
+
+	@Test
+	public void testUploadOldDeposition() throws IOException, MissingConceptDoiException {
+		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
+			ACCESS_TOKEN);
+		Assertions.assertEquals(200, client.uploadOpenDeposition(depositionId));
+
+		File file = new File(getClass()
+			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
+			.getPath());
+
+		InputStream is = new FileInputStream(file);
+
+		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
+
+		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
+
+		Assertions.assertEquals(200, client.sendMretadata(metadata));
+
+		Assertions.assertEquals(202, client.publish());
+
+	}
+
+	@Test
+	public void testNewDeposition() throws IOException {
+
+		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
+			ACCESS_TOKEN);
+		Assertions.assertEquals(201, client.newDeposition());
+
+		File file = new File(getClass()
+			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
+			.getPath());
+
+		InputStream is = new FileInputStream(file);
+
+		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
+
+		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
+
+		Assertions.assertEquals(200, client.sendMretadata(metadata));
+
+		Assertions.assertEquals(202, client.publish());
+
+	}
+
+	@Test
+	public void testNewVersionNewName() throws IOException, MissingConceptDoiException {
+
+		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
+			ACCESS_TOKEN);
+
+		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
+
+		File file = new File(getClass()
+			.getResource("/eu/dnetlib/dhp/common/api/newVersion")
+			.getPath());
+
+		InputStream is = new FileInputStream(file);
+
+		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length()));
+
+		Assertions.assertEquals(202, client.publish());
+
+	}
+
+	@Test
+	public void testNewVersionOldName() throws IOException, MissingConceptDoiException {
+
+		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
+			ACCESS_TOKEN);
+
+		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
+
+		File file = new File(getClass()
+			.getResource("/eu/dnetlib/dhp/common/api/newVersion2")
+			.getPath());
+
+		InputStream is = new FileInputStream(file);
+
+		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length()));
+
+		Assertions.assertEquals(202, client.publish());
+
+	}
+
+}

dhp-common/src/test/java/eu/dnetlib/dhp/oa/merge/AuthorMergerTest.java

@@ -0,0 +1,100 @@
+
+package eu.dnetlib.dhp.oa.merge;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.schema.oaf.Author;
+import eu.dnetlib.dhp.schema.oaf.Publication;
+import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
+import eu.dnetlib.pace.util.MapDocumentUtil;
+import scala.Tuple2;
+
+public class AuthorMergerTest {
+
+	private String publicationsBasePath;
+
+	private List<List<Author>> authors;
+
+	@BeforeEach
+	public void setUp() throws Exception {
+
+		publicationsBasePath = Paths
+			.get(AuthorMergerTest.class.getResource("/eu/dnetlib/dhp/oa/merge").toURI())
+			.toFile()
+			.getAbsolutePath();
+
+		authors = readSample(publicationsBasePath + "/publications_with_authors.json", Publication.class)
+			.stream()
+			.map(p -> p._2().getAuthor())
+			.collect(Collectors.toList());
+
+	}
+
+	@Test
+	public void mergeTest() { // used in the dedup: threshold set to 0.95
+
+		for (List<Author> authors1 : authors) {
+			System.out.println("List " + (authors.indexOf(authors1) + 1));
+			for (Author author : authors1) {
+				System.out.println(authorToString(author));
+			}
+		}
+
+		List<Author> merge = AuthorMerger.merge(authors);
+
+		System.out.println("Merge ");
+		for (Author author : merge) {
+			System.out.println(authorToString(author));
+		}
+
+		Assertions.assertEquals(7, merge.size());
+
+	}
+
+	public <T> List<Tuple2<String, T>> readSample(String path, Class<T> clazz) {
+		List<Tuple2<String, T>> res = new ArrayList<>();
+		BufferedReader reader;
+		try {
+			reader = new BufferedReader(new FileReader(path));
+			String line = reader.readLine();
+			while (line != null) {
+				res
+					.add(
+						new Tuple2<>(
+							MapDocumentUtil.getJPathString("$.id", line),
+							new ObjectMapper().readValue(line, clazz)));
+				// read next line
+				line = reader.readLine();
+			}
+			reader.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+		return res;
+	}
+
+	public String authorToString(Author a) {
+
+		String print = "Fullname = ";
+		print += a.getFullname() + " pid = [";
+		if (a.getPid() != null)
+			for (StructuredProperty sp : a.getPid()) {
+				print += sp.toComparableString() + " ";
+			}
+		print += "]";
+		return print;
+	}
+}

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/metadata.json

@@ -0,0 +1 @@
+{"metadata":{"access_right":"open","communities":[{"identifier":"openaire-research-graph"}],"creators":[{"affiliation":"ISTI - CNR","name":"Bardi, Alessia","orcid":"0000-0002-1112-1292"},{"affiliation":"eifl", "name":"Kuchma, Iryna"},{"affiliation":"BIH", "name":"Brobov, Evgeny"},{"affiliation":"GIDIF RBM", "name":"Truccolo, Ivana"},{"affiliation":"unesp", "name":"Monteiro, Elizabete"},{"affiliation":"und", "name":"Casalegno, Carlotta"},{"affiliation":"CARL ABRC", "name":"Clary, Erin"},{"affiliation":"The University of Edimburgh", "name":"Romanowski, Andrew"},{"affiliation":"ISTI - CNR", "name":"Pavone, Gina"},{"affiliation":"ISTI - CNR", "name":"Artini, Michele"},{"affiliation":"ISTI - CNR","name":"Atzori, Claudio","orcid":"0000-0001-9613-6639"},{"affiliation":"University of Bielefeld","name":"Bäcker, Amelie","orcid":"0000-0001-6015-2063"},{"affiliation":"ISTI - CNR","name":"Baglioni, Miriam","orcid":"0000-0002-2273-9004"},{"affiliation":"University of Bielefeld","name":"Czerniak, Andreas","orcid":"0000-0003-3883-4169"},{"affiliation":"ISTI - CNR","name":"De Bonis, Michele"},{"affiliation":"Athena Research and Innovation Centre","name":"Dimitropoulos, Harry"},{"affiliation":"Athena Research and Innovation Centre","name":"Foufoulas, Ioannis"},{"affiliation":"University of Warsaw","name":"Horst, Marek"},{"affiliation":"Athena Research and Innovation Centre","name":"Iatropoulou, Katerina"},{"affiliation":"University of Warsaw","name":"Jacewicz, Przemyslaw"},{"affiliation":"Athena Research and Innovation Centre","name":"Kokogiannaki, Argiro", "orcid":"0000-0002-3880-0244"},{"affiliation":"ISTI - CNR","name":"La Bruzzo, Sandro","orcid":"0000-0003-2855-1245"},{"affiliation":"ISTI - CNR","name":"Lazzeri, Emma"},{"affiliation":"University of Bielefeld","name":"Löhden, Aenne"},{"affiliation":"ISTI - CNR","name":"Manghi, Paolo","orcid":"0000-0001-7291-3210"},{"affiliation":"ISTI - CNR","name":"Mannocci, Andrea","orcid":"0000-0002-5193-7851"},{"affiliation":"Athena Research and Innovation Center","name":"Manola, Natalia"},{"affiliation":"ISTI - CNR","name":"Ottonello, Enrico"},{"affiliation":"University of Bielefeld","name":"Shirrwagen, Jochen"}],"description":"\\u003cp\\u003eThis dump provides access to the metadata records of publications, research data, software and projects that may be relevant to the Corona Virus Disease (COVID-19) fight. The dump contains records of the OpenAIRE COVID-19 Gateway (https://covid-19.openaire.eu/), identified via full-text mining and inference techniques applied to the OpenAIRE Research Graph (https://explore.openaire.eu/). The Graph is one of the largest Open Access collections of metadata records and links between publications, datasets, software, projects, funders, and organizations, aggregating 12,000+ scientific data sources world-wide, among which the Covid-19 data sources Zenodo COVID-19 Community, WHO (World Health Organization), BIP! FInder for COVID-19, Protein Data Bank, Dimensions, scienceOpen, and RSNA. \\u003cp\\u003eThe dump consists of a gzip file containing one json per line. Each json is compliant to the schema available at https://doi.org/10.5281/zenodo.3974226\\u003c/p\\u003e ","title":"OpenAIRE Covid-19 publications, datasets, software and projects metadata.","upload_type":"dataset","version":"1.0"}}

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newDeposition

@@ -0,0 +1 @@
+This is a test for a new deposition

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newVersion

@@ -0,0 +1 @@
+This is a test for a new version of an old deposition

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newVersion2

@@ -0,0 +1,2 @@
+This is a test for a new version of an old deposition. This should replace the other new version. I expect to have only two
+files in the deposition

dhp-schemas/README.md

@@ -1,11 +0,0 @@
-Description of the project
---------------------------
-This project defines **object schemas** of the OpenAIRE main entities and the relationships that intercur among them. 
-Namely it defines the model for 
-  
--  **research product (result)** which subclasses in publication, dataset, other research product, software
--  **data source** object describing the data provider (institutional repository, aggregators, cris systems)
--  **organization** research bodies managing a data source or participating to a research project
--  **project** research project
- 
-Te serialization of such objects (data store files) are used to pass data between workflow nodes in the processing pipeline.

dhp-schemas/pom.xml

@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>eu.dnetlib.dhp</groupId>
-        <artifactId>dhp</artifactId>
-        <version>1.2.4-SNAPSHOT</version>
-        <relativePath>../</relativePath>
-    </parent>
-
-    <artifactId>dhp-schemas</artifactId>
-    <packaging>jar</packaging>
-
-    <description>This module contains common schema classes meant to be used across the dnet-hadoop submodules</description>
-
-    <dependencies>
-
-        <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-        </dependency>
-
-    </dependencies>
-
-
-</project>

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/action/AtomicAction.java

@@ -1,40 +0,0 @@
-
-package eu.dnetlib.dhp.schema.action;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-
-import eu.dnetlib.dhp.schema.oaf.Oaf;
-
-@JsonDeserialize(using = AtomicActionDeserializer.class)
-public class AtomicAction<T extends Oaf> implements Serializable {
-
-	private Class<T> clazz;
-
-	private T payload;
-
-	public AtomicAction() {
-	}
-
-	public AtomicAction(Class<T> clazz, T payload) {
-		this.clazz = clazz;
-		this.payload = payload;
-	}
-
-	public Class<T> getClazz() {
-		return clazz;
-	}
-
-	public void setClazz(Class<T> clazz) {
-		this.clazz = clazz;
-	}
-
-	public T getPayload() {
-		return payload;
-	}
-
-	public void setPayload(T payload) {
-		this.payload = payload;
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/action/AtomicActionDeserializer.java

@@ -1,32 +0,0 @@
-
-package eu.dnetlib.dhp.schema.action;
-
-import java.io.IOException;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.dhp.schema.oaf.Oaf;
-
-public class AtomicActionDeserializer extends JsonDeserializer {
-
-	@Override
-	public Object deserialize(JsonParser jp, DeserializationContext ctxt)
-		throws IOException {
-		JsonNode node = jp.getCodec().readTree(jp);
-		String classTag = node.get("clazz").asText();
-		JsonNode payload = node.get("payload");
-		ObjectMapper mapper = new ObjectMapper();
-
-		try {
-			final Class<?> clazz = Class.forName(classTag);
-			return new AtomicAction(clazz, (Oaf) mapper.readValue(payload.toString(), clazz));
-		} catch (ClassNotFoundException e) {
-			throw new IOException(e);
-		}
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/EntityType.java

@@ -1,21 +0,0 @@
-
-package eu.dnetlib.dhp.schema.common;
-
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-
-/** Actual entity types in the Graph */
-public enum EntityType {
-	publication, dataset, otherresearchproduct, software, datasource, organization, project;
-
-	/**
-	 * Resolves the EntityType, given the relative class name
-	 *
-	 * @param clazz the given class name
-	 * @param <T> actual OafEntity subclass
-	 * @return the EntityType associated to the given class
-	 */
-	public static <T extends OafEntity> EntityType fromClass(Class<T> clazz) {
-
-		return EntityType.valueOf(clazz.getSimpleName().toLowerCase());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/LicenseComparator.java

@@ -1,69 +0,0 @@
-
-package eu.dnetlib.dhp.schema.common;
-
-import java.util.Comparator;
-
-import eu.dnetlib.dhp.schema.oaf.Qualifier;
-
-public class LicenseComparator implements Comparator<Qualifier> {
-
-	@Override
-	public int compare(Qualifier left, Qualifier right) {
-
-		if (left == null && right == null)
-			return 0;
-		if (left == null)
-			return 1;
-		if (right == null)
-			return -1;
-
-		String lClass = left.getClassid();
-		String rClass = right.getClassid();
-
-		if (lClass.equals(rClass))
-			return 0;
-
-		if (lClass.equals("OPEN SOURCE"))
-			return -1;
-		if (rClass.equals("OPEN SOURCE"))
-			return 1;
-
-		if (lClass.equals("OPEN"))
-			return -1;
-		if (rClass.equals("OPEN"))
-			return 1;
-
-		if (lClass.equals("6MONTHS"))
-			return -1;
-		if (rClass.equals("6MONTHS"))
-			return 1;
-
-		if (lClass.equals("12MONTHS"))
-			return -1;
-		if (rClass.equals("12MONTHS"))
-			return 1;
-
-		if (lClass.equals("EMBARGO"))
-			return -1;
-		if (rClass.equals("EMBARGO"))
-			return 1;
-
-		if (lClass.equals("RESTRICTED"))
-			return -1;
-		if (rClass.equals("RESTRICTED"))
-			return 1;
-
-		if (lClass.equals("CLOSED"))
-			return -1;
-		if (rClass.equals("CLOSED"))
-			return 1;
-
-		if (lClass.equals("UNKNOWN"))
-			return -1;
-		if (rClass.equals("UNKNOWN"))
-			return 1;
-
-		// Else (but unlikely), lexicographical ordering will do.
-		return lClass.compareTo(rClass);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/MainEntityType.java

@@ -1,7 +0,0 @@
-
-package eu.dnetlib.dhp.schema.common;
-
-/** Main entity types in the Graph */
-public enum MainEntityType {
-	result, datasource, organization, project
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java

@@ -1,125 +0,0 @@
-
-package eu.dnetlib.dhp.schema.common;
-
-import java.security.Key;
-
-import eu.dnetlib.dhp.schema.oaf.DataInfo;
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.Qualifier;
-
-public class ModelConstants {
-
-	public static final String DNET_RESULT_TYPOLOGIES = "dnet:result_typologies";
-	public static final String DNET_PUBLICATION_RESOURCE = "dnet:publication_resource";
-	public static final String DNET_ACCESS_MODES = "dnet:access_modes";
-	public static final String DNET_LANGUAGES = "dnet:languages";
-	public static final String DNET_PID_TYPES = "dnet:pid_types";
-	public static final String DNET_DATA_CITE_DATE = "dnet:dataCite_date";
-	public static final String DNET_DATA_CITE_RESOURCE = "dnet:dataCite_resource";
-	public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions";
-	public static final String DNET_COUNTRY_TYPE = "dnet:countries";
-	public static final String DNET_REVIEW_LEVELS = "dnet:review_levels";
-
-	public static final String SYSIMPORT_CROSSWALK_REPOSITORY = "sysimport:crosswalk:repository";
-	public static final String SYSIMPORT_CROSSWALK_ENTITYREGISTRY = "sysimport:crosswalk:entityregistry";
-	public static final String USER_CLAIM = "user:claim";
-
-	public static final String DATASET_RESULTTYPE_CLASSID = "dataset";
-	public static final String PUBLICATION_RESULTTYPE_CLASSID = "publication";
-	public static final String SOFTWARE_RESULTTYPE_CLASSID = "software";
-	public static final String ORP_RESULTTYPE_CLASSID = "other";
-
-	public static final String RESULT_RESULT = "resultResult";
-	/**
-	 * @deprecated Use {@link ModelConstants#RELATIONSHIP} instead.
-	 */
-	@Deprecated
-	public static final String PUBLICATION_DATASET = "publicationDataset";
-	public static final String IS_RELATED_TO = "isRelatedTo";
-	public static final String SUPPLEMENT = "supplement";
-	public static final String IS_SUPPLEMENT_TO = "isSupplementTo";
-	public static final String IS_SUPPLEMENTED_BY = "isSupplementedBy";
-	public static final String PART = "part";
-	public static final String IS_PART_OF = "IsPartOf";
-	public static final String HAS_PARTS = "HasParts";
-	public static final String RELATIONSHIP = "relationship";
-	public static final String CITATION = "citation";
-	public static final String CITES = "cites";
-	public static final String IS_CITED_BY = "IsCitedBy";
-	public static final String REVIEW = "review";
-	public static final String REVIEWS = "reviews";
-	public static final String IS_REVIEWED_BY = "IsReviewedBy";
-
-	public static final String RESULT_PROJECT = "resultProject";
-	public static final String OUTCOME = "outcome";
-	public static final String IS_PRODUCED_BY = "isProducedBy";
-	public static final String PRODUCES = "produces";
-
-	public static final String DATASOURCE_ORGANIZATION = "datasourceOrganization";
-	public static final String PROVISION = "provision";
-	public static final String IS_PROVIDED_BY = "isProvidedBy";
-	public static final String PROVIDES = "provides";
-
-	public static final String PROJECT_ORGANIZATION = "projectOrganization";
-	public static final String PARTICIPATION = "participation";
-	public static final String HAS_PARTICIPANT = "hasParticipant";
-	public static final String IS_PARTICIPANT = "isParticipant";
-
-	public static final String RESULT_ORGANIZATION = "resultOrganization";
-	public static final String AFFILIATION = "affiliation";
-	public static final String IS_AUTHOR_INSTITUTION_OF = "isAuthorInstitutionOf";
-	public static final String HAS_AUTHOR_INSTITUTION = "hasAuthorInstitution";
-
-	public static final String MERGES = "merges";
-
-	public static final String UNKNOWN = "UNKNOWN";
-	public static final String NOT_AVAILABLE = "not available";
-
-	public static final Qualifier PUBLICATION_DEFAULT_RESULTTYPE = qualifier(
-		PUBLICATION_RESULTTYPE_CLASSID, PUBLICATION_RESULTTYPE_CLASSID,
-		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
-
-	public static final Qualifier DATASET_DEFAULT_RESULTTYPE = qualifier(
-		DATASET_RESULTTYPE_CLASSID, DATASET_RESULTTYPE_CLASSID,
-		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
-
-	public static final Qualifier SOFTWARE_DEFAULT_RESULTTYPE = qualifier(
-		SOFTWARE_RESULTTYPE_CLASSID, SOFTWARE_RESULTTYPE_CLASSID,
-		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
-
-	public static final Qualifier ORP_DEFAULT_RESULTTYPE = qualifier(
-		ORP_RESULTTYPE_CLASSID, ORP_RESULTTYPE_CLASSID,
-		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
-
-	public static final Qualifier REPOSITORY_PROVENANCE_ACTIONS = qualifier(
-		SYSIMPORT_CROSSWALK_REPOSITORY, SYSIMPORT_CROSSWALK_REPOSITORY,
-		DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS);
-
-	public static final Qualifier ENTITYREGISTRY_PROVENANCE_ACTION = qualifier(
-		SYSIMPORT_CROSSWALK_ENTITYREGISTRY, SYSIMPORT_CROSSWALK_ENTITYREGISTRY,
-		DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS);
-
-	public static final KeyValue UNKNOWN_REPOSITORY = keyValue(
-		"10|openaire____::55045bd2a65019fd8e6741a755395c8c", "Unknown Repository");
-
-	private static Qualifier qualifier(
-		final String classid,
-		final String classname,
-		final String schemeid,
-		final String schemename) {
-		final Qualifier q = new Qualifier();
-		q.setClassid(classid);
-		q.setClassname(classname);
-		q.setSchemeid(schemeid);
-		q.setSchemename(schemename);
-		return q;
-	}
-
-	private static KeyValue keyValue(String key, String value) {
-		KeyValue kv = new KeyValue();
-		kv.setKey(key);
-		kv.setValue(value);
-		kv.setDataInfo(new DataInfo());
-		return kv;
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java

@@ -1,467 +0,0 @@
-
-package eu.dnetlib.dhp.schema.common;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.function.Function;
-
-import org.apache.commons.lang3.StringUtils;
-
-import com.google.common.collect.Maps;
-
-import eu.dnetlib.dhp.schema.oaf.*;
-
-/** Oaf model utility methods. */
-public class ModelSupport {
-
-	/** Defines the mapping between the actual entity type and the main entity type */
-	private static Map<EntityType, MainEntityType> entityMapping = Maps.newHashMap();
-
-	static {
-		entityMapping.put(EntityType.publication, MainEntityType.result);
-		entityMapping.put(EntityType.dataset, MainEntityType.result);
-		entityMapping.put(EntityType.otherresearchproduct, MainEntityType.result);
-		entityMapping.put(EntityType.software, MainEntityType.result);
-		entityMapping.put(EntityType.datasource, MainEntityType.datasource);
-		entityMapping.put(EntityType.organization, MainEntityType.organization);
-		entityMapping.put(EntityType.project, MainEntityType.project);
-	}
-
-	/**
-	 * Defines the mapping between the actual entity types and the relative classes implementing them
-	 */
-	public static final Map<EntityType, Class> entityTypes = Maps.newHashMap();
-
-	static {
-		entityTypes.put(EntityType.datasource, Datasource.class);
-		entityTypes.put(EntityType.organization, Organization.class);
-		entityTypes.put(EntityType.project, Project.class);
-		entityTypes.put(EntityType.dataset, Dataset.class);
-		entityTypes.put(EntityType.otherresearchproduct, OtherResearchProduct.class);
-		entityTypes.put(EntityType.software, Software.class);
-		entityTypes.put(EntityType.publication, Publication.class);
-	}
-
-	public static final Map<String, Class> oafTypes = Maps.newHashMap();
-
-	static {
-		oafTypes.put("datasource", Datasource.class);
-		oafTypes.put("organization", Organization.class);
-		oafTypes.put("project", Project.class);
-		oafTypes.put("dataset", Dataset.class);
-		oafTypes.put("otherresearchproduct", OtherResearchProduct.class);
-		oafTypes.put("software", Software.class);
-		oafTypes.put("publication", Publication.class);
-		oafTypes.put("relation", Relation.class);
-	}
-
-	public static final Map<Class, String> idPrefixMap = Maps.newHashMap();
-
-	static {
-		idPrefixMap.put(Datasource.class, "10");
-		idPrefixMap.put(Organization.class, "20");
-		idPrefixMap.put(Project.class, "40");
-		idPrefixMap.put(Dataset.class, "50");
-		idPrefixMap.put(OtherResearchProduct.class, "50");
-		idPrefixMap.put(Software.class, "50");
-		idPrefixMap.put(Publication.class, "50");
-	}
-
-	public static final Map<String, String> entityIdPrefix = Maps.newHashMap();
-
-	static {
-		entityIdPrefix.put("datasource", "10");
-		entityIdPrefix.put("organization", "20");
-		entityIdPrefix.put("project", "40");
-		entityIdPrefix.put("result", "50");
-	}
-
-	public static final Map<String, RelationInverse> relationInverseMap = Maps.newHashMap();
-
-	static {
-		relationInverseMap
-			.put(
-				"personResult_authorship_isAuthorOf", new RelationInverse()
-					.setRelation("isAuthorOf")
-					.setInverse("hasAuthor")
-					.setRelType("personResult")
-					.setSubReltype("authorship"));
-		relationInverseMap
-			.put(
-				"personResult_authorship_hasAuthor", new RelationInverse()
-					.setInverse("isAuthorOf")
-					.setRelation("hasAuthor")
-					.setRelType("personResult")
-					.setSubReltype("authorship"));
-		relationInverseMap
-			.put(
-				"projectOrganization_participation_isParticipant", new RelationInverse()
-					.setRelation("isParticipant")
-					.setInverse("hasParticipant")
-					.setRelType("projectOrganization")
-					.setSubReltype("participation"));
-		relationInverseMap
-			.put(
-				"projectOrganization_participation_hasParticipant", new RelationInverse()
-					.setInverse("isParticipant")
-					.setRelation("hasParticipant")
-					.setRelType("projectOrganization")
-					.setSubReltype("participation"));
-		relationInverseMap
-			.put(
-				"resultOrganization_affiliation_hasAuthorInstitution", new RelationInverse()
-					.setRelation("hasAuthorInstitution")
-					.setInverse("isAuthorInstitutionOf")
-					.setRelType("resultOrganization")
-					.setSubReltype("affiliation"));
-		relationInverseMap
-			.put(
-				"resultOrganization_affiliation_isAuthorInstitutionOf", new RelationInverse()
-					.setInverse("hasAuthorInstitution")
-					.setRelation("isAuthorInstitutionOf")
-					.setRelType("resultOrganization")
-					.setSubReltype("affiliation"));
-		relationInverseMap
-			.put(
-				"organizationOrganization_dedup_merges", new RelationInverse()
-					.setRelation("merges")
-					.setInverse("isMergedIn")
-					.setRelType("organizationOrganization")
-					.setSubReltype("dedup"));
-		relationInverseMap
-			.put(
-				"organizationOrganization_dedup_isMergedIn", new RelationInverse()
-					.setInverse("merges")
-					.setRelation("isMergedIn")
-					.setRelType("organizationOrganization")
-					.setSubReltype("dedup"));
-		relationInverseMap
-			.put(
-				"organizationOrganization_dedupSimilarity_isSimilarTo", new RelationInverse()
-					.setInverse("isSimilarTo")
-					.setRelation("isSimilarTo")
-					.setRelType("organizationOrganization")
-					.setSubReltype("dedupSimilarity"));
-
-		relationInverseMap
-			.put(
-				"resultProject_outcome_isProducedBy", new RelationInverse()
-					.setRelation("isProducedBy")
-					.setInverse("produces")
-					.setRelType("resultProject")
-					.setSubReltype("outcome"));
-		relationInverseMap
-			.put(
-				"resultProject_outcome_produces", new RelationInverse()
-					.setInverse("isProducedBy")
-					.setRelation("produces")
-					.setRelType("resultProject")
-					.setSubReltype("outcome"));
-		relationInverseMap
-			.put(
-				"projectPerson_contactPerson_isContact", new RelationInverse()
-					.setRelation("isContact")
-					.setInverse("hasContact")
-					.setRelType("projectPerson")
-					.setSubReltype("contactPerson"));
-		relationInverseMap
-			.put(
-				"projectPerson_contactPerson_hasContact", new RelationInverse()
-					.setInverse("isContact")
-					.setRelation("hasContact")
-					.setRelType("personPerson")
-					.setSubReltype("coAuthorship"));
-		relationInverseMap
-			.put(
-				"personPerson_coAuthorship_isCoauthorOf", new RelationInverse()
-					.setInverse("isCoAuthorOf")
-					.setRelation("isCoAuthorOf")
-					.setRelType("personPerson")
-					.setSubReltype("coAuthorship"));
-		relationInverseMap
-			.put(
-				"personPerson_dedup_merges", new RelationInverse()
-					.setInverse("isMergedIn")
-					.setRelation("merges")
-					.setRelType("personPerson")
-					.setSubReltype("dedup"));
-		relationInverseMap
-			.put(
-				"personPerson_dedup_isMergedIn", new RelationInverse()
-					.setInverse("merges")
-					.setRelation("isMergedIn")
-					.setRelType("personPerson")
-					.setSubReltype("dedup"));
-		relationInverseMap
-			.put(
-				"personPerson_dedupSimilarity_isSimilarTo", new RelationInverse()
-					.setInverse("isSimilarTo")
-					.setRelation("isSimilarTo")
-					.setRelType("personPerson")
-					.setSubReltype("dedupSimilarity"));
-		relationInverseMap
-			.put(
-				"datasourceOrganization_provision_isProvidedBy", new RelationInverse()
-					.setInverse("provides")
-					.setRelation("isProvidedBy")
-					.setRelType("datasourceOrganization")
-					.setSubReltype("provision"));
-		relationInverseMap
-			.put(
-				"datasourceOrganization_provision_provides", new RelationInverse()
-					.setInverse("isProvidedBy")
-					.setRelation("provides")
-					.setRelType("datasourceOrganization")
-					.setSubReltype("provision"));
-		relationInverseMap
-			.put(
-				"resultResult_similarity_hasAmongTopNSimilarDocuments", new RelationInverse()
-					.setInverse("isAmongTopNSimilarDocuments")
-					.setRelation("hasAmongTopNSimilarDocuments")
-					.setRelType("resultResult")
-					.setSubReltype("similarity"));
-		relationInverseMap
-			.put(
-				"resultResult_similarity_isAmongTopNSimilarDocuments", new RelationInverse()
-					.setInverse("hasAmongTopNSimilarDocuments")
-					.setRelation("isAmongTopNSimilarDocuments")
-					.setRelType("resultResult")
-					.setSubReltype("similarity"));
-		relationInverseMap
-			.put(
-				"resultResult_relationship_isRelatedTo", new RelationInverse()
-					.setInverse("isRelatedTo")
-					.setRelation("isRelatedTo")
-					.setRelType("resultResult")
-					.setSubReltype("relationship"));
-		relationInverseMap
-			.put(
-				"resultResult_similarity_isAmongTopNSimilarDocuments", new RelationInverse()
-					.setInverse("hasAmongTopNSimilarDocuments")
-					.setRelation("isAmongTopNSimilarDocuments")
-					.setRelType("resultResult")
-					.setSubReltype("similarity"));
-		relationInverseMap
-			.put(
-				"resultResult_supplement_isSupplementTo", new RelationInverse()
-					.setInverse("isSupplementedBy")
-					.setRelation("isSupplementTo")
-					.setRelType("resultResult")
-					.setSubReltype("supplement"));
-		relationInverseMap
-			.put(
-				"resultResult_supplement_isSupplementedBy", new RelationInverse()
-					.setInverse("isSupplementTo")
-					.setRelation("isSupplementedBy")
-					.setRelType("resultResult")
-					.setSubReltype("supplement"));
-		relationInverseMap
-			.put(
-				"resultResult_part_isPartOf", new RelationInverse()
-					.setInverse("hasPart")
-					.setRelation("isPartOf")
-					.setRelType("resultResult")
-					.setSubReltype("part"));
-		relationInverseMap
-			.put(
-				"resultResult_part_hasPart", new RelationInverse()
-					.setInverse("isPartOf")
-					.setRelation("hasPart")
-					.setRelType("resultResult")
-					.setSubReltype("part"));
-		relationInverseMap
-			.put(
-				"resultResult_dedup_merges", new RelationInverse()
-					.setInverse("isMergedIn")
-					.setRelation("merges")
-					.setRelType("resultResult")
-					.setSubReltype("dedup"));
-		relationInverseMap
-			.put(
-				"resultResult_dedup_isMergedIn", new RelationInverse()
-					.setInverse("merges")
-					.setRelation("isMergedIn")
-					.setRelType("resultResult")
-					.setSubReltype("dedup"));
-		relationInverseMap
-			.put(
-				"resultResult_dedupSimilarity_isSimilarTo", new RelationInverse()
-					.setInverse("isSimilarTo")
-					.setRelation("isSimilarTo")
-					.setRelType("resultResult")
-					.setSubReltype("dedupSimilarity"));
-
-	}
-
-	private static final String schemeTemplate = "dnet:%s_%s_relations";
-
-	private ModelSupport() {
-	}
-
-	public static <E extends OafEntity> String getIdPrefix(Class<E> clazz) {
-		return idPrefixMap.get(clazz);
-	}
-
-	/**
-	 * Checks subclass-superclass relationship.
-	 *
-	 * @param subClazzObject Subclass object instance
-	 * @param superClazzObject Superclass object instance
-	 * @param <X> Subclass type
-	 * @param <Y> Superclass type
-	 * @return True if X is a subclass of Y
-	 */
-	public static <X extends Oaf, Y extends Oaf> Boolean isSubClass(
-		X subClazzObject, Y superClazzObject) {
-		return isSubClass(subClazzObject.getClass(), superClazzObject.getClass());
-	}
-
-	/**
-	 * Checks subclass-superclass relationship.
-	 *
-	 * @param subClazzObject Subclass object instance
-	 * @param superClazz Superclass class
-	 * @param <X> Subclass type
-	 * @param <Y> Superclass type
-	 * @return True if X is a subclass of Y
-	 */
-	public static <X extends Oaf, Y extends Oaf> Boolean isSubClass(
-		X subClazzObject, Class<Y> superClazz) {
-		return isSubClass(subClazzObject.getClass(), superClazz);
-	}
-
-	/**
-	 * Checks subclass-superclass relationship.
-	 *
-	 * @param subClazz Subclass class
-	 * @param superClazz Superclass class
-	 * @param <X> Subclass type
-	 * @param <Y> Superclass type
-	 * @return True if X is a subclass of Y
-	 */
-	public static <X extends Oaf, Y extends Oaf> Boolean isSubClass(
-		Class<X> subClazz, Class<Y> superClazz) {
-		return superClazz.isAssignableFrom(subClazz);
-	}
-
-	/**
-	 * Lists all the OAF model classes
-	 *
-	 * @param <T>
-	 * @return
-	 */
-	public static <T extends Oaf> Class<T>[] getOafModelClasses() {
-		return new Class[] {
-			Author.class,
-			Context.class,
-			Country.class,
-			DataInfo.class,
-			Dataset.class,
-			Datasource.class,
-			ExternalReference.class,
-			ExtraInfo.class,
-			Field.class,
-			GeoLocation.class,
-			Instance.class,
-			Journal.class,
-			KeyValue.class,
-			Oaf.class,
-			OafEntity.class,
-			OAIProvenance.class,
-			Organization.class,
-			OriginDescription.class,
-			OtherResearchProduct.class,
-			Project.class,
-			Publication.class,
-			Qualifier.class,
-			Relation.class,
-			Result.class,
-			Software.class,
-			StructuredProperty.class
-		};
-	}
-
-	public static String getMainType(final EntityType type) {
-		return entityMapping.get(type).name();
-	}
-
-	public static boolean isResult(EntityType type) {
-		return MainEntityType.result.name().equals(getMainType(type));
-	}
-
-	public static String getScheme(final String sourceType, final String targetType) {
-		return String
-			.format(
-				schemeTemplate,
-				entityMapping.get(EntityType.valueOf(sourceType)).name(),
-				entityMapping.get(EntityType.valueOf(targetType)).name());
-	}
-
-	public static <T extends Oaf> String tableIdentifier(String dbName, String tableName) {
-
-		checkArgument(StringUtils.isNotBlank(dbName), "DB name cannot be empty");
-		checkArgument(StringUtils.isNotBlank(tableName), "table name cannot be empty");
-
-		return String.format("%s.%s", dbName, tableName);
-	}
-
-	public static <T extends Oaf> String tableIdentifier(String dbName, Class<T> clazz) {
-
-		checkArgument(Objects.nonNull(clazz), "clazz is needed to derive the table name, thus cannot be null");
-
-		return tableIdentifier(dbName, clazz.getSimpleName().toLowerCase());
-	}
-
-	public static <T extends Oaf> Function<T, String> idFn() {
-		return x -> {
-			if (isSubClass(x, Relation.class)) {
-				return idFnForRelation(x);
-			}
-			return idFnForOafEntity(x);
-		};
-	}
-
-	private static <T extends Oaf> String idFnForRelation(T t) {
-		Relation r = (Relation) t;
-		return Optional
-			.ofNullable(r.getSource())
-			.map(
-				source -> Optional
-					.ofNullable(r.getTarget())
-					.map(
-						target -> Optional
-							.ofNullable(r.getRelType())
-							.map(
-								relType -> Optional
-									.ofNullable(r.getSubRelType())
-									.map(
-										subRelType -> Optional
-											.ofNullable(r.getRelClass())
-											.map(
-												relClass -> String
-													.join(
-														source,
-														target,
-														relType,
-														subRelType,
-														relClass))
-											.orElse(
-												String
-													.join(
-														source,
-														target,
-														relType,
-														subRelType)))
-									.orElse(String.join(source, target, relType)))
-							.orElse(String.join(source, target)))
-					.orElse(source))
-			.orElse(null);
-	}
-
-	private static <T extends Oaf> String idFnForOafEntity(T t) {
-		return ((OafEntity) t).getId();
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/RelationInverse.java

@@ -1,46 +0,0 @@
-
-package eu.dnetlib.dhp.schema.common;
-
-public class RelationInverse {
-	private String relation;
-	private String inverse;
-	private String relType;
-	private String subReltype;
-
-	public String getRelType() {
-		return relType;
-	}
-
-	public RelationInverse setRelType(String relType) {
-		this.relType = relType;
-		return this;
-	}
-
-	public String getSubReltype() {
-		return subReltype;
-	}
-
-	public RelationInverse setSubReltype(String subReltype) {
-		this.subReltype = subReltype;
-		return this;
-	}
-
-	public String getRelation() {
-		return relation;
-	}
-
-	public RelationInverse setRelation(String relation) {
-		this.relation = relation;
-		return this;
-	}
-
-	public String getInverse() {
-		return inverse;
-	}
-
-	public RelationInverse setInverse(String inverse) {
-		this.inverse = inverse;
-		return this;
-	}
-
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Author.java

@@ -1,89 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.*;
-
-public class Author implements Serializable {
-
-	private String fullname;
-
-	private String name;
-
-	private String surname;
-
-	private Integer rank;
-
-	private List<StructuredProperty> pid;
-
-	private List<Field<String>> affiliation;
-
-	public String getFullname() {
-		return fullname;
-	}
-
-	public void setFullname(String fullname) {
-		this.fullname = fullname;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getSurname() {
-		return surname;
-	}
-
-	public void setSurname(String surname) {
-		this.surname = surname;
-	}
-
-	public Integer getRank() {
-		return rank;
-	}
-
-	public void setRank(Integer rank) {
-		this.rank = rank;
-	}
-
-	public List<StructuredProperty> getPid() {
-		return pid;
-	}
-
-	public void setPid(List<StructuredProperty> pid) {
-		this.pid = pid;
-	}
-
-	public List<Field<String>> getAffiliation() {
-		return affiliation;
-	}
-
-	public void setAffiliation(List<Field<String>> affiliation) {
-		this.affiliation = affiliation;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		Author author = (Author) o;
-		return Objects.equals(fullname, author.fullname)
-			&& Objects.equals(name, author.name)
-			&& Objects.equals(surname, author.surname)
-			&& Objects.equals(rank, author.rank)
-			&& Objects.equals(pid, author.pid)
-			&& Objects.equals(affiliation, author.affiliation);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(fullname, name, surname, rank, pid, affiliation);
-	}
-
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Context.java

@@ -1,46 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Context implements Serializable {
-	private String id;
-
-	private List<DataInfo> dataInfo;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public List<DataInfo> getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(List<DataInfo> dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	@Override
-	public int hashCode() {
-		return id == null ? 0 : id.hashCode();
-	}
-
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj)
-			return true;
-		if (obj == null)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-
-		Context other = (Context) obj;
-
-		return id.equals(other.getId());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Country.java

@@ -1,34 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.util.Objects;
-
-public class Country extends Qualifier {
-
-	private DataInfo dataInfo;
-
-	public DataInfo getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(DataInfo dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		if (!super.equals(o))
-			return false;
-		Country country = (Country) o;
-		return Objects.equals(dataInfo, country.dataInfo);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(super.hashCode(), dataInfo);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/DataInfo.java

@@ -1,85 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class DataInfo implements Serializable {
-
-	private Boolean invisible = false;
-	private Boolean inferred;
-	private Boolean deletedbyinference = false;
-	private String trust;
-	private String inferenceprovenance;
-	private Qualifier provenanceaction;
-
-	public Boolean getInvisible() {
-		return invisible;
-	}
-
-	public void setInvisible(Boolean invisible) {
-		this.invisible = invisible;
-	}
-
-	public Boolean getInferred() {
-		return inferred;
-	}
-
-	public void setInferred(Boolean inferred) {
-		this.inferred = inferred;
-	}
-
-	public Boolean getDeletedbyinference() {
-		return deletedbyinference;
-	}
-
-	public void setDeletedbyinference(Boolean deletedbyinference) {
-		this.deletedbyinference = deletedbyinference;
-	}
-
-	public String getTrust() {
-		return trust;
-	}
-
-	public void setTrust(String trust) {
-		this.trust = trust;
-	}
-
-	public String getInferenceprovenance() {
-		return inferenceprovenance;
-	}
-
-	public void setInferenceprovenance(String inferenceprovenance) {
-		this.inferenceprovenance = inferenceprovenance;
-	}
-
-	public Qualifier getProvenanceaction() {
-		return provenanceaction;
-	}
-
-	public void setProvenanceaction(Qualifier provenanceaction) {
-		this.provenanceaction = provenanceaction;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		DataInfo dataInfo = (DataInfo) o;
-		return Objects.equals(invisible, dataInfo.invisible)
-			&& Objects.equals(inferred, dataInfo.inferred)
-			&& Objects.equals(deletedbyinference, dataInfo.deletedbyinference)
-			&& Objects.equals(trust, dataInfo.trust)
-			&& Objects.equals(inferenceprovenance, dataInfo.inferenceprovenance)
-			&& Objects.equals(provenanceaction, dataInfo.provenanceaction);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects
-			.hash(
-				invisible, inferred, deletedbyinference, trust, inferenceprovenance, provenanceaction);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java

@@ -1,116 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-
-public class Dataset extends Result implements Serializable {
-
-	private Field<String> storagedate;
-
-	// candidate for removal
-	private Field<String> device;
-
-	private Field<String> size;
-
-	private Field<String> version;
-
-	private Field<String> lastmetadataupdate;
-
-	private Field<String> metadataversionnumber;
-
-	private List<GeoLocation> geolocation;
-
-	public Dataset() {
-		setResulttype(ModelConstants.DATASET_DEFAULT_RESULTTYPE);
-	}
-
-	public Field<String> getStoragedate() {
-		return storagedate;
-	}
-
-	public void setStoragedate(Field<String> storagedate) {
-		this.storagedate = storagedate;
-	}
-
-	public Field<String> getDevice() {
-		return device;
-	}
-
-	public void setDevice(Field<String> device) {
-		this.device = device;
-	}
-
-	public Field<String> getSize() {
-		return size;
-	}
-
-	public void setSize(Field<String> size) {
-		this.size = size;
-	}
-
-	public Field<String> getVersion() {
-		return version;
-	}
-
-	public void setVersion(Field<String> version) {
-		this.version = version;
-	}
-
-	public Field<String> getLastmetadataupdate() {
-		return lastmetadataupdate;
-	}
-
-	public void setLastmetadataupdate(Field<String> lastmetadataupdate) {
-		this.lastmetadataupdate = lastmetadataupdate;
-	}
-
-	public Field<String> getMetadataversionnumber() {
-		return metadataversionnumber;
-	}
-
-	public void setMetadataversionnumber(Field<String> metadataversionnumber) {
-		this.metadataversionnumber = metadataversionnumber;
-	}
-
-	public List<GeoLocation> getGeolocation() {
-		return geolocation;
-	}
-
-	public void setGeolocation(List<GeoLocation> geolocation) {
-		this.geolocation = geolocation;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!Dataset.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		final Dataset d = (Dataset) e;
-
-		storagedate = d.getStoragedate() != null && compareTrust(this, e) < 0 ? d.getStoragedate() : storagedate;
-
-		device = d.getDevice() != null && compareTrust(this, e) < 0 ? d.getDevice() : device;
-
-		size = d.getSize() != null && compareTrust(this, e) < 0 ? d.getSize() : size;
-
-		version = d.getVersion() != null && compareTrust(this, e) < 0 ? d.getVersion() : version;
-
-		lastmetadataupdate = d.getLastmetadataupdate() != null && compareTrust(this, e) < 0
-			? d.getLastmetadataupdate()
-			: lastmetadataupdate;
-
-		metadataversionnumber = d.getMetadataversionnumber() != null && compareTrust(this, e) < 0
-			? d.getMetadataversionnumber()
-			: metadataversionnumber;
-
-		geolocation = mergeLists(geolocation, d.getGeolocation());
-
-		mergeOAFDataInfo(d);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java

@@ -1,472 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Datasource extends OafEntity implements Serializable {
-
-	private Qualifier datasourcetype;
-
-	private Qualifier openairecompatibility;
-
-	private Field<String> officialname;
-
-	private Field<String> englishname;
-
-	private Field<String> websiteurl;
-
-	private Field<String> logourl;
-
-	private Field<String> contactemail;
-
-	private Field<String> namespaceprefix;
-
-	private Field<String> latitude;
-
-	private Field<String> longitude;
-
-	private Field<String> dateofvalidation;
-
-	private Field<String> description;
-
-	private List<StructuredProperty> subjects;
-
-	// opendoar specific fields (od*)
-	private Field<String> odnumberofitems;
-
-	private Field<String> odnumberofitemsdate;
-
-	private Field<String> odpolicies;
-
-	private List<Field<String>> odlanguages;
-
-	private List<Field<String>> odcontenttypes;
-
-	private List<Field<String>> accessinfopackage;
-
-	// re3data fields
-	private Field<String> releasestartdate;
-
-	private Field<String> releaseenddate;
-
-	private Field<String> missionstatementurl;
-
-	private Field<Boolean> dataprovider;
-
-	private Field<Boolean> serviceprovider;
-
-	// {open, restricted or closed}
-	private Field<String> databaseaccesstype;
-
-	// {open, restricted or closed}
-	private Field<String> datauploadtype;
-
-	// {feeRequired, registration, other}
-	private Field<String> databaseaccessrestriction;
-
-	// {feeRequired, registration, other}
-	private Field<String> datauploadrestriction;
-
-	private Field<Boolean> versioning;
-
-	private Field<String> citationguidelineurl;
-
-	// {yes, no, uknown}
-	private Field<String> qualitymanagementkind;
-
-	private Field<String> pidsystems;
-
-	private Field<String> certificates;
-
-	private List<KeyValue> policies;
-
-	private Journal journal;
-
-	public Qualifier getDatasourcetype() {
-		return datasourcetype;
-	}
-
-	public void setDatasourcetype(Qualifier datasourcetype) {
-		this.datasourcetype = datasourcetype;
-	}
-
-	public Qualifier getOpenairecompatibility() {
-		return openairecompatibility;
-	}
-
-	public void setOpenairecompatibility(Qualifier openairecompatibility) {
-		this.openairecompatibility = openairecompatibility;
-	}
-
-	public Field<String> getOfficialname() {
-		return officialname;
-	}
-
-	public void setOfficialname(Field<String> officialname) {
-		this.officialname = officialname;
-	}
-
-	public Field<String> getEnglishname() {
-		return englishname;
-	}
-
-	public void setEnglishname(Field<String> englishname) {
-		this.englishname = englishname;
-	}
-
-	public Field<String> getWebsiteurl() {
-		return websiteurl;
-	}
-
-	public void setWebsiteurl(Field<String> websiteurl) {
-		this.websiteurl = websiteurl;
-	}
-
-	public Field<String> getLogourl() {
-		return logourl;
-	}
-
-	public void setLogourl(Field<String> logourl) {
-		this.logourl = logourl;
-	}
-
-	public Field<String> getContactemail() {
-		return contactemail;
-	}
-
-	public void setContactemail(Field<String> contactemail) {
-		this.contactemail = contactemail;
-	}
-
-	public Field<String> getNamespaceprefix() {
-		return namespaceprefix;
-	}
-
-	public void setNamespaceprefix(Field<String> namespaceprefix) {
-		this.namespaceprefix = namespaceprefix;
-	}
-
-	public Field<String> getLatitude() {
-		return latitude;
-	}
-
-	public void setLatitude(Field<String> latitude) {
-		this.latitude = latitude;
-	}
-
-	public Field<String> getLongitude() {
-		return longitude;
-	}
-
-	public void setLongitude(Field<String> longitude) {
-		this.longitude = longitude;
-	}
-
-	public Field<String> getDateofvalidation() {
-		return dateofvalidation;
-	}
-
-	public void setDateofvalidation(Field<String> dateofvalidation) {
-		this.dateofvalidation = dateofvalidation;
-	}
-
-	public Field<String> getDescription() {
-		return description;
-	}
-
-	public void setDescription(Field<String> description) {
-		this.description = description;
-	}
-
-	public List<StructuredProperty> getSubjects() {
-		return subjects;
-	}
-
-	public void setSubjects(List<StructuredProperty> subjects) {
-		this.subjects = subjects;
-	}
-
-	public Field<String> getOdnumberofitems() {
-		return odnumberofitems;
-	}
-
-	public void setOdnumberofitems(Field<String> odnumberofitems) {
-		this.odnumberofitems = odnumberofitems;
-	}
-
-	public Field<String> getOdnumberofitemsdate() {
-		return odnumberofitemsdate;
-	}
-
-	public void setOdnumberofitemsdate(Field<String> odnumberofitemsdate) {
-		this.odnumberofitemsdate = odnumberofitemsdate;
-	}
-
-	public Field<String> getOdpolicies() {
-		return odpolicies;
-	}
-
-	public void setOdpolicies(Field<String> odpolicies) {
-		this.odpolicies = odpolicies;
-	}
-
-	public List<Field<String>> getOdlanguages() {
-		return odlanguages;
-	}
-
-	public void setOdlanguages(List<Field<String>> odlanguages) {
-		this.odlanguages = odlanguages;
-	}
-
-	public List<Field<String>> getOdcontenttypes() {
-		return odcontenttypes;
-	}
-
-	public void setOdcontenttypes(List<Field<String>> odcontenttypes) {
-		this.odcontenttypes = odcontenttypes;
-	}
-
-	public List<Field<String>> getAccessinfopackage() {
-		return accessinfopackage;
-	}
-
-	public void setAccessinfopackage(List<Field<String>> accessinfopackage) {
-		this.accessinfopackage = accessinfopackage;
-	}
-
-	public Field<String> getReleasestartdate() {
-		return releasestartdate;
-	}
-
-	public void setReleasestartdate(Field<String> releasestartdate) {
-		this.releasestartdate = releasestartdate;
-	}
-
-	public Field<String> getReleaseenddate() {
-		return releaseenddate;
-	}
-
-	public void setReleaseenddate(Field<String> releaseenddate) {
-		this.releaseenddate = releaseenddate;
-	}
-
-	public Field<String> getMissionstatementurl() {
-		return missionstatementurl;
-	}
-
-	public void setMissionstatementurl(Field<String> missionstatementurl) {
-		this.missionstatementurl = missionstatementurl;
-	}
-
-	public Field<Boolean> getDataprovider() {
-		return dataprovider;
-	}
-
-	public void setDataprovider(Field<Boolean> dataprovider) {
-		this.dataprovider = dataprovider;
-	}
-
-	public Field<Boolean> getServiceprovider() {
-		return serviceprovider;
-	}
-
-	public void setServiceprovider(Field<Boolean> serviceprovider) {
-		this.serviceprovider = serviceprovider;
-	}
-
-	public Field<String> getDatabaseaccesstype() {
-		return databaseaccesstype;
-	}
-
-	public void setDatabaseaccesstype(Field<String> databaseaccesstype) {
-		this.databaseaccesstype = databaseaccesstype;
-	}
-
-	public Field<String> getDatauploadtype() {
-		return datauploadtype;
-	}
-
-	public void setDatauploadtype(Field<String> datauploadtype) {
-		this.datauploadtype = datauploadtype;
-	}
-
-	public Field<String> getDatabaseaccessrestriction() {
-		return databaseaccessrestriction;
-	}
-
-	public void setDatabaseaccessrestriction(Field<String> databaseaccessrestriction) {
-		this.databaseaccessrestriction = databaseaccessrestriction;
-	}
-
-	public Field<String> getDatauploadrestriction() {
-		return datauploadrestriction;
-	}
-
-	public void setDatauploadrestriction(Field<String> datauploadrestriction) {
-		this.datauploadrestriction = datauploadrestriction;
-	}
-
-	public Field<Boolean> getVersioning() {
-		return versioning;
-	}
-
-	public void setVersioning(Field<Boolean> versioning) {
-		this.versioning = versioning;
-	}
-
-	public Field<String> getCitationguidelineurl() {
-		return citationguidelineurl;
-	}
-
-	public void setCitationguidelineurl(Field<String> citationguidelineurl) {
-		this.citationguidelineurl = citationguidelineurl;
-	}
-
-	public Field<String> getQualitymanagementkind() {
-		return qualitymanagementkind;
-	}
-
-	public void setQualitymanagementkind(Field<String> qualitymanagementkind) {
-		this.qualitymanagementkind = qualitymanagementkind;
-	}
-
-	public Field<String> getPidsystems() {
-		return pidsystems;
-	}
-
-	public void setPidsystems(Field<String> pidsystems) {
-		this.pidsystems = pidsystems;
-	}
-
-	public Field<String> getCertificates() {
-		return certificates;
-	}
-
-	public void setCertificates(Field<String> certificates) {
-		this.certificates = certificates;
-	}
-
-	public List<KeyValue> getPolicies() {
-		return policies;
-	}
-
-	public void setPolicies(List<KeyValue> policies) {
-		this.policies = policies;
-	}
-
-	public Journal getJournal() {
-		return journal;
-	}
-
-	public void setJournal(Journal journal) {
-		this.journal = journal;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!Datasource.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		Datasource d = (Datasource) e;
-
-		datasourcetype = d.getDatasourcetype() != null && compareTrust(this, e) < 0
-			? d.getDatasourcetype()
-			: datasourcetype;
-		openairecompatibility = d.getOpenairecompatibility() != null && compareTrust(this, e) < 0
-			? d.getOpenairecompatibility()
-			: openairecompatibility;
-		officialname = d.getOfficialname() != null && compareTrust(this, e) < 0
-			? d.getOfficialname()
-			: officialname;
-		englishname = d.getEnglishname() != null && compareTrust(this, e) < 0 ? d.getEnglishname() : officialname;
-		websiteurl = d.getWebsiteurl() != null && compareTrust(this, e) < 0 ? d.getWebsiteurl() : websiteurl;
-		logourl = d.getLogourl() != null && compareTrust(this, e) < 0 ? d.getLogourl() : getLogourl();
-		contactemail = d.getContactemail() != null && compareTrust(this, e) < 0
-			? d.getContactemail()
-			: contactemail;
-		namespaceprefix = d.getNamespaceprefix() != null && compareTrust(this, e) < 0
-			? d.getNamespaceprefix()
-			: namespaceprefix;
-		latitude = d.getLatitude() != null && compareTrust(this, e) < 0 ? d.getLatitude() : latitude;
-		longitude = d.getLongitude() != null && compareTrust(this, e) < 0 ? d.getLongitude() : longitude;
-		dateofvalidation = d.getDateofvalidation() != null && compareTrust(this, e) < 0
-			? d.getDateofvalidation()
-			: dateofvalidation;
-		description = d.getDescription() != null && compareTrust(this, e) < 0 ? d.getDescription() : description;
-		subjects = mergeLists(subjects, d.getSubjects());
-
-		// opendoar specific fields (od*)
-		odnumberofitems = d.getOdnumberofitems() != null && compareTrust(this, e) < 0
-			? d.getOdnumberofitems()
-			: odnumberofitems;
-		odnumberofitemsdate = d.getOdnumberofitemsdate() != null && compareTrust(this, e) < 0
-			? d.getOdnumberofitemsdate()
-			: odnumberofitemsdate;
-		odpolicies = d.getOdpolicies() != null && compareTrust(this, e) < 0 ? d.getOdpolicies() : odpolicies;
-		odlanguages = mergeLists(odlanguages, d.getOdlanguages());
-		odcontenttypes = mergeLists(odcontenttypes, d.getOdcontenttypes());
-		accessinfopackage = mergeLists(accessinfopackage, d.getAccessinfopackage());
-
-		// re3data fields
-		releasestartdate = d.getReleasestartdate() != null && compareTrust(this, e) < 0
-			? d.getReleasestartdate()
-			: releasestartdate;
-		releaseenddate = d.getReleaseenddate() != null && compareTrust(this, e) < 0
-			? d.getReleaseenddate()
-			: releaseenddate;
-		missionstatementurl = d.getMissionstatementurl() != null && compareTrust(this, e) < 0
-			? d.getMissionstatementurl()
-			: missionstatementurl;
-		dataprovider = d.getDataprovider() != null && compareTrust(this, e) < 0
-			? d.getDataprovider()
-			: dataprovider;
-		serviceprovider = d.getServiceprovider() != null && compareTrust(this, e) < 0
-			? d.getServiceprovider()
-			: serviceprovider;
-
-		// {open, restricted or closed}
-		databaseaccesstype = d.getDatabaseaccesstype() != null && compareTrust(this, e) < 0
-			? d.getDatabaseaccesstype()
-			: databaseaccesstype;
-
-		// {open, restricted or closed}
-		datauploadtype = d.getDatauploadtype() != null && compareTrust(this, e) < 0
-			? d.getDatauploadtype()
-			: datauploadtype;
-
-		// {feeRequired, registration, other}
-		databaseaccessrestriction = d.getDatabaseaccessrestriction() != null && compareTrust(this, e) < 0
-			? d.getDatabaseaccessrestriction()
-			: databaseaccessrestriction;
-
-		// {feeRequired, registration, other}
-		datauploadrestriction = d.getDatauploadrestriction() != null && compareTrust(this, e) < 0
-			? d.getDatauploadrestriction()
-			: datauploadrestriction;
-
-		versioning = d.getVersioning() != null && compareTrust(this, e) < 0 ? d.getVersioning() : versioning;
-		citationguidelineurl = d.getCitationguidelineurl() != null && compareTrust(this, e) < 0
-			? d.getCitationguidelineurl()
-			: citationguidelineurl;
-
-		// {yes, no, unknown}
-		qualitymanagementkind = d.getQualitymanagementkind() != null && compareTrust(this, e) < 0
-			? d.getQualitymanagementkind()
-			: qualitymanagementkind;
-		pidsystems = d.getPidsystems() != null && compareTrust(this, e) < 0 ? d.getPidsystems() : pidsystems;
-
-		certificates = d.getCertificates() != null && compareTrust(this, e) < 0
-			? d.getCertificates()
-			: certificates;
-
-		policies = mergeLists(policies, d.getPolicies());
-
-		journal = d.getJournal() != null && compareTrust(this, e) < 0 ? d.getJournal() : journal;
-
-		mergeOAFDataInfo(e);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/ExternalReference.java

@@ -1,119 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class ExternalReference implements Serializable {
-	// source
-	private String sitename;
-
-	// title
-	private String label;
-
-	// text()
-	private String url;
-
-	// ?? not mapped yet ??
-	private String description;
-
-	// type
-	private Qualifier qualifier;
-
-	// site internal identifier
-	private String refidentifier;
-
-	// maps the oaf:reference/@query attribute
-	private String query;
-
-	// ExternalReferences might be also inferred
-	private DataInfo dataInfo;
-
-	public String getSitename() {
-		return sitename;
-	}
-
-	public void setSitename(String sitename) {
-		this.sitename = sitename;
-	}
-
-	public String getLabel() {
-		return label;
-	}
-
-	public void setLabel(String label) {
-		this.label = label;
-	}
-
-	public String getUrl() {
-		return url;
-	}
-
-	public void setUrl(String url) {
-		this.url = url;
-	}
-
-	public String getDescription() {
-		return description;
-	}
-
-	public void setDescription(String description) {
-		this.description = description;
-	}
-
-	public Qualifier getQualifier() {
-		return qualifier;
-	}
-
-	public void setQualifier(Qualifier qualifier) {
-		this.qualifier = qualifier;
-	}
-
-	public String getRefidentifier() {
-		return refidentifier;
-	}
-
-	public void setRefidentifier(String refidentifier) {
-		this.refidentifier = refidentifier;
-	}
-
-	public String getQuery() {
-		return query;
-	}
-
-	public void setQuery(String query) {
-		this.query = query;
-	}
-
-	public DataInfo getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(DataInfo dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		ExternalReference that = (ExternalReference) o;
-		return Objects.equals(sitename, that.sitename)
-			&& Objects.equals(label, that.label)
-			&& Objects.equals(url, that.url)
-			&& Objects.equals(description, that.description)
-			&& Objects.equals(qualifier, that.qualifier)
-			&& Objects.equals(refidentifier, that.refidentifier)
-			&& Objects.equals(query, that.query)
-			&& Objects.equals(dataInfo, that.dataInfo);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects
-			.hash(
-				sitename, label, url, description, qualifier, refidentifier, query, dataInfo);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/ExtraInfo.java

@@ -1,77 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class ExtraInfo implements Serializable {
-	private String name;
-
-	private String typology;
-
-	private String provenance;
-
-	private String trust;
-
-	// json containing a Citation or Statistics
-	private String value;
-
-	public String getName() {
-		return name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getTypology() {
-		return typology;
-	}
-
-	public void setTypology(String typology) {
-		this.typology = typology;
-	}
-
-	public String getProvenance() {
-		return provenance;
-	}
-
-	public void setProvenance(String provenance) {
-		this.provenance = provenance;
-	}
-
-	public String getTrust() {
-		return trust;
-	}
-
-	public void setTrust(String trust) {
-		this.trust = trust;
-	}
-
-	public String getValue() {
-		return value;
-	}
-
-	public void setValue(String value) {
-		this.value = value;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		ExtraInfo extraInfo = (ExtraInfo) o;
-		return Objects.equals(name, extraInfo.name)
-			&& Objects.equals(typology, extraInfo.typology)
-			&& Objects.equals(provenance, extraInfo.provenance)
-			&& Objects.equals(trust, extraInfo.trust)
-			&& Objects.equals(value, extraInfo.value);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(name, typology, provenance, trust, value);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Field.java

@@ -1,45 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class Field<T> implements Serializable {
-
-	private T value;
-
-	private DataInfo dataInfo;
-
-	public T getValue() {
-		return value;
-	}
-
-	public void setValue(T value) {
-		this.value = value;
-	}
-
-	public DataInfo getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(DataInfo dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	@Override
-	public int hashCode() {
-		return getValue() == null ? 0 : getValue().hashCode();
-	}
-
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj)
-			return true;
-		if (obj == null)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-		Field<T> other = (Field<T>) obj;
-		return Objects.equals(getValue(), other.getValue());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/GeoLocation.java

@@ -1,76 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-
-import org.apache.commons.lang3.StringUtils;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-public class GeoLocation implements Serializable {
-
-	private String point;
-
-	private String box;
-
-	private String place;
-
-	public String getPoint() {
-		return point;
-	}
-
-	public void setPoint(String point) {
-		this.point = point;
-	}
-
-	public String getBox() {
-		return box;
-	}
-
-	public void setBox(String box) {
-		this.box = box;
-	}
-
-	public String getPlace() {
-		return place;
-	}
-
-	public void setPlace(String place) {
-		this.place = place;
-	}
-
-	@JsonIgnore
-	public boolean isBlank() {
-		return StringUtils.isBlank(point) && StringUtils.isBlank(box) && StringUtils.isBlank(place);
-	}
-
-	public String toComparableString() {
-		return isBlank()
-			? ""
-			: String
-				.format(
-					"%s::%s%s",
-					point != null ? point.toLowerCase() : "",
-					box != null ? box.toLowerCase() : "",
-					place != null ? place.toLowerCase() : "");
-	}
-
-	@Override
-	public int hashCode() {
-		return toComparableString().hashCode();
-	}
-
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj)
-			return true;
-		if (obj == null)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-
-		GeoLocation other = (GeoLocation) obj;
-
-		return toComparableString().equals(other.toComparableString());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java

@@ -1,152 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Instance implements Serializable {
-
-	private Field<String> license;
-
-	private Qualifier accessright;
-
-	private Qualifier instancetype;
-
-	private KeyValue hostedby;
-
-	private List<String> url;
-
-	// other research products specifc
-	private String distributionlocation;
-
-	private KeyValue collectedfrom;
-
-	private Field<String> dateofacceptance;
-
-	// ( article | book ) processing charges. Defined here to cope with possible wrongly typed
-	// results
-	private Field<String> processingchargeamount;
-
-	// currency - alphabetic code describe in ISO-4217. Defined here to cope with possible wrongly
-	// typed results
-	private Field<String> processingchargecurrency;
-
-	private Qualifier refereed; // peer-review status
-
-	public Field<String> getLicense() {
-		return license;
-	}
-
-	public void setLicense(Field<String> license) {
-		this.license = license;
-	}
-
-	public Qualifier getAccessright() {
-		return accessright;
-	}
-
-	public void setAccessright(Qualifier accessright) {
-		this.accessright = accessright;
-	}
-
-	public Qualifier getInstancetype() {
-		return instancetype;
-	}
-
-	public void setInstancetype(Qualifier instancetype) {
-		this.instancetype = instancetype;
-	}
-
-	public KeyValue getHostedby() {
-		return hostedby;
-	}
-
-	public void setHostedby(KeyValue hostedby) {
-		this.hostedby = hostedby;
-	}
-
-	public List<String> getUrl() {
-		return url;
-	}
-
-	public void setUrl(List<String> url) {
-		this.url = url;
-	}
-
-	public String getDistributionlocation() {
-		return distributionlocation;
-	}
-
-	public void setDistributionlocation(String distributionlocation) {
-		this.distributionlocation = distributionlocation;
-	}
-
-	public KeyValue getCollectedfrom() {
-		return collectedfrom;
-	}
-
-	public void setCollectedfrom(KeyValue collectedfrom) {
-		this.collectedfrom = collectedfrom;
-	}
-
-	public Field<String> getDateofacceptance() {
-		return dateofacceptance;
-	}
-
-	public void setDateofacceptance(Field<String> dateofacceptance) {
-		this.dateofacceptance = dateofacceptance;
-	}
-
-	public Field<String> getProcessingchargeamount() {
-		return processingchargeamount;
-	}
-
-	public void setProcessingchargeamount(Field<String> processingchargeamount) {
-		this.processingchargeamount = processingchargeamount;
-	}
-
-	public Field<String> getProcessingchargecurrency() {
-		return processingchargecurrency;
-	}
-
-	public void setProcessingchargecurrency(Field<String> processingchargecurrency) {
-		this.processingchargecurrency = processingchargecurrency;
-	}
-
-	public Qualifier getRefereed() {
-		return refereed;
-	}
-
-	public void setRefereed(Qualifier refereed) {
-		this.refereed = refereed;
-	}
-
-	public String toComparableString() {
-		return String
-			.format(
-				"%s::%s::%s::%s",
-				hostedby != null && hostedby.getKey() != null ? hostedby.getKey().toLowerCase() : "",
-				accessright != null && accessright.getClassid() != null ? accessright.getClassid() : "",
-				instancetype != null && instancetype.getClassid() != null ? instancetype.getClassid() : "",
-				url != null ? url : "");
-	}
-
-	@Override
-	public int hashCode() {
-		return toComparableString().hashCode();
-	}
-
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj)
-			return true;
-		if (obj == null)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-
-		Instance other = (Instance) obj;
-
-		return toComparableString().equals(other.toComparableString());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Journal.java

@@ -1,167 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class Journal implements Serializable {
-
-	private String name;
-
-	private String issnPrinted;
-
-	private String issnOnline;
-
-	private String issnLinking;
-
-	private String ep;
-
-	private String iss;
-
-	private String sp;
-
-	private String vol;
-
-	private String edition;
-
-	private String conferenceplace;
-
-	private String conferencedate;
-
-	private DataInfo dataInfo;
-
-	public String getName() {
-		return name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getIssnPrinted() {
-		return issnPrinted;
-	}
-
-	public void setIssnPrinted(String issnPrinted) {
-		this.issnPrinted = issnPrinted;
-	}
-
-	public String getIssnOnline() {
-		return issnOnline;
-	}
-
-	public void setIssnOnline(String issnOnline) {
-		this.issnOnline = issnOnline;
-	}
-
-	public String getIssnLinking() {
-		return issnLinking;
-	}
-
-	public void setIssnLinking(String issnLinking) {
-		this.issnLinking = issnLinking;
-	}
-
-	public String getEp() {
-		return ep;
-	}
-
-	public void setEp(String ep) {
-		this.ep = ep;
-	}
-
-	public String getIss() {
-		return iss;
-	}
-
-	public void setIss(String iss) {
-		this.iss = iss;
-	}
-
-	public String getSp() {
-		return sp;
-	}
-
-	public void setSp(String sp) {
-		this.sp = sp;
-	}
-
-	public String getVol() {
-		return vol;
-	}
-
-	public void setVol(String vol) {
-		this.vol = vol;
-	}
-
-	public String getEdition() {
-		return edition;
-	}
-
-	public void setEdition(String edition) {
-		this.edition = edition;
-	}
-
-	public String getConferenceplace() {
-		return conferenceplace;
-	}
-
-	public void setConferenceplace(String conferenceplace) {
-		this.conferenceplace = conferenceplace;
-	}
-
-	public String getConferencedate() {
-		return conferencedate;
-	}
-
-	public void setConferencedate(String conferencedate) {
-		this.conferencedate = conferencedate;
-	}
-
-	public DataInfo getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(DataInfo dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		Journal journal = (Journal) o;
-		return Objects.equals(name, journal.name)
-			&& Objects.equals(issnPrinted, journal.issnPrinted)
-			&& Objects.equals(issnOnline, journal.issnOnline)
-			&& Objects.equals(issnLinking, journal.issnLinking)
-			&& Objects.equals(ep, journal.ep)
-			&& Objects.equals(iss, journal.iss)
-			&& Objects.equals(sp, journal.sp)
-			&& Objects.equals(vol, journal.vol)
-			&& Objects.equals(edition, journal.edition)
-			&& Objects.equals(conferenceplace, journal.conferenceplace)
-			&& Objects.equals(conferencedate, journal.conferencedate)
-			&& Objects.equals(dataInfo, journal.dataInfo);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects
-			.hash(
-				name,
-				issnPrinted,
-				issnOnline,
-				issnLinking,
-				ep,
-				iss,
-				sp,
-				vol,
-				edition,
-				conferenceplace,
-				conferencedate,
-				dataInfo);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/KeyValue.java

@@ -1,74 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-
-import org.apache.commons.lang3.StringUtils;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-public class KeyValue implements Serializable {
-
-	private String key;
-
-	private String value;
-
-	private DataInfo dataInfo;
-
-	public String getKey() {
-		return key;
-	}
-
-	public void setKey(String key) {
-		this.key = key;
-	}
-
-	public String getValue() {
-		return value;
-	}
-
-	public void setValue(String value) {
-		this.value = value;
-	}
-
-	public DataInfo getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(DataInfo dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	public String toComparableString() {
-		return isBlank()
-			? ""
-			: String
-				.format(
-					"%s::%s",
-					key != null ? key.toLowerCase() : "", value != null ? value.toLowerCase() : "");
-	}
-
-	@JsonIgnore
-	public boolean isBlank() {
-		return StringUtils.isBlank(key) && StringUtils.isBlank(value);
-	}
-
-	@Override
-	public int hashCode() {
-		return toComparableString().hashCode();
-	}
-
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj)
-			return true;
-		if (obj == null)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-
-		KeyValue other = (KeyValue) obj;
-
-		return toComparableString().equals(other.toComparableString());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java

@@ -1,59 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.util.List;
-
-import com.google.common.base.Objects;
-
-/**
- * Represent a measure, must be further described by a system available resource providing name and descriptions.
- */
-public class Measure {
-
-	/**
-	 * Unique measure identifier.
-	 */
-	private String id;
-
-	/**
-	 * List of units associated with this measure. KeyValue provides a pair to store the laber (key) and the value, plus
-	 * common provenance information.
-	 */
-	private List<KeyValue> unit;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public List<KeyValue> getUnit() {
-		return unit;
-	}
-
-	public void setUnit(List<KeyValue> unit) {
-		this.unit = unit;
-	}
-
-	public void mergeFrom(Measure m) {
-		// TODO
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		Measure measure = (Measure) o;
-		return Objects.equal(id, measure.id) &&
-			Objects.equal(unit, measure.unit);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hashCode(id, unit);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OAIProvenance.java

@@ -1,33 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class OAIProvenance implements Serializable {
-
-	private OriginDescription originDescription;
-
-	public OriginDescription getOriginDescription() {
-		return originDescription;
-	}
-
-	public void setOriginDescription(OriginDescription originDescription) {
-		this.originDescription = originDescription;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		OAIProvenance that = (OAIProvenance) o;
-		return Objects.equals(originDescription, that.originDescription);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(originDescription);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java

@@ -1,73 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Objects;
-
-public abstract class Oaf implements Serializable {
-
-	/**
-	 * The list of datasource id/name pairs providing this relationship.
-	 */
-	protected List<KeyValue> collectedfrom;
-
-	private DataInfo dataInfo;
-
-	private Long lastupdatetimestamp;
-
-	public List<KeyValue> getCollectedfrom() {
-		return collectedfrom;
-	}
-
-	public void setCollectedfrom(List<KeyValue> collectedfrom) {
-		this.collectedfrom = collectedfrom;
-	}
-
-	public DataInfo getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(DataInfo dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	public Long getLastupdatetimestamp() {
-		return lastupdatetimestamp;
-	}
-
-	public void setLastupdatetimestamp(Long lastupdatetimestamp) {
-		this.lastupdatetimestamp = lastupdatetimestamp;
-	}
-
-	public void mergeOAFDataInfo(Oaf e) {
-		if (e.getDataInfo() != null && compareTrust(this, e) < 0)
-			dataInfo = e.getDataInfo();
-	}
-
-	protected String extractTrust(Oaf e) {
-		if (e == null || e.getDataInfo() == null || e.getDataInfo().getTrust() == null)
-			return "0.0";
-		return e.getDataInfo().getTrust();
-	}
-
-	protected int compareTrust(Oaf a, Oaf b) {
-		return extractTrust(a).compareTo(extractTrust(b));
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		Oaf oaf = (Oaf) o;
-		return Objects.equals(dataInfo, oaf.dataInfo)
-			&& Objects.equals(lastupdatetimestamp, oaf.lastupdatetimestamp);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(dataInfo, lastupdatetimestamp);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java

@@ -1,130 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.*;
-import java.util.stream.Collectors;
-
-public abstract class OafEntity extends Oaf implements Serializable {
-
-	private String id;
-
-	private List<String> originalId;
-
-	private List<StructuredProperty> pid;
-
-	private String dateofcollection;
-
-	private String dateoftransformation;
-
-	private List<ExtraInfo> extraInfo;
-
-	private OAIProvenance oaiprovenance;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public List<String> getOriginalId() {
-		return originalId;
-	}
-
-	public void setOriginalId(List<String> originalId) {
-		this.originalId = originalId;
-	}
-
-	public List<StructuredProperty> getPid() {
-		return pid;
-	}
-
-	public void setPid(List<StructuredProperty> pid) {
-		this.pid = pid;
-	}
-
-	public String getDateofcollection() {
-		return dateofcollection;
-	}
-
-	public void setDateofcollection(String dateofcollection) {
-		this.dateofcollection = dateofcollection;
-	}
-
-	public String getDateoftransformation() {
-		return dateoftransformation;
-	}
-
-	public void setDateoftransformation(String dateoftransformation) {
-		this.dateoftransformation = dateoftransformation;
-	}
-
-	public List<ExtraInfo> getExtraInfo() {
-		return extraInfo;
-	}
-
-	public void setExtraInfo(List<ExtraInfo> extraInfo) {
-		this.extraInfo = extraInfo;
-	}
-
-	public OAIProvenance getOaiprovenance() {
-		return oaiprovenance;
-	}
-
-	public void setOaiprovenance(OAIProvenance oaiprovenance) {
-		this.oaiprovenance = oaiprovenance;
-	}
-
-	public void mergeFrom(OafEntity e) {
-
-		if (e == null)
-			return;
-
-		originalId = mergeLists(originalId, e.getOriginalId());
-
-		collectedfrom = mergeLists(collectedfrom, e.getCollectedfrom());
-
-		pid = mergeLists(pid, e.getPid());
-
-		if (e.getDateofcollection() != null && compareTrust(this, e) < 0)
-			dateofcollection = e.getDateofcollection();
-
-		if (e.getDateoftransformation() != null && compareTrust(this, e) < 0)
-			dateoftransformation = e.getDateoftransformation();
-
-		extraInfo = mergeLists(extraInfo, e.getExtraInfo());
-
-		if (e.getOaiprovenance() != null && compareTrust(this, e) < 0)
-			oaiprovenance = e.getOaiprovenance();
-	}
-
-	protected <T> List<T> mergeLists(final List<T>... lists) {
-
-		return Arrays
-			.stream(lists)
-			.filter(Objects::nonNull)
-			.flatMap(List::stream)
-			.filter(Objects::nonNull)
-			.distinct()
-			.collect(Collectors.toList());
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		if (!super.equals(o))
-			return false;
-		OafEntity oafEntity = (OafEntity) o;
-		return Objects.equals(id, oafEntity.id);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(super.hashCode(), id);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Organization.java

@@ -1,214 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Organization extends OafEntity implements Serializable {
-
-	private Field<String> legalshortname;
-
-	private Field<String> legalname;
-
-	private List<Field<String>> alternativeNames;
-
-	private Field<String> websiteurl;
-
-	private Field<String> logourl;
-
-	private Field<String> eclegalbody;
-
-	private Field<String> eclegalperson;
-
-	private Field<String> ecnonprofit;
-
-	private Field<String> ecresearchorganization;
-
-	private Field<String> echighereducation;
-
-	private Field<String> ecinternationalorganizationeurinterests;
-
-	private Field<String> ecinternationalorganization;
-
-	private Field<String> ecenterprise;
-
-	private Field<String> ecsmevalidated;
-
-	private Field<String> ecnutscode;
-
-	private Qualifier country;
-
-	public Field<String> getLegalshortname() {
-		return legalshortname;
-	}
-
-	public void setLegalshortname(Field<String> legalshortname) {
-		this.legalshortname = legalshortname;
-	}
-
-	public Field<String> getLegalname() {
-		return legalname;
-	}
-
-	public void setLegalname(Field<String> legalname) {
-		this.legalname = legalname;
-	}
-
-	public List<Field<String>> getAlternativeNames() {
-		return alternativeNames;
-	}
-
-	public void setAlternativeNames(List<Field<String>> alternativeNames) {
-		this.alternativeNames = alternativeNames;
-	}
-
-	public Field<String> getWebsiteurl() {
-		return websiteurl;
-	}
-
-	public void setWebsiteurl(Field<String> websiteurl) {
-		this.websiteurl = websiteurl;
-	}
-
-	public Field<String> getLogourl() {
-		return logourl;
-	}
-
-	public void setLogourl(Field<String> logourl) {
-		this.logourl = logourl;
-	}
-
-	public Field<String> getEclegalbody() {
-		return eclegalbody;
-	}
-
-	public void setEclegalbody(Field<String> eclegalbody) {
-		this.eclegalbody = eclegalbody;
-	}
-
-	public Field<String> getEclegalperson() {
-		return eclegalperson;
-	}
-
-	public void setEclegalperson(Field<String> eclegalperson) {
-		this.eclegalperson = eclegalperson;
-	}
-
-	public Field<String> getEcnonprofit() {
-		return ecnonprofit;
-	}
-
-	public void setEcnonprofit(Field<String> ecnonprofit) {
-		this.ecnonprofit = ecnonprofit;
-	}
-
-	public Field<String> getEcresearchorganization() {
-		return ecresearchorganization;
-	}
-
-	public void setEcresearchorganization(Field<String> ecresearchorganization) {
-		this.ecresearchorganization = ecresearchorganization;
-	}
-
-	public Field<String> getEchighereducation() {
-		return echighereducation;
-	}
-
-	public void setEchighereducation(Field<String> echighereducation) {
-		this.echighereducation = echighereducation;
-	}
-
-	public Field<String> getEcinternationalorganizationeurinterests() {
-		return ecinternationalorganizationeurinterests;
-	}
-
-	public void setEcinternationalorganizationeurinterests(
-		Field<String> ecinternationalorganizationeurinterests) {
-		this.ecinternationalorganizationeurinterests = ecinternationalorganizationeurinterests;
-	}
-
-	public Field<String> getEcinternationalorganization() {
-		return ecinternationalorganization;
-	}
-
-	public void setEcinternationalorganization(Field<String> ecinternationalorganization) {
-		this.ecinternationalorganization = ecinternationalorganization;
-	}
-
-	public Field<String> getEcenterprise() {
-		return ecenterprise;
-	}
-
-	public void setEcenterprise(Field<String> ecenterprise) {
-		this.ecenterprise = ecenterprise;
-	}
-
-	public Field<String> getEcsmevalidated() {
-		return ecsmevalidated;
-	}
-
-	public void setEcsmevalidated(Field<String> ecsmevalidated) {
-		this.ecsmevalidated = ecsmevalidated;
-	}
-
-	public Field<String> getEcnutscode() {
-		return ecnutscode;
-	}
-
-	public void setEcnutscode(Field<String> ecnutscode) {
-		this.ecnutscode = ecnutscode;
-	}
-
-	public Qualifier getCountry() {
-		return country;
-	}
-
-	public void setCountry(Qualifier country) {
-		this.country = country;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!Organization.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		final Organization o = (Organization) e;
-		legalshortname = o.getLegalshortname() != null && compareTrust(this, e) < 0
-			? o.getLegalshortname()
-			: legalshortname;
-		legalname = o.getLegalname() != null && compareTrust(this, e) < 0 ? o.getLegalname() : legalname;
-		alternativeNames = mergeLists(o.getAlternativeNames(), alternativeNames);
-		websiteurl = o.getWebsiteurl() != null && compareTrust(this, e) < 0 ? o.getWebsiteurl() : websiteurl;
-		logourl = o.getLogourl() != null && compareTrust(this, e) < 0 ? o.getLogourl() : logourl;
-		eclegalbody = o.getEclegalbody() != null && compareTrust(this, e) < 0 ? o.getEclegalbody() : eclegalbody;
-		eclegalperson = o.getEclegalperson() != null && compareTrust(this, e) < 0
-			? o.getEclegalperson()
-			: eclegalperson;
-		ecnonprofit = o.getEcnonprofit() != null && compareTrust(this, e) < 0 ? o.getEcnonprofit() : ecnonprofit;
-		ecresearchorganization = o.getEcresearchorganization() != null && compareTrust(this, e) < 0
-			? o.getEcresearchorganization()
-			: ecresearchorganization;
-		echighereducation = o.getEchighereducation() != null && compareTrust(this, e) < 0
-			? o.getEchighereducation()
-			: echighereducation;
-		ecinternationalorganizationeurinterests = o.getEcinternationalorganizationeurinterests() != null
-			&& compareTrust(this, e) < 0
-				? o.getEcinternationalorganizationeurinterests()
-				: ecinternationalorganizationeurinterests;
-		ecinternationalorganization = o.getEcinternationalorganization() != null && compareTrust(this, e) < 0
-			? o.getEcinternationalorganization()
-			: ecinternationalorganization;
-		ecenterprise = o.getEcenterprise() != null && compareTrust(this, e) < 0
-			? o.getEcenterprise()
-			: ecenterprise;
-		ecsmevalidated = o.getEcsmevalidated() != null && compareTrust(this, e) < 0
-			? o.getEcsmevalidated()
-			: ecsmevalidated;
-		ecnutscode = o.getEcnutscode() != null && compareTrust(this, e) < 0 ? o.getEcnutscode() : ecnutscode;
-		country = o.getCountry() != null && compareTrust(this, e) < 0 ? o.getCountry() : country;
-		mergeOAFDataInfo(o);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OriginDescription.java

@@ -1,88 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class OriginDescription implements Serializable {
-
-	private String harvestDate;
-
-	private Boolean altered = true;
-
-	private String baseURL;
-
-	private String identifier;
-
-	private String datestamp;
-
-	private String metadataNamespace;
-
-	public String getHarvestDate() {
-		return harvestDate;
-	}
-
-	public void setHarvestDate(String harvestDate) {
-		this.harvestDate = harvestDate;
-	}
-
-	public Boolean getAltered() {
-		return altered;
-	}
-
-	public void setAltered(Boolean altered) {
-		this.altered = altered;
-	}
-
-	public String getBaseURL() {
-		return baseURL;
-	}
-
-	public void setBaseURL(String baseURL) {
-		this.baseURL = baseURL;
-	}
-
-	public String getIdentifier() {
-		return identifier;
-	}
-
-	public void setIdentifier(String identifier) {
-		this.identifier = identifier;
-	}
-
-	public String getDatestamp() {
-		return datestamp;
-	}
-
-	public void setDatestamp(String datestamp) {
-		this.datestamp = datestamp;
-	}
-
-	public String getMetadataNamespace() {
-		return metadataNamespace;
-	}
-
-	public void setMetadataNamespace(String metadataNamespace) {
-		this.metadataNamespace = metadataNamespace;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		OriginDescription that = (OriginDescription) o;
-		return Objects.equals(harvestDate, that.harvestDate)
-			&& Objects.equals(altered, that.altered)
-			&& Objects.equals(baseURL, that.baseURL)
-			&& Objects.equals(identifier, that.identifier)
-			&& Objects.equals(datestamp, that.datestamp)
-			&& Objects.equals(metadataNamespace, that.metadataNamespace);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(harvestDate, altered, baseURL, identifier, datestamp, metadataNamespace);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OtherResearchProduct.java

@@ -1,60 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-
-public class OtherResearchProduct extends Result implements Serializable {
-
-	private List<Field<String>> contactperson;
-
-	private List<Field<String>> contactgroup;
-
-	private List<Field<String>> tool;
-
-	public OtherResearchProduct() {
-		setResulttype(ModelConstants.ORP_DEFAULT_RESULTTYPE);
-	}
-
-	public List<Field<String>> getContactperson() {
-		return contactperson;
-	}
-
-	public void setContactperson(List<Field<String>> contactperson) {
-		this.contactperson = contactperson;
-	}
-
-	public List<Field<String>> getContactgroup() {
-		return contactgroup;
-	}
-
-	public void setContactgroup(List<Field<String>> contactgroup) {
-		this.contactgroup = contactgroup;
-	}
-
-	public List<Field<String>> getTool() {
-		return tool;
-	}
-
-	public void setTool(List<Field<String>> tool) {
-		this.tool = tool;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!OtherResearchProduct.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		OtherResearchProduct o = (OtherResearchProduct) e;
-
-		contactperson = mergeLists(contactperson, o.getContactperson());
-		contactgroup = mergeLists(contactgroup, o.getContactgroup());
-		tool = mergeLists(tool, o.getTool());
-		mergeOAFDataInfo(e);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java

@@ -1,38 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-public class Programme implements Serializable {
-	private String code;
-	private String description;
-
-	public String getCode() {
-		return code;
-	}
-
-	public void setCode(String code) {
-		this.code = code;
-	}
-
-	public String getDescription() {
-		return description;
-	}
-
-	public void setDescription(String description) {
-		this.description = description;
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-
-		Programme programme = (Programme) o;
-		return Objects.equals(code, programme.code);
-	}
-
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java

@@ -1,338 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Project extends OafEntity implements Serializable {
-
-	private Field<String> websiteurl;
-
-	private Field<String> code;
-
-	private Field<String> acronym;
-
-	private Field<String> title;
-
-	private Field<String> startdate;
-
-	private Field<String> enddate;
-
-	private Field<String> callidentifier;
-
-	private Field<String> keywords;
-
-	private Field<String> duration;
-
-	private Field<String> ecsc39;
-
-	private Field<String> oamandatepublications;
-
-	private Field<String> ecarticle29_3;
-
-	private List<StructuredProperty> subjects;
-
-	private List<Field<String>> fundingtree;
-
-	private Qualifier contracttype;
-
-	private Field<String> optional1;
-
-	private Field<String> optional2;
-
-	private Field<String> jsonextrainfo;
-
-	private Field<String> contactfullname;
-
-	private Field<String> contactfax;
-
-	private Field<String> contactphone;
-
-	private Field<String> contactemail;
-
-	private Field<String> summary;
-
-	private Field<String> currency;
-
-	private Float totalcost;
-
-	private Float fundedamount;
-
-	private List<Programme> programme;
-
-	public Field<String> getWebsiteurl() {
-		return websiteurl;
-	}
-
-	public void setWebsiteurl(Field<String> websiteurl) {
-		this.websiteurl = websiteurl;
-	}
-
-	public Field<String> getCode() {
-		return code;
-	}
-
-	public void setCode(Field<String> code) {
-		this.code = code;
-	}
-
-	public Field<String> getAcronym() {
-		return acronym;
-	}
-
-	public void setAcronym(Field<String> acronym) {
-		this.acronym = acronym;
-	}
-
-	public Field<String> getTitle() {
-		return title;
-	}
-
-	public void setTitle(Field<String> title) {
-		this.title = title;
-	}
-
-	public Field<String> getStartdate() {
-		return startdate;
-	}
-
-	public void setStartdate(Field<String> startdate) {
-		this.startdate = startdate;
-	}
-
-	public Field<String> getEnddate() {
-		return enddate;
-	}
-
-	public void setEnddate(Field<String> enddate) {
-		this.enddate = enddate;
-	}
-
-	public Field<String> getCallidentifier() {
-		return callidentifier;
-	}
-
-	public void setCallidentifier(Field<String> callidentifier) {
-		this.callidentifier = callidentifier;
-	}
-
-	public Field<String> getKeywords() {
-		return keywords;
-	}
-
-	public void setKeywords(Field<String> keywords) {
-		this.keywords = keywords;
-	}
-
-	public Field<String> getDuration() {
-		return duration;
-	}
-
-	public void setDuration(Field<String> duration) {
-		this.duration = duration;
-	}
-
-	public Field<String> getEcsc39() {
-		return ecsc39;
-	}
-
-	public void setEcsc39(Field<String> ecsc39) {
-		this.ecsc39 = ecsc39;
-	}
-
-	public Field<String> getOamandatepublications() {
-		return oamandatepublications;
-	}
-
-	public void setOamandatepublications(Field<String> oamandatepublications) {
-		this.oamandatepublications = oamandatepublications;
-	}
-
-	public Field<String> getEcarticle29_3() {
-		return ecarticle29_3;
-	}
-
-	public void setEcarticle29_3(Field<String> ecarticle29_3) {
-		this.ecarticle29_3 = ecarticle29_3;
-	}
-
-	public List<StructuredProperty> getSubjects() {
-		return subjects;
-	}
-
-	public void setSubjects(List<StructuredProperty> subjects) {
-		this.subjects = subjects;
-	}
-
-	public List<Field<String>> getFundingtree() {
-		return fundingtree;
-	}
-
-	public void setFundingtree(List<Field<String>> fundingtree) {
-		this.fundingtree = fundingtree;
-	}
-
-	public Qualifier getContracttype() {
-		return contracttype;
-	}
-
-	public void setContracttype(Qualifier contracttype) {
-		this.contracttype = contracttype;
-	}
-
-	public Field<String> getOptional1() {
-		return optional1;
-	}
-
-	public void setOptional1(Field<String> optional1) {
-		this.optional1 = optional1;
-	}
-
-	public Field<String> getOptional2() {
-		return optional2;
-	}
-
-	public void setOptional2(Field<String> optional2) {
-		this.optional2 = optional2;
-	}
-
-	public Field<String> getJsonextrainfo() {
-		return jsonextrainfo;
-	}
-
-	public void setJsonextrainfo(Field<String> jsonextrainfo) {
-		this.jsonextrainfo = jsonextrainfo;
-	}
-
-	public Field<String> getContactfullname() {
-		return contactfullname;
-	}
-
-	public void setContactfullname(Field<String> contactfullname) {
-		this.contactfullname = contactfullname;
-	}
-
-	public Field<String> getContactfax() {
-		return contactfax;
-	}
-
-	public void setContactfax(Field<String> contactfax) {
-		this.contactfax = contactfax;
-	}
-
-	public Field<String> getContactphone() {
-		return contactphone;
-	}
-
-	public void setContactphone(Field<String> contactphone) {
-		this.contactphone = contactphone;
-	}
-
-	public Field<String> getContactemail() {
-		return contactemail;
-	}
-
-	public void setContactemail(Field<String> contactemail) {
-		this.contactemail = contactemail;
-	}
-
-	public Field<String> getSummary() {
-		return summary;
-	}
-
-	public void setSummary(Field<String> summary) {
-		this.summary = summary;
-	}
-
-	public Field<String> getCurrency() {
-		return currency;
-	}
-
-	public void setCurrency(Field<String> currency) {
-		this.currency = currency;
-	}
-
-	public Float getTotalcost() {
-		return totalcost;
-	}
-
-	public void setTotalcost(Float totalcost) {
-		this.totalcost = totalcost;
-	}
-
-	public Float getFundedamount() {
-		return fundedamount;
-	}
-
-	public void setFundedamount(Float fundedamount) {
-		this.fundedamount = fundedamount;
-	}
-
-	public List<Programme> getProgramme() {
-		return programme;
-	}
-
-	public void setProgramme(List<Programme> programme) {
-		this.programme = programme;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!Project.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		Project p = (Project) e;
-
-		websiteurl = p.getWebsiteurl() != null && compareTrust(this, e) < 0 ? p.getWebsiteurl() : websiteurl;
-		code = p.getCode() != null && compareTrust(this, e) < 0 ? p.getCode() : code;
-		acronym = p.getAcronym() != null && compareTrust(this, e) < 0 ? p.getAcronym() : acronym;
-		title = p.getTitle() != null && compareTrust(this, e) < 0 ? p.getTitle() : title;
-		startdate = p.getStartdate() != null && compareTrust(this, e) < 0 ? p.getStartdate() : startdate;
-		enddate = p.getEnddate() != null && compareTrust(this, e) < 0 ? p.getEnddate() : enddate;
-		callidentifier = p.getCallidentifier() != null && compareTrust(this, e) < 0
-			? p.getCallidentifier()
-			: callidentifier;
-		keywords = p.getKeywords() != null && compareTrust(this, e) < 0 ? p.getKeywords() : keywords;
-		duration = p.getDuration() != null && compareTrust(this, e) < 0 ? p.getDuration() : duration;
-		ecsc39 = p.getEcsc39() != null && compareTrust(this, e) < 0 ? p.getEcsc39() : ecsc39;
-		oamandatepublications = p.getOamandatepublications() != null && compareTrust(this, e) < 0
-			? p.getOamandatepublications()
-			: oamandatepublications;
-		ecarticle29_3 = p.getEcarticle29_3() != null && compareTrust(this, e) < 0
-			? p.getEcarticle29_3()
-			: ecarticle29_3;
-		subjects = mergeLists(subjects, p.getSubjects());
-		fundingtree = mergeLists(fundingtree, p.getFundingtree());
-		contracttype = p.getContracttype() != null && compareTrust(this, e) < 0
-			? p.getContracttype()
-			: contracttype;
-		optional1 = p.getOptional1() != null && compareTrust(this, e) < 0 ? p.getOptional1() : optional1;
-		optional2 = p.getOptional2() != null && compareTrust(this, e) < 0 ? p.getOptional2() : optional2;
-		jsonextrainfo = p.getJsonextrainfo() != null && compareTrust(this, e) < 0
-			? p.getJsonextrainfo()
-			: jsonextrainfo;
-		contactfullname = p.getContactfullname() != null && compareTrust(this, e) < 0
-			? p.getContactfullname()
-			: contactfullname;
-		contactfax = p.getContactfax() != null && compareTrust(this, e) < 0 ? p.getContactfax() : contactfax;
-		contactphone = p.getContactphone() != null && compareTrust(this, e) < 0
-			? p.getContactphone()
-			: contactphone;
-		contactemail = p.getContactemail() != null && compareTrust(this, e) < 0
-			? p.getContactemail()
-			: contactemail;
-		summary = p.getSummary() != null && compareTrust(this, e) < 0 ? p.getSummary() : summary;
-		currency = p.getCurrency() != null && compareTrust(this, e) < 0 ? p.getCurrency() : currency;
-		totalcost = p.getTotalcost() != null && compareTrust(this, e) < 0 ? p.getTotalcost() : totalcost;
-		fundedamount = p.getFundedamount() != null && compareTrust(this, e) < 0
-			? p.getFundedamount()
-			: fundedamount;
-
-		programme = mergeLists(programme, p.getProgramme());
-
-		mergeOAFDataInfo(e);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Publication.java

@@ -1,39 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-
-public class Publication extends Result implements Serializable {
-
-	// publication specific
-	private Journal journal;
-
-	public Publication() {
-		setResulttype(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE);
-	}
-
-	public Journal getJournal() {
-		return journal;
-	}
-
-	public void setJournal(Journal journal) {
-		this.journal = journal;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!Publication.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		Publication p = (Publication) e;
-
-		if (p.getJournal() != null && compareTrust(this, e) < 0)
-			journal = p.getJournal();
-		mergeOAFDataInfo(e);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Qualifier.java

@@ -1,87 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-
-import org.apache.commons.lang3.StringUtils;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-public class Qualifier implements Serializable {
-
-	private String classid;
-	private String classname;
-	private String schemeid;
-	private String schemename;
-
-	public String getClassid() {
-		return classid;
-	}
-
-	public void setClassid(String classid) {
-		this.classid = classid;
-	}
-
-	public String getClassname() {
-		return classname;
-	}
-
-	public void setClassname(String classname) {
-		this.classname = classname;
-	}
-
-	public String getSchemeid() {
-		return schemeid;
-	}
-
-	public void setSchemeid(String schemeid) {
-		this.schemeid = schemeid;
-	}
-
-	public String getSchemename() {
-		return schemename;
-	}
-
-	public void setSchemename(String schemename) {
-		this.schemename = schemename;
-	}
-
-	public String toComparableString() {
-		return isBlank()
-			? ""
-			: String
-				.format(
-					"%s::%s::%s::%s",
-					classid != null ? classid : "",
-					classname != null ? classname : "",
-					schemeid != null ? schemeid : "",
-					schemename != null ? schemename : "");
-	}
-
-	@JsonIgnore
-	public boolean isBlank() {
-		return StringUtils.isBlank(classid)
-			&& StringUtils.isBlank(classname)
-			&& StringUtils.isBlank(schemeid)
-			&& StringUtils.isBlank(schemename);
-	}
-
-	@Override
-	public int hashCode() {
-		return toComparableString().hashCode();
-	}
-
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj)
-			return true;
-		if (obj == null)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-
-		Qualifier other = (Qualifier) obj;
-
-		return toComparableString().equals(other.toComparableString());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java

@@ -1,167 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * Relation models any edge between two nodes in the OpenAIRE graph. It has a source id and a target id pointing to
- * graph node identifiers and it is further characterised by the semantic of the link through the fields relType,
- * subRelType and relClass. Provenance information is modeled according to the dataInfo element and collectedFrom, while
- * individual relationship types can provide extra information via the properties field.
- */
-public class Relation extends Oaf {
-
-	/**
-	 * Main relationship classifier, values include 'resultResult', 'resultProject', 'resultOrganization', etc.
-	 */
-	private String relType;
-
-	/**
-	 * Further classifies a relationship, values include 'affiliation', 'similarity', 'supplement', etc.
-	 */
-	private String subRelType;
-
-	/**
-	 * Indicates the direction of the relationship, values include 'isSupplementTo', 'isSupplementedBy', 'merges,
-	 * 'isMergedIn'.
-	 */
-	private String relClass;
-
-	/**
-	 * The source entity id.
-	 */
-	private String source;
-
-	/**
-	 * The target entity id.
-	 */
-	private String target;
-
-	/**
-	 * Was this relationship authoritatively validated?
-	 */
-	private Boolean validated;
-
-	/**
-	 * When was this relationship authoritatively validated.
-	 */
-	private String validationDate;
-
-	/**
-	 * List of relation specific properties. Values include 'similarityLevel', indicating the similarity score between a
-	 * pair of publications.
-	 */
-	private List<KeyValue> properties = new ArrayList<>();
-
-	public String getRelType() {
-		return relType;
-	}
-
-	public void setRelType(final String relType) {
-		this.relType = relType;
-	}
-
-	public String getSubRelType() {
-		return subRelType;
-	}
-
-	public void setSubRelType(final String subRelType) {
-		this.subRelType = subRelType;
-	}
-
-	public String getRelClass() {
-		return relClass;
-	}
-
-	public void setRelClass(final String relClass) {
-		this.relClass = relClass;
-	}
-
-	public String getSource() {
-		return source;
-	}
-
-	public void setSource(final String source) {
-		this.source = source;
-	}
-
-	public String getTarget() {
-		return target;
-	}
-
-	public void setTarget(final String target) {
-		this.target = target;
-	}
-
-	public List<KeyValue> getProperties() {
-		return properties;
-	}
-
-	public void setProperties(List<KeyValue> properties) {
-		this.properties = properties;
-	}
-
-	public Boolean getValidated() {
-		return validated;
-	}
-
-	public void setValidated(Boolean validated) {
-		this.validated = validated;
-	}
-
-	public String getValidationDate() {
-		return validationDate;
-	}
-
-	public void setValidationDate(String validationDate) {
-		this.validationDate = validationDate;
-	}
-
-	public void mergeFrom(final Relation r) {
-
-		checkArgument(Objects.equals(getSource(), r.getSource()), "source ids must be equal");
-		checkArgument(Objects.equals(getTarget(), r.getTarget()), "target ids must be equal");
-		checkArgument(Objects.equals(getRelType(), r.getRelType()), "relType(s) must be equal");
-		checkArgument(
-			Objects.equals(getSubRelType(), r.getSubRelType()), "subRelType(s) must be equal");
-		checkArgument(Objects.equals(getRelClass(), r.getRelClass()), "relClass(es) must be equal");
-
-		setCollectedfrom(
-			Stream
-				.concat(
-					Optional
-						.ofNullable(getCollectedfrom())
-						.map(Collection::stream)
-						.orElse(Stream.empty()),
-					Optional
-						.ofNullable(r.getCollectedfrom())
-						.map(Collection::stream)
-						.orElse(Stream.empty()))
-				.distinct() // relies on KeyValue.equals
-				.collect(Collectors.toList()));
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o)
-			return true;
-		if (o == null || getClass() != o.getClass())
-			return false;
-		Relation relation = (Relation) o;
-		return relType.equals(relation.relType)
-			&& subRelType.equals(relation.subRelType)
-			&& relClass.equals(relation.relClass)
-			&& source.equals(relation.source)
-			&& target.equals(relation.target);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(relType, subRelType, relClass, source, target, collectedfrom);
-	}
-
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java

@@ -1,348 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class Result extends OafEntity implements Serializable {
-
-	private List<Measure> measures;
-
-	private List<Author> author;
-
-	// resulttype allows subclassing results into publications | datasets | software
-	private Qualifier resulttype;
-
-	// common fields
-	private Qualifier language;
-
-	private List<Country> country;
-
-	private List<StructuredProperty> subject;
-
-	private List<StructuredProperty> title;
-
-	private List<StructuredProperty> relevantdate;
-
-	private List<Field<String>> description;
-
-	private Field<String> dateofacceptance;
-
-	private Field<String> publisher;
-
-	private Field<String> embargoenddate;
-
-	private List<Field<String>> source;
-
-	private List<Field<String>> fulltext; // remove candidate
-
-	private List<Field<String>> format;
-
-	private List<Field<String>> contributor;
-
-	private Qualifier resourcetype;
-
-	private List<Field<String>> coverage;
-
-	private Qualifier bestaccessright;
-
-	private List<Context> context;
-
-	private List<ExternalReference> externalReference;
-
-	private List<Instance> instance;
-
-	public List<Measure> getMeasures() {
-		return measures;
-	}
-
-	public void setMeasures(List<Measure> measures) {
-		this.measures = measures;
-	}
-
-	public List<Author> getAuthor() {
-		return author;
-	}
-
-	public void setAuthor(List<Author> author) {
-		this.author = author;
-	}
-
-	public Qualifier getResulttype() {
-		return resulttype;
-	}
-
-	public void setResulttype(Qualifier resulttype) {
-		this.resulttype = resulttype;
-	}
-
-	public Qualifier getLanguage() {
-		return language;
-	}
-
-	public void setLanguage(Qualifier language) {
-		this.language = language;
-	}
-
-	public List<Country> getCountry() {
-		return country;
-	}
-
-	public void setCountry(List<Country> country) {
-		this.country = country;
-	}
-
-	public List<StructuredProperty> getSubject() {
-		return subject;
-	}
-
-	public void setSubject(List<StructuredProperty> subject) {
-		this.subject = subject;
-	}
-
-	public List<StructuredProperty> getTitle() {
-		return title;
-	}
-
-	public void setTitle(List<StructuredProperty> title) {
-		this.title = title;
-	}
-
-	public List<StructuredProperty> getRelevantdate() {
-		return relevantdate;
-	}
-
-	public void setRelevantdate(List<StructuredProperty> relevantdate) {
-		this.relevantdate = relevantdate;
-	}
-
-	public List<Field<String>> getDescription() {
-		return description;
-	}
-
-	public void setDescription(List<Field<String>> description) {
-		this.description = description;
-	}
-
-	public Field<String> getDateofacceptance() {
-		return dateofacceptance;
-	}
-
-	public void setDateofacceptance(Field<String> dateofacceptance) {
-		this.dateofacceptance = dateofacceptance;
-	}
-
-	public Field<String> getPublisher() {
-		return publisher;
-	}
-
-	public void setPublisher(Field<String> publisher) {
-		this.publisher = publisher;
-	}
-
-	public Field<String> getEmbargoenddate() {
-		return embargoenddate;
-	}
-
-	public void setEmbargoenddate(Field<String> embargoenddate) {
-		this.embargoenddate = embargoenddate;
-	}
-
-	public List<Field<String>> getSource() {
-		return source;
-	}
-
-	public void setSource(List<Field<String>> source) {
-		this.source = source;
-	}
-
-	public List<Field<String>> getFulltext() {
-		return fulltext;
-	}
-
-	public void setFulltext(List<Field<String>> fulltext) {
-		this.fulltext = fulltext;
-	}
-
-	public List<Field<String>> getFormat() {
-		return format;
-	}
-
-	public void setFormat(List<Field<String>> format) {
-		this.format = format;
-	}
-
-	public List<Field<String>> getContributor() {
-		return contributor;
-	}
-
-	public void setContributor(List<Field<String>> contributor) {
-		this.contributor = contributor;
-	}
-
-	public Qualifier getResourcetype() {
-		return resourcetype;
-	}
-
-	public void setResourcetype(Qualifier resourcetype) {
-		this.resourcetype = resourcetype;
-	}
-
-	public List<Field<String>> getCoverage() {
-		return coverage;
-	}
-
-	public void setCoverage(List<Field<String>> coverage) {
-		this.coverage = coverage;
-	}
-
-	public Qualifier getBestaccessright() {
-		return bestaccessright;
-	}
-
-	public void setBestaccessright(Qualifier bestaccessright) {
-		this.bestaccessright = bestaccessright;
-	}
-
-	public List<Context> getContext() {
-		return context;
-	}
-
-	public void setContext(List<Context> context) {
-		this.context = context;
-	}
-
-	public List<ExternalReference> getExternalReference() {
-		return externalReference;
-	}
-
-	public void setExternalReference(List<ExternalReference> externalReference) {
-		this.externalReference = externalReference;
-	}
-
-	public List<Instance> getInstance() {
-		return instance;
-	}
-
-	public void setInstance(List<Instance> instance) {
-		this.instance = instance;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!Result.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		Result r = (Result) e;
-
-		// TODO consider merging also Measures
-
-		instance = mergeLists(instance, r.getInstance());
-
-		if (r.getBestaccessright() != null && compareTrust(this, r) < 0)
-			bestaccessright = r.getBestaccessright();
-
-		if (r.getResulttype() != null && compareTrust(this, r) < 0)
-			resulttype = r.getResulttype();
-
-		if (r.getLanguage() != null && compareTrust(this, r) < 0)
-			language = r.getLanguage();
-
-		country = mergeLists(country, r.getCountry());
-
-		subject = mergeLists(subject, r.getSubject());
-
-		// merge title lists: main title with higher trust and distinct between the others
-		StructuredProperty baseMainTitle = null;
-		if (title != null) {
-			baseMainTitle = getMainTitle(title);
-			if (baseMainTitle != null) {
-				final StructuredProperty p = baseMainTitle;
-				title = title.stream().filter(t -> t != p).collect(Collectors.toList());
-			}
-		}
-
-		StructuredProperty newMainTitle = null;
-		if (r.getTitle() != null) {
-			newMainTitle = getMainTitle(r.getTitle());
-			if (newMainTitle != null) {
-				final StructuredProperty p = newMainTitle;
-				r.setTitle(r.getTitle().stream().filter(t -> t != p).collect(Collectors.toList()));
-			}
-		}
-
-		if (newMainTitle != null && compareTrust(this, r) < 0) {
-			baseMainTitle = newMainTitle;
-		}
-
-		title = mergeLists(title, r.getTitle());
-		if (title != null && baseMainTitle != null) {
-			title.add(baseMainTitle);
-		}
-
-		relevantdate = mergeLists(relevantdate, r.getRelevantdate());
-
-		description = longestLists(description, r.getDescription());
-
-		if (r.getPublisher() != null && compareTrust(this, r) < 0)
-			publisher = r.getPublisher();
-
-		if (r.getEmbargoenddate() != null && compareTrust(this, r) < 0)
-			embargoenddate = r.getEmbargoenddate();
-
-		source = mergeLists(source, r.getSource());
-
-		fulltext = mergeLists(fulltext, r.getFulltext());
-
-		format = mergeLists(format, r.getFormat());
-
-		contributor = mergeLists(contributor, r.getContributor());
-
-		if (r.getResourcetype() != null)
-			resourcetype = r.getResourcetype();
-
-		coverage = mergeLists(coverage, r.getCoverage());
-
-		context = mergeLists(context, r.getContext());
-
-		externalReference = mergeLists(externalReference, r.getExternalReference());
-	}
-
-	private List<Field<String>> longestLists(List<Field<String>> a, List<Field<String>> b) {
-		if (a == null || b == null)
-			return a == null ? b : a;
-		if (a.size() == b.size()) {
-			int msa = a
-				.stream()
-				.filter(i -> i.getValue() != null)
-				.map(i -> i.getValue().length())
-				.max(Comparator.naturalOrder())
-				.orElse(0);
-			int msb = b
-				.stream()
-				.filter(i -> i.getValue() != null)
-				.map(i -> i.getValue().length())
-				.max(Comparator.naturalOrder())
-				.orElse(0);
-			return msa > msb ? a : b;
-		}
-		return a.size() > b.size() ? a : b;
-	}
-
-	private StructuredProperty getMainTitle(List<StructuredProperty> titles) {
-		// need to check if the list of titles contains more than 1 main title? (in that case, we should chose which
-		// main title select in the list)
-		for (StructuredProperty title : titles) {
-			if (title.getQualifier() != null && title.getQualifier().getClassid() != null)
-				if (title.getQualifier().getClassid().equals("main title"))
-					return title;
-		}
-		return null;
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java

@@ -1,80 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-import java.util.List;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-
-public class Software extends Result implements Serializable {
-
-	private List<Field<String>> documentationUrl;
-
-	// candidate for removal
-	private List<StructuredProperty> license;
-
-	// candidate for removal
-	private Field<String> codeRepositoryUrl;
-
-	private Qualifier programmingLanguage;
-
-	public Software() {
-		setResulttype(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE);
-	}
-
-	public List<Field<String>> getDocumentationUrl() {
-		return documentationUrl;
-	}
-
-	public void setDocumentationUrl(List<Field<String>> documentationUrl) {
-		this.documentationUrl = documentationUrl;
-	}
-
-	public List<StructuredProperty> getLicense() {
-		return license;
-	}
-
-	public void setLicense(List<StructuredProperty> license) {
-		this.license = license;
-	}
-
-	public Field<String> getCodeRepositoryUrl() {
-		return codeRepositoryUrl;
-	}
-
-	public void setCodeRepositoryUrl(Field<String> codeRepositoryUrl) {
-		this.codeRepositoryUrl = codeRepositoryUrl;
-	}
-
-	public Qualifier getProgrammingLanguage() {
-		return programmingLanguage;
-	}
-
-	public void setProgrammingLanguage(Qualifier programmingLanguage) {
-		this.programmingLanguage = programmingLanguage;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-
-		if (!Software.class.isAssignableFrom(e.getClass())) {
-			return;
-		}
-
-		final Software s = (Software) e;
-		documentationUrl = mergeLists(documentationUrl, s.getDocumentationUrl());
-
-		license = mergeLists(license, s.getLicense());
-
-		codeRepositoryUrl = s.getCodeRepositoryUrl() != null && compareTrust(this, s) < 0
-			? s.getCodeRepositoryUrl()
-			: codeRepositoryUrl;
-
-		programmingLanguage = s.getProgrammingLanguage() != null && compareTrust(this, s) < 0
-			? s.getProgrammingLanguage()
-			: programmingLanguage;
-
-		mergeOAFDataInfo(e);
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/StructuredProperty.java

@@ -1,60 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.Serializable;
-
-public class StructuredProperty implements Serializable {
-
-	private String value;
-
-	private Qualifier qualifier;
-
-	private DataInfo dataInfo;
-
-	public String getValue() {
-		return value;
-	}
-
-	public void setValue(String value) {
-		this.value = value;
-	}
-
-	public Qualifier getQualifier() {
-		return qualifier;
-	}
-
-	public void setQualifier(Qualifier qualifier) {
-		this.qualifier = qualifier;
-	}
-
-	public DataInfo getDataInfo() {
-		return dataInfo;
-	}
-
-	public void setDataInfo(DataInfo dataInfo) {
-		this.dataInfo = dataInfo;
-	}
-
-	public String toComparableString() {
-		return value != null ? value.toLowerCase() : "";
-	}
-
-	@Override
-	public int hashCode() {
-		return toComparableString().hashCode();
-	}
-
-	@Override
-	public boolean equals(Object obj) {
-		if (this == obj)
-			return true;
-		if (obj == null)
-			return false;
-		if (getClass() != obj.getClass())
-			return false;
-
-		StructuredProperty other = (StructuredProperty) obj;
-
-		return toComparableString().equals(other.toComparableString());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIDataset.java

@@ -1,89 +0,0 @@
-
-package eu.dnetlib.dhp.schema.scholexplorer;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.StringUtils;
-
-import eu.dnetlib.dhp.schema.oaf.Dataset;
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-
-public class DLIDataset extends Dataset {
-
-	private String originalObjIdentifier;
-
-	private List<ProvenaceInfo> dlicollectedfrom;
-
-	private String completionStatus;
-
-	public String getCompletionStatus() {
-		return completionStatus;
-	}
-
-	public void setCompletionStatus(String completionStatus) {
-		this.completionStatus = completionStatus;
-	}
-
-	public List<ProvenaceInfo> getDlicollectedfrom() {
-		return dlicollectedfrom;
-	}
-
-	public void setDlicollectedfrom(List<ProvenaceInfo> dlicollectedfrom) {
-		this.dlicollectedfrom = dlicollectedfrom;
-	}
-
-	public String getOriginalObjIdentifier() {
-		return originalObjIdentifier;
-	}
-
-	public void setOriginalObjIdentifier(String originalObjIdentifier) {
-		this.originalObjIdentifier = originalObjIdentifier;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-		DLIDataset p = (DLIDataset) e;
-		if (StringUtils.isBlank(completionStatus) && StringUtils.isNotBlank(p.completionStatus))
-			completionStatus = p.completionStatus;
-		if ("complete".equalsIgnoreCase(p.completionStatus))
-			completionStatus = "complete";
-		dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
-	}
-
-	private List<ProvenaceInfo> mergeProvenance(
-		final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
-		Map<String, ProvenaceInfo> result = new HashMap<>();
-		if (a != null)
-			a
-				.forEach(
-					p -> {
-						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
-							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
-								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
-								result.put(p.getId(), p);
-							}
-
-						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
-							result.put(p.getId(), p);
-					});
-		if (b != null)
-			b
-				.forEach(
-					p -> {
-						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
-							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
-								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
-								result.put(p.getId(), p);
-							}
-
-						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
-							result.put(p.getId(), p);
-					});
-
-		return new ArrayList<>(result.values());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIPublication.java

@@ -1,87 +0,0 @@
-
-package eu.dnetlib.dhp.schema.scholexplorer;
-
-import java.io.Serializable;
-import java.util.*;
-
-import org.apache.commons.lang3.StringUtils;
-
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-import eu.dnetlib.dhp.schema.oaf.Publication;
-
-public class DLIPublication extends Publication implements Serializable {
-
-	private String originalObjIdentifier;
-
-	private List<ProvenaceInfo> dlicollectedfrom;
-
-	private String completionStatus;
-
-	public String getCompletionStatus() {
-		return completionStatus;
-	}
-
-	public void setCompletionStatus(String completionStatus) {
-		this.completionStatus = completionStatus;
-	}
-
-	public List<ProvenaceInfo> getDlicollectedfrom() {
-		return dlicollectedfrom;
-	}
-
-	public void setDlicollectedfrom(List<ProvenaceInfo> dlicollectedfrom) {
-		this.dlicollectedfrom = dlicollectedfrom;
-	}
-
-	public String getOriginalObjIdentifier() {
-		return originalObjIdentifier;
-	}
-
-	public void setOriginalObjIdentifier(String originalObjIdentifier) {
-		this.originalObjIdentifier = originalObjIdentifier;
-	}
-
-	@Override
-	public void mergeFrom(OafEntity e) {
-		super.mergeFrom(e);
-		DLIPublication p = (DLIPublication) e;
-		if (StringUtils.isBlank(completionStatus) && StringUtils.isNotBlank(p.completionStatus))
-			completionStatus = p.completionStatus;
-		if ("complete".equalsIgnoreCase(p.completionStatus))
-			completionStatus = "complete";
-		dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
-	}
-
-	private List<ProvenaceInfo> mergeProvenance(
-		final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
-		Map<String, ProvenaceInfo> result = new HashMap<>();
-		if (a != null)
-			a
-				.forEach(
-					p -> {
-						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
-							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
-								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
-								result.put(p.getId(), p);
-							}
-
-						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
-							result.put(p.getId(), p);
-					});
-		if (b != null)
-			b
-				.forEach(
-					p -> {
-						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
-							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
-								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
-								result.put(p.getId(), p);
-							}
-
-						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
-							result.put(p.getId(), p);
-					});
-
-		return new ArrayList<>(result.values());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIRelation.java

@@ -1,30 +0,0 @@
-
-package eu.dnetlib.dhp.schema.scholexplorer;
-
-import java.util.List;
-
-import eu.dnetlib.dhp.schema.oaf.KeyValue;
-import eu.dnetlib.dhp.schema.oaf.Relation;
-
-public class DLIRelation extends Relation {
-
-	private String dateOfCollection;
-
-	private List<KeyValue> collectedFrom;
-
-	public List<KeyValue> getCollectedFrom() {
-		return collectedFrom;
-	}
-
-	public void setCollectedFrom(List<KeyValue> collectedFrom) {
-		this.collectedFrom = collectedFrom;
-	}
-
-	public String getDateOfCollection() {
-		return dateOfCollection;
-	}
-
-	public void setDateOfCollection(String dateOfCollection) {
-		this.dateOfCollection = dateOfCollection;
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIUnknown.java

@@ -1,115 +0,0 @@
-
-package eu.dnetlib.dhp.schema.scholexplorer;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.StringUtils;
-
-import eu.dnetlib.dhp.schema.oaf.Oaf;
-import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
-
-public class DLIUnknown extends Oaf implements Serializable {
-
-	private String id;
-
-	private List<StructuredProperty> pid;
-
-	private String dateofcollection;
-
-	private String dateoftransformation;
-
-	private List<ProvenaceInfo> dlicollectedfrom;
-
-	private String completionStatus = "incomplete";
-
-	public String getCompletionStatus() {
-		return completionStatus;
-	}
-
-	public void setCompletionStatus(String completionStatus) {
-		this.completionStatus = completionStatus;
-	}
-
-	public List<ProvenaceInfo> getDlicollectedfrom() {
-		return dlicollectedfrom;
-	}
-
-	public void setDlicollectedfrom(List<ProvenaceInfo> dlicollectedfrom) {
-		this.dlicollectedfrom = dlicollectedfrom;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public List<StructuredProperty> getPid() {
-		return pid;
-	}
-
-	public void setPid(List<StructuredProperty> pid) {
-		this.pid = pid;
-	}
-
-	public String getDateofcollection() {
-		return dateofcollection;
-	}
-
-	public void setDateofcollection(String dateofcollection) {
-		this.dateofcollection = dateofcollection;
-	}
-
-	public String getDateoftransformation() {
-		return dateoftransformation;
-	}
-
-	public void setDateoftransformation(String dateoftransformation) {
-		this.dateoftransformation = dateoftransformation;
-	}
-
-	public void mergeFrom(DLIUnknown p) {
-		if ("complete".equalsIgnoreCase(p.completionStatus))
-			completionStatus = "complete";
-		dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
-	}
-
-	private List<ProvenaceInfo> mergeProvenance(
-		final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
-		Map<String, ProvenaceInfo> result = new HashMap<>();
-		if (a != null)
-			a
-				.forEach(
-					p -> {
-						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
-							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
-								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
-								result.put(p.getId(), p);
-							}
-
-						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
-							result.put(p.getId(), p);
-					});
-		if (b != null)
-			b
-				.forEach(
-					p -> {
-						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
-							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
-								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
-								result.put(p.getId(), p);
-							}
-
-						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
-							result.put(p.getId(), p);
-					});
-
-		return new ArrayList<>(result.values());
-	}
-}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/ProvenaceInfo.java

@@ -1,47 +0,0 @@
-
-package eu.dnetlib.dhp.schema.scholexplorer;
-
-import java.io.Serializable;
-
-public class ProvenaceInfo implements Serializable {
-
-	private String id;
-
-	private String name;
-
-	private String completionStatus;
-
-	private String collectionMode = "collected";
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getCompletionStatus() {
-		return completionStatus;
-	}
-
-	public void setCompletionStatus(String completionStatus) {
-		this.completionStatus = completionStatus;
-	}
-
-	public String getCollectionMode() {
-		return collectionMode;
-	}
-
-	public void setCollectionMode(String collectionMode) {
-		this.collectionMode = collectionMode;
-	}
-}

dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/action/AtomicActionTest.java

@@ -1,40 +0,0 @@
-
-package eu.dnetlib.dhp.schema.action;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.io.IOException;
-
-import org.apache.commons.lang3.StringUtils;
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.dhp.schema.oaf.Relation;
-
-/** @author claudio.atzori */
-public class AtomicActionTest {
-
-	@Test
-	public void serializationTest() throws IOException {
-
-		Relation rel = new Relation();
-		rel.setSource("1");
-		rel.setTarget("2");
-		rel.setRelType("resultResult");
-		rel.setSubRelType("dedup");
-		rel.setRelClass("merges");
-
-		AtomicAction aa1 = new AtomicAction(Relation.class, rel);
-
-		final ObjectMapper mapper = new ObjectMapper();
-		String json = mapper.writeValueAsString(aa1);
-
-		assertTrue(StringUtils.isNotBlank(json));
-
-		AtomicAction aa2 = mapper.readValue(json, AtomicAction.class);
-
-		assertEquals(aa1.getClazz(), aa2.getClazz());
-		assertEquals(aa1.getPayload(), aa2.getPayload());
-	}
-}

dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/common/ModelSupportTest.java

@@ -1,37 +0,0 @@
-
-package eu.dnetlib.dhp.schema.common;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import org.junit.jupiter.api.Nested;
-import org.junit.jupiter.api.Test;
-
-import eu.dnetlib.dhp.schema.oaf.OafEntity;
-import eu.dnetlib.dhp.schema.oaf.Relation;
-import eu.dnetlib.dhp.schema.oaf.Result;
-
-public class ModelSupportTest {
-
-	@Nested
-	class IsSubClass {
-
-		@Test
-		public void shouldReturnFalseWhenSubClassDoesNotExtendSuperClass() {
-			// when
-			Boolean result = ModelSupport.isSubClass(Relation.class, OafEntity.class);
-
-			// then
-			assertFalse(result);
-		}
-
-		@Test
-		public void shouldReturnTrueWhenSubClassExtendsSuperClass() {
-			// when
-			Boolean result = ModelSupport.isSubClass(Result.class, OafEntity.class);
-
-			// then
-			assertTrue(result);
-		}
-	}
-}

dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MeasureTest.java

@@ -1,57 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-
-public class MeasureTest {
-
-	public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
-		.setSerializationInclusion(JsonInclude.Include.NON_NULL);
-
-	@Test
-	public void testMeasureSerialization() throws IOException {
-
-		Measure popularity = new Measure();
-		popularity.setId("popularity");
-		popularity
-			.setUnit(
-				Lists
-					.newArrayList(
-						unit("score", "0.5")));
-
-		Measure influence = new Measure();
-		influence.setId("influence");
-		influence
-			.setUnit(
-				Lists
-					.newArrayList(
-						unit("score", "0.3")));
-
-		List<Measure> m = Lists.newArrayList(popularity, influence);
-
-		String s = OBJECT_MAPPER.writeValueAsString(m);
-		System.out.println(s);
-
-		List<Measure> mm = OBJECT_MAPPER.readValue(s, new TypeReference<List<Measure>>() {
-		});
-
-		Assertions.assertNotNull(mm);
-	}
-
-	private KeyValue unit(String key, String value) {
-		KeyValue unit = new KeyValue();
-		unit.setKey(key);
-		unit.setValue(value);
-		return unit;
-	}
-
-}

dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/oaf/MergeTest.java

@@ -1,88 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-public class MergeTest {
-
-	OafEntity oaf;
-
-	@BeforeEach
-	public void setUp() {
-		oaf = new Publication();
-	}
-
-	@Test
-	public void mergeListsTest() {
-
-		// string list merge test
-		List<String> a = Arrays.asList("a", "b", "c", "e");
-		List<String> b = Arrays.asList("a", "b", "c", "d");
-		List<String> c = null;
-
-		System.out.println("merge result 1 = " + oaf.mergeLists(a, b));
-
-		System.out.println("merge result 2 = " + oaf.mergeLists(a, c));
-
-		System.out.println("merge result 3 = " + oaf.mergeLists(c, c));
-	}
-
-	@Test
-	public void mergePublicationCollectedFromTest() {
-
-		Publication a = new Publication();
-		Publication b = new Publication();
-
-		a.setCollectedfrom(Arrays.asList(setKV("a", "open"), setKV("b", "closed")));
-		b.setCollectedfrom(Arrays.asList(setKV("A", "open"), setKV("b", "Open")));
-
-		a.mergeFrom(b);
-
-		assertNotNull(a.getCollectedfrom());
-		assertEquals(3, a.getCollectedfrom().size());
-	}
-
-	@Test
-	public void mergePublicationSubjectTest() {
-
-		Publication a = new Publication();
-		Publication b = new Publication();
-
-		a.setSubject(Arrays.asList(setSP("a", "open", "classe"), setSP("b", "open", "classe")));
-		b.setSubject(Arrays.asList(setSP("A", "open", "classe"), setSP("c", "open", "classe")));
-
-		a.mergeFrom(b);
-
-		assertNotNull(a.getSubject());
-		assertEquals(3, a.getSubject().size());
-	}
-
-	private KeyValue setKV(final String key, final String value) {
-
-		KeyValue k = new KeyValue();
-
-		k.setKey(key);
-		k.setValue(value);
-
-		return k;
-	}
-
-	private StructuredProperty setSP(
-		final String value, final String schema, final String classname) {
-		StructuredProperty s = new StructuredProperty();
-		s.setValue(value);
-		Qualifier q = new Qualifier();
-		q.setClassname(classname);
-		q.setClassid(classname);
-		q.setSchemename(schema);
-		q.setSchemeid(schema);
-		s.setQualifier(q);
-		return s;
-	}
-}

dhp-schemas/src/test/java/eu/dnetlib/dhp/schema/scholexplorer/DLItest.java

@@ -1,83 +0,0 @@
-
-package eu.dnetlib.dhp.schema.scholexplorer;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-
-import eu.dnetlib.dhp.schema.oaf.Qualifier;
-import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
-
-public class DLItest {
-
-	@Test
-	public void testMergePublication() throws JsonProcessingException {
-		DLIPublication a1 = new DLIPublication();
-		a1.setPid(Arrays.asList(createSP("123456", "pdb", "dnet:pid_types")));
-		a1.setTitle(Collections.singletonList(createSP("Un Titolo", "title", "dnetTitle")));
-		a1.setDlicollectedfrom(Arrays.asList(createCollectedFrom("znd", "Zenodo", "complete")));
-		a1.setCompletionStatus("complete");
-
-		DLIPublication a = new DLIPublication();
-		a
-			.setPid(
-				Arrays
-					.asList(
-						createSP("10.11", "doi", "dnet:pid_types"),
-						createSP("123456", "pdb", "dnet:pid_types")));
-		a.setTitle(Collections.singletonList(createSP("A Title", "title", "dnetTitle")));
-		a
-			.setDlicollectedfrom(
-				Arrays
-					.asList(
-						createCollectedFrom("dct", "datacite", "complete"),
-						createCollectedFrom("dct", "datacite", "incomplete")));
-		a.setCompletionStatus("incomplete");
-
-		a.mergeFrom(a1);
-
-		ObjectMapper mapper = new ObjectMapper();
-		System.out.println(mapper.writeValueAsString(a));
-	}
-
-	@Test
-	public void testDeserialization() throws IOException {
-
-		final String json = "{\"dataInfo\":{\"invisible\":false,\"inferred\":null,\"deletedbyinference\":false,\"trust\":\"0.9\",\"inferenceprovenance\":null,\"provenanceaction\":null},\"lastupdatetimestamp\":null,\"id\":\"60|bd9352547098929a394655ad1a44a479\",\"originalId\":[\"bd9352547098929a394655ad1a44a479\"],\"collectedfrom\":[{\"key\":\"dli_________::datacite\",\"value\":\"Datasets in Datacite\",\"dataInfo\":null,\"blank\":false}],\"pid\":[{\"value\":\"10.7925/DRS1.DUCHAS_5078760\",\"qualifier\":{\"classid\":\"doi\",\"classname\":\"doi\",\"schemeid\":\"dnet:pid_types\",\"schemename\":\"dnet:pid_types\",\"blank\":false},\"dataInfo\":null}],\"dateofcollection\":\"2020-01-09T08:29:31.885Z\",\"dateoftransformation\":null,\"extraInfo\":null,\"oaiprovenance\":null,\"author\":[{\"fullname\":\"Cathail, S. Ó\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Donnell, Breda Mc\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Ireland. Department of Arts, Culture, and the Gaeltacht\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"University College Dublin\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"National Folklore Foundation\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Cathail, S. Ó\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null},{\"fullname\":\"Donnell, Breda Mc\",\"name\":null,\"surname\":null,\"rank\":null,\"pid\":null,\"affiliation\":null}],\"resulttype\":null,\"language\":null,\"country\":null,\"subject\":[{\"value\":\"Recreation\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null},{\"value\":\"Entertainments and recreational activities\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null},{\"value\":\"Siamsaíocht agus caitheamh aimsire\",\"qualifier\":{\"classid\":\"dnet:subject\",\"classname\":\"dnet:subject\",\"schemeid\":\"unknown\",\"schemename\":\"unknown\",\"blank\":false},\"dataInfo\":null}],\"title\":[{\"value\":\"Games We Play\",\"qualifier\":null,\"dataInfo\":null}],\"relevantdate\":[{\"value\":\"1938-09-28\",\"qualifier\":{\"classid\":\"date\",\"classname\":\"date\",\"schemeid\":\"dnet::date\",\"schemename\":\"dnet::date\",\"blank\":false},\"dataInfo\":null}],\"description\":[{\"value\":\"Story collected by Breda Mc Donnell, a student at Tenure school (Tinure, Co. Louth) (no informant identified).\",\"dataInfo\":null}],\"dateofacceptance\":null,\"publisher\":{\"value\":\"University College Dublin\",\"dataInfo\":null},\"embargoenddate\":null,\"source\":null,\"fulltext\":null,\"format\":null,\"contributor\":null,\"resourcetype\":null,\"coverage\":null,\"refereed\":null,\"context\":null,\"processingchargeamount\":null,\"processingchargecurrency\":null,\"externalReference\":null,\"instance\":[],\"storagedate\":null,\"device\":null,\"size\":null,\"version\":null,\"lastmetadataupdate\":null,\"metadataversionnumber\":null,\"geolocation\":null,\"dlicollectedfrom\":[{\"id\":\"dli_________::datacite\",\"name\":\"Datasets in Datacite\",\"completionStatus\":\"complete\",\"collectionMode\":\"resolved\"}],\"completionStatus\":\"complete\"}";
-
-		ObjectMapper mapper = new ObjectMapper();
-		mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-		DLIDataset dliDataset = mapper.readValue(json, DLIDataset.class);
-		mapper.enable(SerializationFeature.INDENT_OUTPUT);
-		System.out.println(mapper.writeValueAsString(dliDataset));
-	}
-
-	private ProvenaceInfo createCollectedFrom(
-		final String id, final String name, final String completionStatus) {
-		ProvenaceInfo p = new ProvenaceInfo();
-		p.setId(id);
-		p.setName(name);
-		p.setCompletionStatus(completionStatus);
-		return p;
-	}
-
-	private StructuredProperty createSP(
-		final String value, final String className, final String schemeName) {
-		StructuredProperty p = new StructuredProperty();
-		p.setValue(value);
-		Qualifier schema = new Qualifier();
-		schema.setClassname(className);
-		schema.setClassid(className);
-		schema.setSchemename(schemeName);
-		schema.setSchemeid(schemeName);
-		p.setQualifier(schema);
-		return p;
-	}
-}

dhp-workflows/dhp-actionmanager/pom.xml

@@ -59,7 +59,6 @@
         <dependency>
             <groupId>eu.dnetlib.dhp</groupId>
             <artifactId>dhp-schemas</artifactId>
-            <version>${project.version}</version>
         </dependency>
 
         <dependency>

dhp-workflows/dhp-actionmanager/src/main/java/eu/dnetlib/dhp/actionmanager/ISClient.java

@@ -3,20 +3,23 @@ package eu.dnetlib.dhp.actionmanager;
 
 import java.io.Serializable;
 import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
+import java.util.*;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.tuple.Triple;
 import org.dom4j.Document;
+import org.dom4j.DocumentException;
 import org.dom4j.Element;
 import org.dom4j.io.SAXReader;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 import eu.dnetlib.actionmanager.rmi.ActionManagerException;
 import eu.dnetlib.actionmanager.set.ActionManagerSet;
@@ -25,6 +28,7 @@ import eu.dnetlib.dhp.actionmanager.partition.PartitionActionSetsByPayloadTypeJo
 import eu.dnetlib.dhp.utils.ISLookupClientFactory;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
 import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import scala.Tuple2;
 
 public class ISClient implements Serializable {
 
@@ -40,80 +44,52 @@ public class ISClient implements Serializable {
 
 	public List<String> getLatestRawsetPaths(String setIds) {
 
-		List<String> ids = Lists
-			.newArrayList(
+		final Set<String> ids = Sets
+			.newHashSet(
 				Splitter
 					.on(INPUT_ACTION_SET_ID_SEPARATOR)
 					.omitEmptyStrings()
 					.trimResults()
 					.split(setIds));
-
-		return ids
-			.stream()
-			.map(id -> getSet(isLookup, id))
-			.map(as -> as.getPathToLatest())
-			.collect(Collectors.toCollection(ArrayList::new));
-	}
-
-	private ActionManagerSet getSet(ISLookUpService isLookup, final String setId) {
-
-		final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') "
-			+ "where $x//SET/@id = '"
-			+ setId
-			+ "' return $x";
-
 		try {
 			final String basePath = getBasePathHDFS(isLookup);
-			final String setProfile = isLookup.getResourceProfileByQuery(q);
-			return getActionManagerSet(basePath, setProfile);
-		} catch (ISLookUpException | ActionManagerException e) {
-			throw new RuntimeException("Error accessing Sets, using query: " + q);
+
+			// <SET id="..." directory="..." latest="xxx"/>
+			final String xquery = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') "
+				+
+				"return <SET id='{$x//SET/@id/string()}' directory='{$x//SET/@directory/string()}' latest='{$x//LATEST/@id/string()}'/>";
+			return Optional
+				.ofNullable(isLookup.quickSearchProfile(xquery))
+				.map(
+					sets -> sets
+						.stream()
+						.map(set -> parseSetInfo(set))
+						.filter(t -> ids.contains(t.getLeft()))
+						.map(t -> buildDirectory(basePath, t))
+						.collect(Collectors.toList()))
+				.orElseThrow(() -> new IllegalStateException("empty set list"));
+		} catch (ActionManagerException | ISLookUpException e) {
+			throw new IllegalStateException("unable to query ActionSets info from the IS");
 		}
 	}
 
-	private ActionManagerSet getActionManagerSet(final String basePath, final String profile)
-		throws ActionManagerException {
-		final SAXReader reader = new SAXReader();
-		final ActionManagerSet set = new ActionManagerSet();
-
+	private Triple<String, String, String> parseSetInfo(String set) {
 		try {
-			final Document doc = reader.read(new StringReader(profile));
-
-			set.setId(doc.valueOf("//SET/@id").trim());
-			set.setName(doc.valueOf("//SET").trim());
-			set.setImpact(ImpactTypes.valueOf(doc.valueOf("//IMPACT").trim()));
-			set
-				.setLatest(
-					doc.valueOf("//RAW_SETS/LATEST/@id"),
-					doc.valueOf("//RAW_SETS/LATEST/@creationDate"),
-					doc.valueOf("//RAW_SETS/LATEST/@lastUpdate"));
-			set.setDirectory(doc.valueOf("//SET/@directory"));
-			final List expiredNodes = doc.selectNodes("//RAW_SETS/EXPIRED");
-			if (expiredNodes != null) {
-				for (int i = 0; i < expiredNodes.size(); i++) {
-					Element ex = (Element) expiredNodes.get(i);
-					set
-						.addExpired(
-							ex.attributeValue("id"),
-							ex.attributeValue("creationDate"),
-							ex.attributeValue("lastUpdate"));
-				}
-			}
-
-			final StringBuilder sb = new StringBuilder();
-			sb.append(basePath);
-			sb.append("/");
-			sb.append(doc.valueOf("//SET/@directory"));
-			sb.append("/");
-			sb.append(doc.valueOf("//RAW_SETS/LATEST/@id"));
-			set.setPathToLatest(sb.toString());
-
-			return set;
-		} catch (Exception e) {
-			throw new ActionManagerException("Error creating set from profile: " + profile, e);
+			Document doc = new SAXReader().read(new StringReader(set));
+			return Triple
+				.of(
+					doc.valueOf("//SET/@id"),
+					doc.valueOf("//SET/@directory"),
+					doc.valueOf("//SET/@latest"));
+		} catch (DocumentException e) {
+			throw new IllegalStateException(e);
 		}
 	}
 
+	private String buildDirectory(String basePath, Triple<String, String, String> t) {
+		return Joiner.on("/").join(basePath, t.getMiddle(), t.getRight());
+	}
+
 	private String getBasePathHDFS(ISLookUpService isLookup) throws ActionManagerException {
 		return queryServiceProperty(isLookup, "basePath");
 	}

dhp-workflows/dhp-aggregation/pom.xml

@@ -35,7 +35,6 @@
          <dependency>
             <groupId>eu.dnetlib.dhp</groupId>
             <artifactId>dhp-schemas</artifactId>
-            <version>${project.version}</version>
         </dependency>
 
 
@@ -66,6 +65,19 @@
         </dependency>
 
 
+        <!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+        </dependency>
+
+
 
     </dependencies>
 

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/BipDeserialize.java

@@ -0,0 +1,28 @@
+
+package eu.dnetlib.dhp.actionmanager.bipfinder;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Class that maps the model of the bipFinder! input data.
+ * Only needed for deserialization purposes
+ */
+
+public class BipDeserialize extends HashMap<String, List<Score>> implements Serializable {
+
+	public BipDeserialize() {
+		super();
+	}
+
+	public List<Score> get(String key) {
+
+		if (super.get(key) == null) {
+			return new ArrayList<>();
+		}
+		return super.get(key);
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/BipScore.java

@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.actionmanager.bipfinder;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Rewriting of the bipFinder input data by extracting the identifier of the result (doi)
+ */
+
+public class BipScore implements Serializable {
+	private String id; // doi
+	private List<Score> scoreList; // unit as given in the inputfile
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public List<Score> getScoreList() {
+		return scoreList;
+	}
+
+	public void setScoreList(List<Score> scoreList) {
+		this.scoreList = scoreList;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/CollectAndSave.java

@@ -0,0 +1,85 @@
+
+package eu.dnetlib.dhp.actionmanager.bipfinder;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.io.Serializable;
+import java.util.Optional;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.oaf.Result;
+
+/**
+ * Just collects all the atomic actions produced for the different results and saves them in
+ * outputpath for the ActionSet
+ */
+public class CollectAndSave implements Serializable {
+
+	private static final Logger log = LoggerFactory.getLogger(CollectAndSave.class);
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	public static <I extends Result> void main(String[] args) throws Exception {
+
+		String jsonConfiguration = IOUtils
+			.toString(
+				CollectAndSave.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/actionmanager/bipfinder/input_actionset_parameter.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		final String inputPath = parser.get("inputPath");
+		log.info("inputPath {}: ", inputPath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath {}: ", outputPath);
+
+		SparkConf conf = new SparkConf();
+
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				removeOutputDir(spark, outputPath);
+				collectAndSave(spark, inputPath, outputPath);
+			});
+	}
+
+	private static void collectAndSave(SparkSession spark, String inputPath, String outputPath) {
+		JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
+
+		sc
+			.sequenceFile(inputPath + "/publication", Text.class, Text.class)
+			.union(sc.sequenceFile(inputPath + "/dataset", Text.class, Text.class))
+			.union(sc.sequenceFile(inputPath + "/otherresearchproduct", Text.class, Text.class))
+			.union(sc.sequenceFile(inputPath + "/software", Text.class, Text.class))
+			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
+		;
+	}
+
+	private static void removeOutputDir(SparkSession spark, String path) {
+		HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/KeyValue.java

@@ -0,0 +1,26 @@
+
+package eu.dnetlib.dhp.actionmanager.bipfinder;
+
+import java.io.Serializable;
+
+public class KeyValue implements Serializable {
+
+	private String key;
+	private String value;
+
+	public String getKey() {
+		return key;
+	}
+
+	public void setKey(String key) {
+		this.key = key;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(String value) {
+		this.value = value;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/PreparedResult.java

@@ -0,0 +1,28 @@
+
+package eu.dnetlib.dhp.actionmanager.bipfinder;
+
+import java.io.Serializable;
+
+/**
+ * Subset of the information of the generic results that are needed to create the atomic action
+ */
+public class PreparedResult implements Serializable {
+	private String id; // openaire id
+	private String value; // doi
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(String value) {
+		this.value = value;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/Score.java

@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.actionmanager.bipfinder;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * represents the score in the input file
+ */
+public class Score implements Serializable {
+
+	private String id;
+	private List<KeyValue> unit;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public List<KeyValue> getUnit() {
+		return unit;
+	}
+
+	public void setUnit(List<KeyValue> unit) {
+		this.unit = unit;
+	}
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java

@@ -0,0 +1,200 @@
+
+package eu.dnetlib.dhp.actionmanager.bipfinder;
+
+import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.MapFunction;
+import org.apache.spark.api.java.function.MapGroupsFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.HdfsSupport;
+import eu.dnetlib.dhp.schema.action.AtomicAction;
+import eu.dnetlib.dhp.schema.oaf.*;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import scala.Tuple2;
+
+/**
+ * created the Atomic Action for each tipe of results
+ */
+public class SparkAtomicActionScoreJob implements Serializable {
+
+	private static String DOI = "doi";
+	private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionScoreJob.class);
+	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+	public static <I extends Result> void main(String[] args) throws Exception {
+
+		String jsonConfiguration = IOUtils
+			.toString(
+				SparkAtomicActionScoreJob.class
+					.getResourceAsStream(
+						"/eu/dnetlib/dhp/actionmanager/bipfinder/input_parameters.json"));
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
+
+		parser.parseArgument(args);
+
+		Boolean isSparkSessionManaged = Optional
+			.ofNullable(parser.get("isSparkSessionManaged"))
+			.map(Boolean::valueOf)
+			.orElse(Boolean.TRUE);
+
+		log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
+
+		final String inputPath = parser.get("inputPath");
+		log.info("inputPath {}: ", inputPath);
+
+		final String outputPath = parser.get("outputPath");
+		log.info("outputPath {}: ", outputPath);
+
+		final String bipScorePath = parser.get("bipScorePath");
+		log.info("bipScorePath: {}", bipScorePath);
+
+		final String resultClassName = parser.get("resultTableName");
+		log.info("resultTableName: {}", resultClassName);
+
+		Class<I> inputClazz = (Class<I>) Class.forName(resultClassName);
+
+		SparkConf conf = new SparkConf();
+
+		runWithSparkSession(
+			conf,
+			isSparkSessionManaged,
+			spark -> {
+				removeOutputDir(spark, outputPath);
+				prepareResults(spark, inputPath, outputPath, bipScorePath, inputClazz);
+			});
+	}
+
+	private static <I extends Result> void prepareResults(SparkSession spark, String inputPath, String outputPath,
+		String bipScorePath, Class<I> inputClazz) {
+
+		final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
+
+		JavaRDD<BipDeserialize> bipDeserializeJavaRDD = sc
+			.textFile(bipScorePath)
+			.map(item -> OBJECT_MAPPER.readValue(item, BipDeserialize.class));
+
+		Dataset<BipScore> bipScores = spark
+			.createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> {
+				BipScore bs = new BipScore();
+				bs.setId(key);
+				bs.setScoreList(entry.get(key));
+				return bs;
+			}).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class));
+
+		System.out.println(bipScores.count());
+
+		Dataset<I> results = readPath(spark, inputPath, inputClazz);
+
+		results.createOrReplaceTempView("result");
+
+		Dataset<PreparedResult> preparedResult = spark
+			.sql(
+				"select pIde.value value, id " +
+					"from result " +
+					"lateral view explode (pid) p as pIde " +
+					"where dataInfo.deletedbyinference = false and pIde.qualifier.classid = '" + DOI + "'")
+			.as(Encoders.bean(PreparedResult.class));
+
+		bipScores
+			.joinWith(
+				preparedResult, bipScores.col("id").equalTo(preparedResult.col("value")),
+				"inner")
+			.map((MapFunction<Tuple2<BipScore, PreparedResult>, BipScore>) value -> {
+				BipScore ret = value._1();
+				ret.setId(value._2().getId());
+				return ret;
+			}, Encoders.bean(BipScore.class))
+			.groupByKey((MapFunction<BipScore, String>) value -> value.getId(), Encoders.STRING())
+			.mapGroups((MapGroupsFunction<String, BipScore, Result>) (k, it) -> {
+				Result ret = new Result();
+				ret.setDataInfo(getDataInfo());
+				BipScore first = it.next();
+				ret.setId(first.getId());
+
+				ret.setMeasures(getMeasure(first));
+				it.forEachRemaining(value -> ret.getMeasures().addAll(getMeasure(value)));
+
+				return ret;
+			}, Encoders.bean(Result.class))
+			.toJavaRDD()
+			.map(p -> new AtomicAction(inputClazz, p))
+			.mapToPair(
+				aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()),
+					new Text(OBJECT_MAPPER.writeValueAsString(aa))))
+			.saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);
+
+	}
+
+	private static List<Measure> getMeasure(BipScore value) {
+		return value
+			.getScoreList()
+			.stream()
+			.map(score -> {
+				Measure m = new Measure();
+				m.setId(score.getId());
+				m
+					.setUnit(
+						score
+							.getUnit()
+							.stream()
+							.map(unit -> {
+								KeyValue kv = new KeyValue();
+								kv.setValue(unit.getValue());
+								kv.setKey(unit.getKey());
+								kv.setDataInfo(getDataInfo());
+								return kv;
+							})
+							.collect(Collectors.toList()));
+				return m;
+			})
+			.collect(Collectors.toList());
+	}
+
+	private static DataInfo getDataInfo() {
+		DataInfo di = new DataInfo();
+		di.setInferred(false);
+		di.setInvisible(false);
+		di.setDeletedbyinference(false);
+		di.setTrust("");
+		Qualifier qualifier = new Qualifier();
+		qualifier.setClassid("sysimport:actionset");
+		qualifier.setClassname("Harvested");
+		qualifier.setSchemename("dnet:provenanceActions");
+		qualifier.setSchemeid("dnet:provenanceActions");
+		di.setProvenanceaction(qualifier);
+		return di;
+	}
+
+	private static void removeOutputDir(SparkSession spark, String path) {
+		HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
+	}
+
+	public static <R> Dataset<R> readPath(
+		SparkSession spark, String inputPath, Class<R> clazz) {
+		return spark
+			.read()
+			.textFile(inputPath)
+			.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
+	}
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProgramme.java

@@ -4,11 +4,15 @@ package eu.dnetlib.dhp.actionmanager.project;
 import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.sql.*;
 import org.slf4j.Logger;
@@ -16,11 +20,79 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme;
+import eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import scala.Tuple2;
 
+/**
+ * Among all the programmes provided in the csv file, selects those in H2020 framework that have an english title.
+ *
+ * The title is then handled to get the programme description at a certain level. The set of programme titles will then
+ * be used to associate a classification for the programme.
+ *
+ * The programme code describes an hierarchy that can be exploited to provide the classification. To determine the hierarchy
+ * the code can be split by '.'. If the length of the splitted code is less than or equal to 2 it can be directly used
+ * as the classification: H2020-EU -> Horizon 2020 Framework Programme (It will never be repeated),
+ * H2020-EU.1. -> Excellent science, H2020-EU.2. -> Industrial leadership etc.
+ *
+ * The codes are ordered and for all of them the concatenation of all the titles (from the element in position 1 of
+ * the splitted code) handled as below is used to create the classification. For example:
+ *
+ *   H2020-EU.1.1       -> Excellent science | European Research Council (ERC)
+ *   from H2020-EU.1. -> Excellence science and H2020-EU.1.1. -> European Research Council (ERC)
+ *
+ *   H2020-EU.3.1.3.1. -> Societal challenges | Health, demographic change and well-being | Treating and managing disease | Treating disease, including developing regenerative medicine
+ *   from H2020-EU.3.       -> Societal challenges,
+ *        H2020-EU.3.1.     -> Health, demographic change and well-being
+ *        H2020-EU.3.1.3    -> Treating and managing disease
+ *        H2020-EU.3.1.3.1. -> Treating disease, including developing regenerative medicine
+ *
+ * The classification up to level three, will be split in dedicated variables, while the complete classification will be stored
+ * in a variable called classification and provided as shown above.
+ *
+ * The programme title is not give in a standardized way:
+ *
+ *  - Sometimes associated to the higher level in the hierarchy we can find Priority in title other times it is not the
+ *    case. Since it is not uniform, we removed priority from the handled titles:
+ *
+ *    H2020-EU.1. -> PRIORITY 'Excellent science'
+ *    H2020-EU.2. -> PRIORITY 'Industrial leadership'
+ *    H2020-EU.3. -> PRIORITY 'Societal challenges
+ *
+ *    will become
+ *
+ *    H2020-EU.1. -> Excellent science
+ *    H2020-EU.2. -> Industrial leadership
+ *    H2020-EU.3. -> Societal challenges
+ *
+ *  - Sometimes the title of the parent is repeated in the title for the code, but it is not always the case, so, titles
+ *    associated to previous levels in the hierarchy are removed from the code title.
+ *
+ *	  H2020-EU.1.2. -> EXCELLENT SCIENCE - Future and Emerging Technologies (FET)
+ *	  H2020-EU.2.2. -> INDUSTRIAL LEADERSHIP - Access to risk finance
+ *    H2020-EU.3.4. -> SOCIETAL CHALLENGES - Smart, Green And Integrated Transport
+ *
+ *    will become
+ *
+ *    H2020-EU.1.2. -> Future and Emerging Technologies (FET)
+ *    H2020-EU.2.2. -> Access to risk finance
+ *    H2020-EU.3.4. -> Smart, Green And Integrated Transport
+ *
+ *    This holds at all levels in the hierarchy. Hence
+ *
+ *    H2020-EU.2.1.2. -> INDUSTRIAL LEADERSHIP - Leadership in enabling and industrial technologies – Nanotechnologies
+ *
+ *    will become
+ *
+ *    H2020-EU.2.1.2. -> Nanotechnologies
+ *
+ *  - Euratom is not given in the way the other programmes are: H2020-EU. but H2020-Euratom- . So we need to write
+ *    specific code for it
+ *
+ *
+ *
+ */
 public class PrepareProgramme {
 
 	private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class);
@@ -69,49 +141,148 @@ public class PrepareProgramme {
 	private static void exec(SparkSession spark, String programmePath, String outputPath) {
 		Dataset<CSVProgramme> programme = readPath(spark, programmePath, CSVProgramme.class);
 
-		programme
+		JavaRDD<CSVProgramme> h2020Programmes = programme
 			.toJavaRDD()
-			.filter(p -> !p.getCode().contains("FP7"))
 			.mapToPair(csvProgramme -> new Tuple2<>(csvProgramme.getCode(), csvProgramme))
 			.reduceByKey((a, b) -> {
-				if (StringUtils.isEmpty(a.getShortTitle())) {
-					if (StringUtils.isEmpty(b.getShortTitle())) {
-						if (StringUtils.isEmpty(a.getTitle())) {
-							if (StringUtils.isNotEmpty(b.getTitle())) {
-								a.setShortTitle(b.getTitle());
-								a.setLanguage(b.getLanguage());
-							}
-						} else {// notIsEmpty a.getTitle
-							if (StringUtils.isEmpty(b.getTitle())) {
-								a.setShortTitle(a.getTitle());
-							} else {
-								if (b.getLanguage().equalsIgnoreCase("en")) {
-									a.setShortTitle(b.getTitle());
-									a.setLanguage(b.getLanguage());
-								} else {
-									a.setShortTitle(a.getTitle());
-								}
-							}
-						}
-					} else {// not isEmpty b.getShortTitle
-						a.setShortTitle(b.getShortTitle());
-						// a.setLanguage(b.getLanguage());
+				if (!a.getLanguage().equals("en")) {
+					if (b.getLanguage().equalsIgnoreCase("en")) {
+						a.setTitle(b.getTitle());
+						a.setLanguage(b.getLanguage());
 					}
 				}
+				if (StringUtils.isEmpty(a.getShortTitle())) {
+					if (!StringUtils.isEmpty(b.getShortTitle())) {
+						a.setShortTitle(b.getShortTitle());
+					}
+				}
+
 				return a;
 
 			})
 			.map(p -> {
 				CSVProgramme csvProgramme = p._2();
-				if (StringUtils.isEmpty(csvProgramme.getShortTitle())) {
-					csvProgramme.setShortTitle(csvProgramme.getTitle());
+				String programmeTitle = csvProgramme.getTitle().trim();
+				if (programmeTitle.length() > 8 && programmeTitle.substring(0, 8).equalsIgnoreCase("PRIORITY")) {
+					programmeTitle = programmeTitle.substring(9);
+					if (programmeTitle.charAt(0) == '\'') {
+						programmeTitle = programmeTitle.substring(1);
+					}
+					if (programmeTitle.charAt(programmeTitle.length() - 1) == '\'') {
+						programmeTitle = programmeTitle.substring(0, programmeTitle.length() - 1);
+					}
+					csvProgramme.setTitle(programmeTitle);
 				}
-				return OBJECT_MAPPER.writeValueAsString(csvProgramme);
+				return csvProgramme;
+			});
+
+		// prepareClassification(h2020Programmes);
+
+		JavaSparkContext jsc = new JavaSparkContext(spark.sparkContext());
+
+		JavaRDD<CSVProgramme> rdd = jsc.parallelize(prepareClassification(h2020Programmes), 1);
+		rdd
+			.map(csvProgramme -> {
+				String tmp = OBJECT_MAPPER.writeValueAsString(csvProgramme);
+				return tmp;
 			})
 			.saveAsTextFile(outputPath);
 
 	}
 
+	private static List<CSVProgramme> prepareClassification(JavaRDD<CSVProgramme> h2020Programmes) {
+		Object[] codedescription = h2020Programmes
+			.map(
+				value -> new Tuple2<>(value.getCode(),
+					new Tuple2<String, String>(value.getTitle(), value.getShortTitle())))
+			.collect()
+			.toArray();
+
+		for (int i = 0; i < codedescription.length - 1; i++) {
+			for (int j = i + 1; j < codedescription.length; j++) {
+				Tuple2<String, Tuple2<String, String>> t2i = (Tuple2<String, Tuple2<String, String>>) codedescription[i];
+				Tuple2<String, Tuple2<String, String>> t2j = (Tuple2<String, Tuple2<String, String>>) codedescription[j];
+				if (t2i._1().compareTo(t2j._1()) > 0) {
+					Tuple2<String, Tuple2<String, String>> temp = t2i;
+					codedescription[i] = t2j;
+					codedescription[j] = temp;
+				}
+			}
+		}
+
+		Map<String, Tuple2<String, String>> map = new HashMap<>();
+		for (int j = 0; j < codedescription.length; j++) {
+			Tuple2<String, Tuple2<String, String>> entry = (Tuple2<String, Tuple2<String, String>>) codedescription[j];
+			String ent = entry._1();
+			if (ent.contains("Euratom-")) {
+				ent = ent.replace("-Euratom-", ".Euratom.");
+			}
+			String[] tmp = ent.split("\\.");
+			if (tmp.length <= 2) {
+				if (StringUtils.isEmpty(entry._2()._2())) {
+					map.put(entry._1(), new Tuple2<String, String>(entry._2()._1(), entry._2()._1()));
+				} else {
+					map.put(entry._1(), entry._2());
+				}
+			} else {
+				if (ent.endsWith(".")) {
+					ent = ent.substring(0, ent.length() - 1);
+				}
+				String key = ent.substring(0, ent.lastIndexOf(".") + 1);
+				if (key.contains("Euratom")) {
+					key = key.replace(".Euratom.", "-Euratom-");
+					ent = ent.replace(".Euratom.", "-Euratom-");
+					if (key.endsWith("-")) {
+						key = key.substring(0, key.length() - 1);
+					}
+				}
+				String current = entry._2()._1();
+				if (!ent.contains("Euratom")) {
+
+					String parent;
+					String tmp_key = tmp[0] + ".";
+					for (int i = 1; i < tmp.length - 1; i++) {
+						tmp_key += tmp[i] + ".";
+						parent = map.get(tmp_key)._1().toLowerCase().trim();
+						if (parent.contains("|")) {
+							parent = parent.substring(parent.lastIndexOf("|") + 1).trim();
+						}
+						if (current.trim().length() > parent.length()
+							&& current.toLowerCase().trim().substring(0, parent.length()).equals(parent)) {
+							current = current.substring(parent.length() + 1);
+							if (current.trim().charAt(0) == '-' || current.trim().charAt(0) == '–') {
+								current = current.trim().substring(1).trim();
+							}
+
+						}
+					}
+
+				}
+				String shortTitle = entry._2()._2();
+				if (StringUtils.isEmpty(shortTitle)) {
+					shortTitle = current;
+				}
+				Tuple2<String, String> newEntry = new Tuple2<>(map.get(key)._1() + " | " + current,
+					map.get(key)._2() + " | " + shortTitle);
+				map.put(ent + ".", newEntry);
+
+			}
+
+		}
+		return h2020Programmes.map(csvProgramme -> {
+
+			String code = csvProgramme.getCode();
+			if (!code.endsWith(".") && !code.contains("Euratom")
+				&& !code.equals("H2020-EC"))
+				code += ".";
+
+			csvProgramme.setClassification(map.get(code)._1());
+			csvProgramme.setClassification_short(map.get(code)._2());
+
+			return csvProgramme;
+		}).collect();
+	}
+
 	public static <R> Dataset<R> readPath(
 		SparkSession spark, String inputPath, Class<R> clazz) {
 		return spark

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java

@@ -6,9 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 import java.util.*;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.sql.Dataset;
@@ -20,17 +18,21 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme;
-import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject;
+
+import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import scala.Tuple2;
 
+/**
+ * Selects only the relevant information collected with the projects: project grant agreement, project programme code and
+ * project topic code for the projects that are also collected from OpenAIRE.
+ */
 public class PrepareProjects {
 
 	private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class);
 	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-	private static final HashMap<String, CSVProgramme> programmeMap = new HashMap<>();
+
 
 	public static void main(String[] args) throws Exception {
 
@@ -97,10 +99,14 @@ public class PrepareProjects {
 			if (csvProject.isPresent()) {
 
 				String[] programme = csvProject.get().getProgramme().split(";");
+				String topic = csvProject.get().getTopics();
+
 				Arrays
 					.stream(programme)
 					.forEach(p -> {
 						CSVProject proj = new CSVProject();
+						proj.setTopics(topic);
+
 						proj.setProgramme(p);
 						proj.setId(csvProject.get().getId());
 						csvProjectList.add(proj);

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ProjectSubset.java

@@ -3,6 +3,9 @@ package eu.dnetlib.dhp.actionmanager.project;
 
 import java.io.Serializable;
 
+/**
+ * Class to store the grande agreement (code) of the collected projects
+ */
 public class ProjectSubset implements Serializable {
 
 	private String code;
@@ -14,4 +17,5 @@ public class ProjectSubset implements Serializable {
 	public void setCode(String code) {
 		this.code = code;
 	}
+
 }

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/ReadProjectsFromDB.java

@@ -25,6 +25,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.DbClient;
 
+/**
+ * queries the OpenAIRE database to get the grant agreement of projects collected from corda__h2020. The code collected
+ * are written on hdfs using the ProjectSubset model
+ */
 public class ReadProjectsFromDB implements Closeable {
 
 	private final DbClient dbClient;
@@ -33,7 +37,7 @@ public class ReadProjectsFromDB implements Closeable {
 	private final BufferedWriter writer;
 	private final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
-	private final static String query = "SELECT code " +
+	private final static String query = "SELECT code  " +
 		"from projects where id like 'corda__h2020%' ";
 
 	public static void main(final String[] args) throws Exception {
@@ -72,7 +76,6 @@ public class ReadProjectsFromDB implements Closeable {
 		try {
 			ProjectSubset p = new ProjectSubset();
 			p.setCode(rs.getString("code"));
-
 			return Arrays.asList(p);
 
 		} catch (final Exception e) {

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/SparkAtomicActionJob.java

@@ -3,47 +3,53 @@ package eu.dnetlib.dhp.actionmanager.project;
 
 import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.function.Consumer;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.function.MapFunction;
 import org.apache.spark.api.java.function.MapGroupsFunction;
-import org.apache.spark.rdd.SequenceFileRDDFunctions;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
-import org.apache.spark.sql.SaveMode;
 import org.apache.spark.sql.SparkSession;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProgramme;
-import eu.dnetlib.dhp.actionmanager.project.csvutils.CSVProject;
+import eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme;
+import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
+import eu.dnetlib.dhp.actionmanager.project.utils.EXCELTopic;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
 import eu.dnetlib.dhp.schema.action.AtomicAction;
 import eu.dnetlib.dhp.schema.common.ModelSupport;
-import eu.dnetlib.dhp.schema.oaf.Programme;
+import eu.dnetlib.dhp.schema.oaf.H2020Classification;
+import eu.dnetlib.dhp.schema.oaf.H2020Programme;
 import eu.dnetlib.dhp.schema.oaf.Project;
 import eu.dnetlib.dhp.utils.DHPUtils;
-import scala.Function1;
 import scala.Tuple2;
-import scala.runtime.BoxedUnit;
 
+/**
+ * Class that makes the ActionSet. To prepare the AS two joins are needed
+ *
+ *  1. join betweem the collected project subset and the programme extenden with the classification on the grant agreement.
+ *     For each entry a
+ *     eu.dnetlib.dhp.Project entity is created and the information about H2020Classification is set together with the
+ *     h2020topiccode variable
+ *  2. join between the output of the previous step and the topic information on the topic code. Each time a match is
+ *     found the h2020topicdescription variable is set.
+ *
+ * To produce one single entry for each project code a step of groupoing is needed: each project can be associated to more
+ * than one programme.
+ *
+ *
+ */
 public class SparkAtomicActionJob {
 	private static final Logger log = LoggerFactory.getLogger(SparkAtomicActionJob.class);
 	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -77,6 +83,9 @@ public class SparkAtomicActionJob {
 		final String programmePath = parser.get("programmePath");
 		log.info("programmePath {}: ", programmePath);
 
+		final String topicPath = parser.get("topicPath");
+		log.info("topic path {}: ", topicPath);
+
 		SparkConf conf = new SparkConf();
 
 		runWithSparkSession(
@@ -88,6 +97,7 @@ public class SparkAtomicActionJob {
 					spark,
 					projectPath,
 					programmePath,
+					topicPath,
 					outputPath);
 			});
 	}
@@ -98,31 +108,54 @@ public class SparkAtomicActionJob {
 
 	private static void getAtomicActions(SparkSession spark, String projectPatH,
 		String programmePath,
+		String topicPath,
 		String outputPath) {
 
 		Dataset<CSVProject> project = readPath(spark, projectPatH, CSVProject.class);
 		Dataset<CSVProgramme> programme = readPath(spark, programmePath, CSVProgramme.class);
+		Dataset<EXCELTopic> topic = readPath(spark, topicPath, EXCELTopic.class);
 
-		project
+		Dataset<Project> aaproject = project
 			.joinWith(programme, project.col("programme").equalTo(programme.col("code")), "left")
-			.map(c -> {
-				CSVProject csvProject = c._1();
-				Optional<CSVProgramme> csvProgramme = Optional.ofNullable(c._2());
-				if (csvProgramme.isPresent()) {
-					Project p = new Project();
-					p
-						.setId(
-							createOpenaireId(
-								ModelSupport.entityIdPrefix.get("project"),
-								"corda__h2020", csvProject.getId()));
-					Programme pm = new Programme();
-					pm.setCode(csvProject.getProgramme());
-					pm.setDescription(csvProgramme.get().getShortTitle());
-					p.setProgramme(Arrays.asList(pm));
-					return p;
-				}
+			.map((MapFunction<Tuple2<CSVProject, CSVProgramme>, Project>) c -> {
 
-				return null;
+				CSVProject csvProject = c._1();
+
+				return Optional
+					.ofNullable(c._2())
+					.map(csvProgramme -> {
+						Project pp = new Project();
+						pp
+							.setId(
+								createOpenaireId(
+									ModelSupport.entityIdPrefix.get("project"),
+									"corda__h2020", csvProject.getId()));
+						pp.setH2020topiccode(csvProject.getTopics());
+						H2020Programme pm = new H2020Programme();
+						H2020Classification h2020classification = new H2020Classification();
+						pm.setCode(csvProject.getProgramme());
+						h2020classification.setClassification(csvProgramme.getClassification());
+						h2020classification.setH2020Programme(pm);
+						setLevelsandProgramme(h2020classification, csvProgramme.getClassification_short());
+						// setProgramme(h2020classification, ocsvProgramme.get().getClassification());
+						pp.setH2020classification(Arrays.asList(h2020classification));
+
+						return pp;
+					})
+					.orElse(null);
+
+			}, Encoders.bean(Project.class))
+			.filter(Objects::nonNull);
+
+		aaproject
+			.joinWith(topic, aaproject.col("h2020topiccode").equalTo(topic.col("code")), "left")
+			.map((MapFunction<Tuple2<Project, EXCELTopic>, Project>) p -> {
+				Optional<EXCELTopic> op = Optional.ofNullable(p._2());
+				Project rp = p._1();
+				if (op.isPresent()) {
+					rp.setH2020topicdescription(op.get().getTitle());
+				}
+				return rp;
 			}, Encoders.bean(Project.class))
 			.filter(Objects::nonNull)
 			.groupByKey(
@@ -144,6 +177,24 @@ public class SparkAtomicActionJob {
 
 	}
 
+	private static void setLevelsandProgramme(H2020Classification h2020Classification, String classification_short) {
+		String[] tmp = classification_short.split(" \\| ");
+		h2020Classification.setLevel1(tmp[0]);
+		if (tmp.length > 1) {
+			h2020Classification.setLevel2(tmp[1]);
+		}
+		if (tmp.length > 2) {
+			h2020Classification.setLevel3(tmp[2]);
+		}
+		h2020Classification.getH2020Programme().setDescription(tmp[tmp.length - 1]);
+	}
+
+//	private static void setProgramme(H2020Classification h2020Classification, String classification) {
+//		String[] tmp = classification.split(" \\| ");
+//
+//		h2020Classification.getH2020Programme().setDescription(tmp[tmp.length - 1]);
+//	}
+
 	public static <R> Dataset<R> readPath(
 		SparkSession spark, String inputPath, Class<R> clazz) {
 		return spark

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVParser.java

@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.actionmanager.project.csvutils;
+package eu.dnetlib.dhp.actionmanager.project.utils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -10,6 +10,9 @@ import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVRecord;
 import org.apache.commons.lang.reflect.FieldUtils;
 
+/**
+ * Reads a generic csv and maps it into classes that mirror its schema
+ */
 public class CSVParser {
 
 	public <R> List<R> parse(String csvFile, String classForName)