WIP: materialize graph as Hive DB

.gitignore

@@ -7,8 +7,6 @@
 *.iws
 *~
 .vscode
-.metals
-.bloop
 .classpath
 /*/.classpath
 /*/*/.classpath
@@ -26,5 +24,4 @@
 spark-warehouse
 /**/job-override.properties
 /**/*.log
-/**/.factorypath
 

dhp-build/dhp-code-style/pom.xml

@@ -15,12 +15,12 @@
         <snapshotRepository>
             <id>dnet45-snapshots</id>
             <name>DNet45 Snapshots</name>
-            <url>https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots</url>
+            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots</url>
             <layout>default</layout>
         </snapshotRepository>
         <repository>
             <id>dnet45-releases</id>
-            <url>https://maven.d4science.org/nexus/content/repositories/dnet45-releases</url>
+            <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases</url>
         </repository>
     </distributionManagement>
 

dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_dnet.xml

@@ -19,7 +19,7 @@
         <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
-        <setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="false"/>
+        <setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
         <setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>

dhp-common/pom.xml

@@ -6,8 +6,7 @@
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dhp</artifactId>
 		<version>1.2.4-SNAPSHOT</version>
-		<relativePath>../pom.xml</relativePath>
-
+		<relativePath>../</relativePath>
 	</parent>
 
 	<artifactId>dhp-common</artifactId>
@@ -21,10 +20,6 @@
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-common</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>com.github.sisyphsu</groupId>
-			<artifactId>dateparser</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-core_2.11</artifactId>
@@ -58,6 +53,11 @@
 			<groupId>com.fasterxml.jackson.core</groupId>
 			<artifactId>jackson-databind</artifactId>
 		</dependency>
+		<!-- https://mvnrepository.com/artifact/com.rabbitmq/amqp-client -->
+		<dependency>
+			<groupId>com.rabbitmq</groupId>
+			<artifactId>amqp-client</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>net.sf.saxon</groupId>
 			<artifactId>Saxon-HE</artifactId>
@@ -87,31 +87,6 @@
 			<groupId>org.postgresql</groupId>
 			<artifactId>postgresql</artifactId>
 		</dependency>
-
-		<dependency>
-			<groupId>com.squareup.okhttp3</groupId>
-			<artifactId>okhttp</artifactId>
-		</dependency>
-
-		<dependency>
-			<groupId>eu.dnetlib</groupId>
-			<artifactId>dnet-pace-core</artifactId>
-		</dependency>
-
-		<dependency>
-			<groupId>org.apache.httpcomponents</groupId>
-			<artifactId>httpclient</artifactId>
-		</dependency>
-
-		<dependency>
-			<groupId>org.mongodb</groupId>
-			<artifactId>mongo-java-driver</artifactId>
-		</dependency>
-
-		<dependency>
-			<groupId>eu.dnetlib.dhp</groupId>
-			<artifactId>dhp-schemas</artifactId>
-		</dependency>
 	</dependencies>
 
 </project>

dhp-common/src/main/java/eu/dnetlib/collector/worker/model/ApiDescriptor.java

@@ -1,5 +1,5 @@
 
-package eu.dnetlib.dhp.collection;
+package eu.dnetlib.collector.worker.model;
 
 import java.util.HashMap;
 import java.util.Map;

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStore.java

@@ -0,0 +1,119 @@
+
+package eu.dnetlib.data.mdstore.manager.common.model;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "mdstores")
+public class MDStore implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = 3160530489149700055L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "format")
+	private String format;
+
+	@Column(name = "layout")
+	private String layout;
+
+	@Column(name = "interpretation")
+	private String interpretation;
+
+	@Column(name = "datasource_name")
+	private String datasourceName;
+
+	@Column(name = "datasource_id")
+	private String datasourceId;
+
+	@Column(name = "api_id")
+	private String apiId;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+	public String getDatasourceName() {
+		return datasourceName;
+	}
+
+	public void setDatasourceName(final String datasourceName) {
+		this.datasourceName = datasourceName;
+	}
+
+	public String getDatasourceId() {
+		return datasourceId;
+	}
+
+	public void setDatasourceId(final String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	public String getApiId() {
+		return apiId;
+	}
+
+	public void setApiId(final String apiId) {
+		this.apiId = apiId;
+	}
+
+	public static MDStore newInstance(
+		final String format, final String layout, final String interpretation) {
+		return newInstance(format, layout, interpretation, null, null, null);
+	}
+
+	public static MDStore newInstance(
+		final String format,
+		final String layout,
+		final String interpretation,
+		final String dsName,
+		final String dsId,
+		final String apiId) {
+		final MDStore md = new MDStore();
+		md.setId("md-" + UUID.randomUUID());
+		md.setFormat(format);
+		md.setLayout(layout);
+		md.setInterpretation(interpretation);
+		md.setDatasourceName(dsName);
+		md.setDatasourceId(dsId);
+		md.setApiId(apiId);
+		return md;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreCurrentVersion.java

@@ -0,0 +1,51 @@
+
+package eu.dnetlib.data.mdstore.manager.common.model;
+
+import java.io.Serializable;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "mdstore_current_versions")
+public class MDStoreCurrentVersion implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = -4757725888593745773L;
+
+	@Id
+	@Column(name = "mdstore")
+	private String mdstore;
+
+	@Column(name = "current_version")
+	private String currentVersion;
+
+	public String getMdstore() {
+		return mdstore;
+	}
+
+	public void setMdstore(final String mdstore) {
+		this.mdstore = mdstore;
+	}
+
+	public String getCurrentVersion() {
+		return currentVersion;
+	}
+
+	public void setCurrentVersion(final String currentVersion) {
+		this.currentVersion = currentVersion;
+	}
+
+	public static MDStoreCurrentVersion newInstance(final String mdId, final String versionId) {
+		final MDStoreCurrentVersion cv = new MDStoreCurrentVersion();
+		cv.setMdstore(mdId);
+		cv.setCurrentVersion(versionId);
+		return cv;
+	}
+
+	public static MDStoreCurrentVersion newInstance(final MDStoreVersion v) {
+		return newInstance(v.getMdstore(), v.getId());
+	}
+}

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreVersion.java

@@ -0,0 +1,99 @@
+
+package eu.dnetlib.data.mdstore.manager.common.model;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+@Entity
+@Table(name = "mdstore_versions")
+public class MDStoreVersion implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = -4763494442274298339L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "mdstore")
+	private String mdstore;
+
+	@Column(name = "writing")
+	private boolean writing;
+
+	@Column(name = "readcount")
+	private int readCount = 0;
+
+	@Column(name = "lastupdate")
+	@Temporal(TemporalType.TIMESTAMP)
+	private Date lastUpdate;
+
+	@Column(name = "size")
+	private long size = 0;
+
+	public static MDStoreVersion newInstance(final String mdId, final boolean writing) {
+		final MDStoreVersion t = new MDStoreVersion();
+		t.setId(mdId + "-" + new Date().getTime());
+		t.setMdstore(mdId);
+		t.setLastUpdate(null);
+		t.setWriting(writing);
+		t.setReadCount(0);
+		t.setSize(0);
+		return t;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getMdstore() {
+		return mdstore;
+	}
+
+	public void setMdstore(final String mdstore) {
+		this.mdstore = mdstore;
+	}
+
+	public boolean isWriting() {
+		return writing;
+	}
+
+	public void setWriting(final boolean writing) {
+		this.writing = writing;
+	}
+
+	public int getReadCount() {
+		return readCount;
+	}
+
+	public void setReadCount(final int readCount) {
+		this.readCount = readCount;
+	}
+
+	public Date getLastUpdate() {
+		return lastUpdate;
+	}
+
+	public void setLastUpdate(final Date lastUpdate) {
+		this.lastUpdate = lastUpdate;
+	}
+
+	public long getSize() {
+		return size;
+	}
+
+	public void setSize(final long size) {
+		this.size = size;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/data/mdstore/manager/common/model/MDStoreWithInfo.java

@@ -0,0 +1,143 @@
+
+package eu.dnetlib.data.mdstore.manager.common.model;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+@Entity
+@Table(name = "mdstores_with_info")
+public class MDStoreWithInfo implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = -8445784770687571492L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "format")
+	private String format;
+
+	@Column(name = "layout")
+	private String layout;
+
+	@Column(name = "interpretation")
+	private String interpretation;
+
+	@Column(name = "datasource_name")
+	private String datasourceName;
+
+	@Column(name = "datasource_id")
+	private String datasourceId;
+
+	@Column(name = "api_id")
+	private String apiId;
+
+	@Column(name = "current_version")
+	private String currentVersion;
+
+	@Column(name = "lastupdate")
+	@Temporal(TemporalType.TIMESTAMP)
+	private Date lastUpdate;
+
+	@Column(name = "size")
+	private long size = 0;
+
+	@Column(name = "n_versions")
+	private long numberOfVersions = 0;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+	public String getDatasourceName() {
+		return datasourceName;
+	}
+
+	public void setDatasourceName(final String datasourceName) {
+		this.datasourceName = datasourceName;
+	}
+
+	public String getDatasourceId() {
+		return datasourceId;
+	}
+
+	public void setDatasourceId(final String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	public String getApiId() {
+		return apiId;
+	}
+
+	public void setApiId(final String apiId) {
+		this.apiId = apiId;
+	}
+
+	public String getCurrentVersion() {
+		return currentVersion;
+	}
+
+	public void setCurrentVersion(final String currentVersion) {
+		this.currentVersion = currentVersion;
+	}
+
+	public Date getLastUpdate() {
+		return lastUpdate;
+	}
+
+	public void setLastUpdate(final Date lastUpdate) {
+		this.lastUpdate = lastUpdate;
+	}
+
+	public long getSize() {
+		return size;
+	}
+
+	public void setSize(final long size) {
+		this.size = size;
+	}
+
+	public long getNumberOfVersions() {
+		return numberOfVersions;
+	}
+
+	public void setNumberOfVersions(final long numberOfVersions) {
+		this.numberOfVersions = numberOfVersions;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/application/ApplicationUtils.java

@@ -1,14 +0,0 @@
-
-package eu.dnetlib.dhp.application;
-
-import java.io.*;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.collect.Maps;
-
-public class ApplicationUtils {
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/application/ArgumentApplicationParser.java

@@ -1,7 +1,10 @@
 
 package eu.dnetlib.dhp.application;
 
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Serializable;
+import java.io.StringWriter;
 import java.util.*;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
@@ -9,21 +12,17 @@ import java.util.zip.GZIPOutputStream;
 import org.apache.commons.cli.*;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 public class ArgumentApplicationParser implements Serializable {
 
-	private static final Logger log = LoggerFactory.getLogger(ArgumentApplicationParser.class);
-
 	private final Options options = new Options();
 	private final Map<String, String> objectMap = new HashMap<>();
 
 	private final List<String> compressedValues = new ArrayList<>();
 
-	public ArgumentApplicationParser(final String json_configuration) throws IOException {
+	public ArgumentApplicationParser(final String json_configuration) throws Exception {
 		final ObjectMapper mapper = new ObjectMapper();
 		final OptionsParameter[] configuration = mapper.readValue(json_configuration, OptionsParameter[].class);
 		createOptionMap(configuration);
@@ -34,6 +33,7 @@ public class ArgumentApplicationParser implements Serializable {
 	}
 
 	private void createOptionMap(final OptionsParameter[] configuration) {
+
 		Arrays
 			.stream(configuration)
 			.map(
@@ -47,6 +47,10 @@ public class ArgumentApplicationParser implements Serializable {
 					return o;
 				})
 			.forEach(options::addOption);
+
+		// HelpFormatter formatter = new HelpFormatter();
+		// formatter.printHelp("myapp", null, options, null, true);
+
 	}
 
 	public static String decompressValue(final String abstractCompressed) {
@@ -57,7 +61,7 @@ public class ArgumentApplicationParser implements Serializable {
 			IOUtils.copy(gis, stringWriter);
 			return stringWriter.toString();
 		} catch (Throwable e) {
-			log.error("Wrong value to decompress:" + abstractCompressed);
+			System.out.println("Wrong value to decompress:" + abstractCompressed);
 			throw new RuntimeException(e);
 		}
 	}
@@ -70,7 +74,7 @@ public class ArgumentApplicationParser implements Serializable {
 		return java.util.Base64.getEncoder().encodeToString(out.toByteArray());
 	}
 
-	public void parseArgument(final String[] args) throws ParseException {
+	public void parseArgument(final String[] args) throws Exception {
 		CommandLineParser parser = new BasicParser();
 		CommandLine cmd = parser.parse(options, args);
 		Arrays

dhp-common/src/main/java/eu/dnetlib/dhp/common/Constants.java

@@ -1,52 +0,0 @@
-
-package eu.dnetlib.dhp.common;
-
-import java.util.Map;
-
-import com.google.common.collect.Maps;
-
-public class Constants {
-
-	public static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
-	public static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
-
-	public static String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
-
-	static {
-		accessRightsCoarMap.put("OPEN", "c_abf2");
-		accessRightsCoarMap.put("RESTRICTED", "c_16ec");
-		accessRightsCoarMap.put("OPEN SOURCE", "c_abf2");
-		accessRightsCoarMap.put("CLOSED", "c_14cb");
-		accessRightsCoarMap.put("EMBARGO", "c_f1cf");
-	}
-
-	static {
-		coarCodeLabelMap.put("c_abf2", "OPEN");
-		coarCodeLabelMap.put("c_16ec", "RESTRICTED");
-		coarCodeLabelMap.put("c_14cb", "CLOSED");
-		coarCodeLabelMap.put("c_f1cf", "EMBARGO");
-	}
-
-	public static final String SEQUENCE_FILE_NAME = "/sequence_file";
-	public static final String REPORT_FILE_NAME = "/report";
-	public static final String MDSTORE_DATA_PATH = "/store";
-	public static final String MDSTORE_SIZE_PATH = "/size";
-
-	public static final String COLLECTION_MODE = "collectionMode";
-	public static final String METADATA_ENCODING = "metadataEncoding";
-	public static final String OOZIE_WF_PATH = "oozieWfPath";
-	public static final String DNET_MESSAGE_MGR_URL = "dnetMessageManagerURL";
-
-	public static final String MAX_NUMBER_OF_RETRY = "maxNumberOfRetry";
-	public static final String REQUEST_DELAY = "requestDelay";
-	public static final String RETRY_DELAY = "retryDelay";
-	public static final String CONNECT_TIMEOUT = "connectTimeOut";
-	public static final String READ_TIMEOUT = "readTimeOut";
-	public static final String FROM_DATE_OVERRIDE = "fromDateOverride";
-	public static final String UNTIL_DATE_OVERRIDE = "untilDateOverride";
-
-	public static final String CONTENT_TOTALITEMS = "TotalItems";
-	public static final String CONTENT_INVALIDRECORDS = "InvalidRecords";
-	public static final String CONTENT_TRANSFORMEDRECORDS = "transformedItems";
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/DbClient.java

@@ -14,7 +14,7 @@ public class DbClient implements Closeable {
 
 	private static final Log log = LogFactory.getLog(DbClient.class);
 
-	private final Connection connection;
+	private Connection connection;
 
 	public DbClient(final String address, final String login, final String password) {
 

dhp-common/src/main/java/eu/dnetlib/dhp/common/GraphResultMapper.java

@@ -1,412 +0,0 @@
-
-package eu.dnetlib.dhp.common;
-
-import java.io.Serializable;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.dump.oaf.*;
-import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityInstance;
-import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
-import eu.dnetlib.dhp.schema.oaf.DataInfo;
-import eu.dnetlib.dhp.schema.oaf.Field;
-import eu.dnetlib.dhp.schema.oaf.Journal;
-import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
-
-public class GraphResultMapper implements Serializable {
-
-	public static <E extends eu.dnetlib.dhp.schema.oaf.OafEntity> Result map(
-		E in) {
-
-		CommunityResult out = new CommunityResult();
-
-		eu.dnetlib.dhp.schema.oaf.Result input = (eu.dnetlib.dhp.schema.oaf.Result) in;
-		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> ort = Optional.ofNullable(input.getResulttype());
-		if (ort.isPresent()) {
-			switch (ort.get().getClassid()) {
-				case "publication":
-					Optional<Journal> journal = Optional
-						.ofNullable(((eu.dnetlib.dhp.schema.oaf.Publication) input).getJournal());
-					if (journal.isPresent()) {
-						Journal j = journal.get();
-						Container c = new Container();
-						c.setConferencedate(j.getConferencedate());
-						c.setConferenceplace(j.getConferenceplace());
-						c.setEdition(j.getEdition());
-						c.setEp(j.getEp());
-						c.setIss(j.getIss());
-						c.setIssnLinking(j.getIssnLinking());
-						c.setIssnOnline(j.getIssnOnline());
-						c.setIssnPrinted(j.getIssnPrinted());
-						c.setName(j.getName());
-						c.setSp(j.getSp());
-						c.setVol(j.getVol());
-						out.setContainer(c);
-						out.setType(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE.getClassname());
-					}
-					break;
-				case "dataset":
-					eu.dnetlib.dhp.schema.oaf.Dataset id = (eu.dnetlib.dhp.schema.oaf.Dataset) input;
-					Optional.ofNullable(id.getSize()).ifPresent(v -> out.setSize(v.getValue()));
-					Optional.ofNullable(id.getVersion()).ifPresent(v -> out.setVersion(v.getValue()));
-
-					out
-						.setGeolocation(
-							Optional
-								.ofNullable(id.getGeolocation())
-								.map(
-									igl -> igl
-										.stream()
-										.filter(Objects::nonNull)
-										.map(gli -> {
-											GeoLocation gl = new GeoLocation();
-											gl.setBox(gli.getBox());
-											gl.setPlace(gli.getPlace());
-											gl.setPoint(gli.getPoint());
-											return gl;
-										})
-										.collect(Collectors.toList()))
-								.orElse(null));
-
-					out.setType(ModelConstants.DATASET_DEFAULT_RESULTTYPE.getClassname());
-					break;
-				case "software":
-
-					eu.dnetlib.dhp.schema.oaf.Software is = (eu.dnetlib.dhp.schema.oaf.Software) input;
-					Optional
-						.ofNullable(is.getCodeRepositoryUrl())
-						.ifPresent(value -> out.setCodeRepositoryUrl(value.getValue()));
-					Optional
-						.ofNullable(is.getDocumentationUrl())
-						.ifPresent(
-							value -> out
-								.setDocumentationUrl(
-									value
-										.stream()
-										.map(v -> v.getValue())
-										.collect(Collectors.toList())));
-
-					Optional
-						.ofNullable(is.getProgrammingLanguage())
-						.ifPresent(value -> out.setProgrammingLanguage(value.getClassid()));
-
-					out.setType(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE.getClassname());
-					break;
-				case "other":
-
-					eu.dnetlib.dhp.schema.oaf.OtherResearchProduct ir = (eu.dnetlib.dhp.schema.oaf.OtherResearchProduct) input;
-					out
-						.setContactgroup(
-							Optional
-								.ofNullable(ir.getContactgroup())
-								.map(value -> value.stream().map(cg -> cg.getValue()).collect(Collectors.toList()))
-								.orElse(null));
-
-					out
-						.setContactperson(
-							Optional
-								.ofNullable(ir.getContactperson())
-								.map(value -> value.stream().map(cp -> cp.getValue()).collect(Collectors.toList()))
-								.orElse(null));
-					out
-						.setTool(
-							Optional
-								.ofNullable(ir.getTool())
-								.map(value -> value.stream().map(t -> t.getValue()).collect(Collectors.toList()))
-								.orElse(null));
-
-					out.setType(ModelConstants.ORP_DEFAULT_RESULTTYPE.getClassname());
-
-					break;
-			}
-
-			Optional
-				.ofNullable(input.getAuthor())
-				.ifPresent(ats -> out.setAuthor(ats.stream().map(at -> getAuthor(at)).collect(Collectors.toList())));
-
-			// I do not map Access Right UNKNOWN or OTHER
-
-			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oar = Optional.ofNullable(input.getBestaccessright());
-			if (oar.isPresent()) {
-				if (Constants.accessRightsCoarMap.containsKey(oar.get().getClassid())) {
-					String code = Constants.accessRightsCoarMap.get(oar.get().getClassid());
-					out
-						.setBestaccessright(
-							AccessRight
-								.newInstance(
-									code,
-									Constants.coarCodeLabelMap.get(code),
-									Constants.COAR_ACCESS_RIGHT_SCHEMA));
-				}
-			}
-
-			final List<String> contributorList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getContributor())
-				.ifPresent(value -> value.stream().forEach(c -> contributorList.add(c.getValue())));
-			out.setContributor(contributorList);
-
-			Optional
-				.ofNullable(input.getCountry())
-				.ifPresent(
-					value -> out
-						.setCountry(
-							value
-								.stream()
-								.map(
-									c -> {
-										if (c.getClassid().equals((ModelConstants.UNKNOWN))) {
-											return null;
-										}
-										Country country = new Country();
-										country.setCode(c.getClassid());
-										country.setLabel(c.getClassname());
-										Optional
-											.ofNullable(c.getDataInfo())
-											.ifPresent(
-												provenance -> country
-													.setProvenance(
-														Provenance
-															.newInstance(
-																provenance
-																	.getProvenanceaction()
-																	.getClassname(),
-																c.getDataInfo().getTrust())));
-										return country;
-									})
-								.filter(Objects::nonNull)
-								.collect(Collectors.toList())));
-
-			final List<String> coverageList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getCoverage())
-				.ifPresent(value -> value.stream().forEach(c -> coverageList.add(c.getValue())));
-			out.setCoverage(coverageList);
-
-			out.setDateofcollection(input.getDateofcollection());
-
-			final List<String> descriptionList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getDescription())
-				.ifPresent(value -> value.forEach(d -> descriptionList.add(d.getValue())));
-			out.setDescription(descriptionList);
-			Optional<Field<String>> oStr = Optional.ofNullable(input.getEmbargoenddate());
-			if (oStr.isPresent()) {
-				out.setEmbargoenddate(oStr.get().getValue());
-			}
-
-			final List<String> formatList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getFormat())
-				.ifPresent(value -> value.stream().forEach(f -> formatList.add(f.getValue())));
-			out.setFormat(formatList);
-			out.setId(input.getId());
-			out.setOriginalId(input.getOriginalId());
-
-			Optional<List<eu.dnetlib.dhp.schema.oaf.Instance>> oInst = Optional
-				.ofNullable(input.getInstance());
-
-			if (oInst.isPresent()) {
-				out
-					.setInstance(
-						oInst.get().stream().map(i -> getInstance(i)).collect(Collectors.toList()));
-
-			}
-
-			Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> oL = Optional.ofNullable(input.getLanguage());
-			if (oL.isPresent()) {
-				eu.dnetlib.dhp.schema.oaf.Qualifier language = oL.get();
-				out.setLanguage(Qualifier.newInstance(language.getClassid(), language.getClassname()));
-			}
-			Optional<Long> oLong = Optional.ofNullable(input.getLastupdatetimestamp());
-			if (oLong.isPresent()) {
-				out.setLastupdatetimestamp(oLong.get());
-			}
-			Optional<List<StructuredProperty>> otitle = Optional.ofNullable(input.getTitle());
-			if (otitle.isPresent()) {
-				List<StructuredProperty> iTitle = otitle
-					.get()
-					.stream()
-					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("main title"))
-					.collect(Collectors.toList());
-				if (iTitle.size() > 0) {
-					out.setMaintitle(iTitle.get(0).getValue());
-				}
-
-				iTitle = otitle
-					.get()
-					.stream()
-					.filter(t -> t.getQualifier().getClassid().equalsIgnoreCase("subtitle"))
-					.collect(Collectors.toList());
-				if (iTitle.size() > 0) {
-					out.setSubtitle(iTitle.get(0).getValue());
-				}
-
-			}
-
-			List<ControlledField> pids = new ArrayList<>();
-			Optional
-				.ofNullable(input.getPid())
-				.ifPresent(
-					value -> value
-						.stream()
-						.forEach(
-							p -> pids
-								.add(
-									ControlledField
-										.newInstance(p.getQualifier().getClassid(), p.getValue()))));
-			out.setPid(pids);
-			oStr = Optional.ofNullable(input.getDateofacceptance());
-			if (oStr.isPresent()) {
-				out.setPublicationdate(oStr.get().getValue());
-			}
-			oStr = Optional.ofNullable(input.getPublisher());
-			if (oStr.isPresent()) {
-				out.setPublisher(oStr.get().getValue());
-			}
-
-			List<String> sourceList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getSource())
-				.ifPresent(value -> value.stream().forEach(s -> sourceList.add(s.getValue())));
-			// out.setSource(input.getSource().stream().map(s -> s.getValue()).collect(Collectors.toList()));
-			List<Subject> subjectList = new ArrayList<>();
-			Optional
-				.ofNullable(input.getSubject())
-				.ifPresent(
-					value -> value
-						.forEach(s -> subjectList.add(getSubject(s))));
-
-			out.setSubjects(subjectList);
-
-			out.setType(input.getResulttype().getClassid());
-		}
-
-		out
-			.setCollectedfrom(
-				input
-					.getCollectedfrom()
-					.stream()
-					.map(cf -> KeyValue.newInstance(cf.getKey(), cf.getValue()))
-					.collect(Collectors.toList()));
-
-		return out;
-
-	}
-
-	private static CommunityInstance getInstance(eu.dnetlib.dhp.schema.oaf.Instance i) {
-		CommunityInstance instance = new CommunityInstance();
-
-		setCommonValue(i, instance);
-
-		instance
-			.setCollectedfrom(
-				KeyValue
-					.newInstance(i.getCollectedfrom().getKey(), i.getCollectedfrom().getValue()));
-
-		instance
-			.setHostedby(
-				KeyValue.newInstance(i.getHostedby().getKey(), i.getHostedby().getValue()));
-
-		return instance;
-
-	}
-
-	private static <I extends Instance> void setCommonValue(eu.dnetlib.dhp.schema.oaf.Instance i, I instance) {
-		Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
-			.ofNullable(i.getAccessright());
-		if (opAr.isPresent()) {
-			if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
-				String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
-				instance
-					.setAccessright(
-						AccessRight
-							.newInstance(
-								code,
-								Constants.coarCodeLabelMap.get(code),
-								Constants.COAR_ACCESS_RIGHT_SCHEMA));
-			}
-		}
-
-		Optional
-			.ofNullable(i.getLicense())
-			.ifPresent(value -> instance.setLicense(value.getValue()));
-		Optional
-			.ofNullable(i.getDateofacceptance())
-			.ifPresent(value -> instance.setPublicationdate(value.getValue()));
-		Optional
-			.ofNullable(i.getRefereed())
-			.ifPresent(value -> instance.setRefereed(value.getClassname()));
-		Optional
-			.ofNullable(i.getInstancetype())
-			.ifPresent(value -> instance.setType(value.getClassname()));
-		Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
-
-	}
-
-	private static Subject getSubject(StructuredProperty s) {
-		Subject subject = new Subject();
-		subject.setSubject(ControlledField.newInstance(s.getQualifier().getClassid(), s.getValue()));
-		Optional<DataInfo> di = Optional.ofNullable(s.getDataInfo());
-		if (di.isPresent()) {
-			Provenance p = new Provenance();
-			p.setProvenance(di.get().getProvenanceaction().getClassname());
-			p.setTrust(di.get().getTrust());
-			subject.setProvenance(p);
-		}
-
-		return subject;
-	}
-
-	private static Author getAuthor(eu.dnetlib.dhp.schema.oaf.Author oa) {
-		Author a = new Author();
-		a.setFullname(oa.getFullname());
-		a.setName(oa.getName());
-		a.setSurname(oa.getSurname());
-		a.setRank(oa.getRank());
-
-		Optional<List<StructuredProperty>> oPids = Optional
-			.ofNullable(oa.getPid());
-		if (oPids.isPresent()) {
-			Pid pid = getOrcid(oPids.get());
-			if (pid != null) {
-				a.setPid(pid);
-			}
-		}
-
-		return a;
-	}
-
-	private static Pid getOrcid(List<StructuredProperty> p) {
-		for (StructuredProperty pid : p) {
-			if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID)) {
-				Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
-				if (di.isPresent()) {
-					return Pid
-						.newInstance(
-							ControlledField
-								.newInstance(
-									pid.getQualifier().getClassid(),
-									pid.getValue()),
-							Provenance
-								.newInstance(
-									di.get().getProvenanceaction().getClassname(),
-									di.get().getTrust()));
-				} else {
-					return Pid
-						.newInstance(
-							ControlledField
-								.newInstance(
-									pid.getQualifier().getClassid(),
-									pid.getValue())
-
-						);
-				}
-
-			}
-		}
-		return null;
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/MakeTarArchive.java

@@ -1,114 +0,0 @@
-
-package eu.dnetlib.dhp.common;
-
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Serializable;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.hadoop.fs.*;
-
-public class MakeTarArchive implements Serializable {
-
-	private static TarArchiveOutputStream getTar(FileSystem fileSystem, String outputPath) throws IOException {
-		Path hdfsWritePath = new Path(outputPath);
-		FSDataOutputStream fsDataOutputStream = null;
-		if (fileSystem.exists(hdfsWritePath)) {
-			fileSystem.delete(hdfsWritePath, true);
-
-		}
-		fsDataOutputStream = fileSystem.create(hdfsWritePath);
-
-		return new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
-	}
-
-	private static void write(FileSystem fileSystem, String inputPath, String outputPath, String dir_name)
-		throws IOException {
-
-		Path hdfsWritePath = new Path(outputPath);
-		FSDataOutputStream fsDataOutputStream = null;
-		if (fileSystem.exists(hdfsWritePath)) {
-			fileSystem.delete(hdfsWritePath, true);
-
-		}
-		fsDataOutputStream = fileSystem.create(hdfsWritePath);
-
-		TarArchiveOutputStream ar = new TarArchiveOutputStream(fsDataOutputStream.getWrappedStream());
-
-		RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
-			.listFiles(
-				new Path(inputPath), true);
-
-		while (fileStatusListIterator.hasNext()) {
-			writeCurrentFile(fileSystem, dir_name, fileStatusListIterator, ar, 0);
-		}
-
-		ar.close();
-	}
-
-	public static void tarMaxSize(FileSystem fileSystem, String inputPath, String outputPath, String dir_name,
-		int gBperSplit) throws IOException {
-		final long bytesPerSplit = 1024L * 1024L * 1024L * gBperSplit;
-
-		long sourceSize = fileSystem.getContentSummary(new Path(inputPath)).getSpaceConsumed();
-
-		if (sourceSize < bytesPerSplit) {
-			write(fileSystem, inputPath, outputPath + ".tar", dir_name);
-		} else {
-			int partNum = 0;
-
-			RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
-				.listFiles(
-					new Path(inputPath), true);
-			boolean next = fileStatusListIterator.hasNext();
-			while (next) {
-				TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar");
-
-				long current_size = 0;
-				while (next && current_size < bytesPerSplit) {
-					current_size = writeCurrentFile(fileSystem, dir_name, fileStatusListIterator, ar, current_size);
-					next = fileStatusListIterator.hasNext();
-
-				}
-
-				partNum += 1;
-				ar.close();
-			}
-
-		}
-
-	}
-
-	private static long writeCurrentFile(FileSystem fileSystem, String dir_name,
-		RemoteIterator<LocatedFileStatus> fileStatusListIterator,
-		TarArchiveOutputStream ar, long current_size) throws IOException {
-		LocatedFileStatus fileStatus = fileStatusListIterator.next();
-
-		Path p = fileStatus.getPath();
-		String p_string = p.toString();
-		if (!p_string.endsWith("_SUCCESS")) {
-			String name = p_string.substring(p_string.lastIndexOf("/") + 1);
-			TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
-			entry.setSize(fileStatus.getLen());
-			current_size += fileStatus.getLen();
-			ar.putArchiveEntry(entry);
-
-			InputStream is = fileSystem.open(fileStatus.getPath());
-
-			BufferedInputStream bis = new BufferedInputStream(is);
-
-			int count;
-			byte[] data = new byte[1024];
-			while ((count = bis.read(data, 0, data.length)) != -1) {
-				ar.write(data, 0, count);
-			}
-			bis.close();
-			ar.closeArchiveEntry();
-
-		}
-		return current_size;
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java

@@ -1,53 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import okhttp3.MediaType;
-import okhttp3.RequestBody;
-import okhttp3.internal.Util;
-import okio.BufferedSink;
-import okio.Okio;
-import okio.Source;
-
-public class InputStreamRequestBody extends RequestBody {
-
-	private final InputStream inputStream;
-	private final MediaType mediaType;
-	private final long lenght;
-
-	public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
-
-		return new InputStreamRequestBody(inputStream, mediaType, len);
-	}
-
-	private InputStreamRequestBody(InputStream inputStream, MediaType mediaType, long len) {
-		this.inputStream = inputStream;
-		this.mediaType = mediaType;
-		this.lenght = len;
-	}
-
-	@Override
-	public MediaType contentType() {
-		return mediaType;
-	}
-
-	@Override
-	public long contentLength() {
-
-		return lenght;
-
-	}
-
-	@Override
-	public void writeTo(BufferedSink sink) throws IOException {
-		Source source = null;
-		try {
-			source = Okio.source(inputStream);
-			sink.writeAll(source);
-		} finally {
-			Util.closeQuietly(source);
-		}
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java

@@ -1,8 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-public class MissingConceptDoiException extends Throwable {
-	public MissingConceptDoiException(String message) {
-		super(message);
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java

@@ -1,315 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.*;
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-import com.google.gson.Gson;
-
-import eu.dnetlib.dhp.common.api.zenodo.ZenodoModel;
-import eu.dnetlib.dhp.common.api.zenodo.ZenodoModelList;
-import okhttp3.*;
-
-public class ZenodoAPIClient implements Serializable {
-
-	String urlString;
-	String bucket;
-
-	String deposition_id;
-	String access_token;
-
-	public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
-
-	private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip");
-
-	public String getUrlString() {
-		return urlString;
-	}
-
-	public void setUrlString(String urlString) {
-		this.urlString = urlString;
-	}
-
-	public String getBucket() {
-		return bucket;
-	}
-
-	public void setBucket(String bucket) {
-		this.bucket = bucket;
-	}
-
-	public void setDeposition_id(String deposition_id) {
-		this.deposition_id = deposition_id;
-	}
-
-	public ZenodoAPIClient(String urlString, String access_token) throws IOException {
-
-		this.urlString = urlString;
-		this.access_token = access_token;
-	}
-
-	/**
-	 * Brand new deposition in Zenodo. It sets the deposition_id and the bucket where to store the files to upload
-	 *
-	 * @return response code
-	 * @throws IOException
-	 */
-	public int newDeposition() throws IOException {
-		String json = "{}";
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
-
-		Request request = new Request.Builder()
-			.url(urlString)
-			.addHeader("Content-Type", "application/json") // add request headers
-			.addHeader("Authorization", "Bearer " + access_token)
-			.post(body)
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			// Get response body
-			json = response.body().string();
-
-			ZenodoModel newSubmission = new Gson().fromJson(json, ZenodoModel.class);
-			this.bucket = newSubmission.getLinks().getBucket();
-			this.deposition_id = newSubmission.getId();
-
-			return response.code();
-
-		}
-
-	}
-
-	/**
-	 * Upload files in Zenodo.
-	 *
-	 * @param is the inputStream for the file to upload
-	 * @param file_name the name of the file as it will appear on Zenodo
-	 * @param len the size of the file
-	 * @return the response code
-	 */
-	public int uploadIS(InputStream is, String file_name, long len) throws IOException {
-		OkHttpClient httpClient = new OkHttpClient.Builder()
-			.writeTimeout(600, TimeUnit.SECONDS)
-			.readTimeout(600, TimeUnit.SECONDS)
-			.connectTimeout(600, TimeUnit.SECONDS)
-			.build();
-
-		Request request = new Request.Builder()
-			.url(bucket + "/" + file_name)
-			.addHeader("Content-Type", "application/zip") // add request headers
-			.addHeader("Authorization", "Bearer " + access_token)
-			.put(InputStreamRequestBody.create(MEDIA_TYPE_ZIP, is, len))
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-			return response.code();
-		}
-	}
-
-	/**
-	 * Associates metadata information to the current deposition
-	 *
-	 * @param metadata the metadata
-	 * @return response code
-	 * @throws IOException
-	 */
-	public int sendMretadata(String metadata) throws IOException {
-
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		RequestBody body = RequestBody.create(metadata, MEDIA_TYPE_JSON);
-
-		Request request = new Request.Builder()
-			.url(urlString + "/" + deposition_id)
-			.addHeader("Content-Type", "application/json") // add request headers
-			.addHeader("Authorization", "Bearer " + access_token)
-			.put(body)
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			return response.code();
-
-		}
-
-	}
-
-	/**
-	 * To publish the current deposition. It works for both new deposition or new version of an old deposition
-	 *
-	 * @return response code
-	 * @throws IOException
-	 */
-	public int publish() throws IOException {
-
-		String json = "{}";
-
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
-
-		Request request = new Request.Builder()
-			.url(urlString + "/" + deposition_id + "/actions/publish")
-			.addHeader("Authorization", "Bearer " + access_token)
-			.post(body)
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			return response.code();
-
-		}
-	}
-
-	/**
-	 * To create a new version of an already published deposition. It sets the deposition_id and the bucket to be used
-	 * for the new version.
-	 *
-	 * @param concept_rec_id the concept record id of the deposition for which to create a new version. It is the last
-	 *            part of the url for the DOI Zenodo suggests to use to cite all versions: DOI: 10.xxx/zenodo.656930
-	 *            concept_rec_id = 656930
-	 * @return response code
-	 * @throws IOException
-	 * @throws MissingConceptDoiException
-	 */
-	public int newVersion(String concept_rec_id) throws IOException, MissingConceptDoiException {
-		setDepositionId(concept_rec_id);
-		String json = "{}";
-
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
-
-		Request request = new Request.Builder()
-			.url(urlString + "/" + deposition_id + "/actions/newversion")
-			.addHeader("Authorization", "Bearer " + access_token)
-			.post(body)
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
-			String latest_draft = zenodoModel.getLinks().getLatest_draft();
-			deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1);
-			bucket = getBucket(latest_draft);
-			return response.code();
-
-		}
-	}
-
-	/**
-	 * To finish uploading a version or new deposition not published
-	 * It sets the deposition_id and the bucket to be used
-	 *
-	 *
-	 * @param deposition_id the deposition id of the not yet published upload
-	 *            concept_rec_id = 656930
-	 * @return response code
-	 * @throws IOException
-	 * @throws MissingConceptDoiException
-	 */
-	public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
-
-		this.deposition_id = deposition_id;
-
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		Request request = new Request.Builder()
-			.url(urlString + "/" + deposition_id)
-			.addHeader("Authorization", "Bearer " + access_token)
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
-			bucket = zenodoModel.getLinks().getBucket();
-			return response.code();
-
-		}
-
-	}
-
-	private void setDepositionId(String concept_rec_id) throws IOException, MissingConceptDoiException {
-
-		ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class);
-
-		for (ZenodoModel zm : zenodoModelList) {
-			if (zm.getConceptrecid().equals(concept_rec_id)) {
-				deposition_id = zm.getId();
-				return;
-			}
-		}
-
-		throw new MissingConceptDoiException("The concept record id specified was missing in the list of depositions");
-
-	}
-
-	private String getPrevDepositions() throws IOException {
-		OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
-
-		Request request = new Request.Builder()
-			.url(urlString)
-			.addHeader("Content-Type", "application/json") // add request headers
-			.addHeader("Authorization", "Bearer " + access_token)
-			.get()
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			return response.body().string();
-
-		}
-
-	}
-
-	private String getBucket(String url) throws IOException {
-		OkHttpClient httpClient = new OkHttpClient.Builder()
-			.connectTimeout(600, TimeUnit.SECONDS)
-			.build();
-
-		Request request = new Request.Builder()
-			.url(url)
-			.addHeader("Content-Type", "application/json") // add request headers
-			.addHeader("Authorization", "Bearer " + access_token)
-			.get()
-			.build();
-
-		try (Response response = httpClient.newCall(request).execute()) {
-
-			if (!response.isSuccessful())
-				throw new IOException("Unexpected code " + response + response.body().string());
-
-			// Get response body
-			ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
-
-			return zenodoModel.getLinks().getBucket();
-
-		}
-
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java

@@ -1,14 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-public class Community {
-	private String identifier;
-
-	public String getIdentifier() {
-		return identifier;
-	}
-
-	public void setIdentifier(String identifier) {
-		this.identifier = identifier;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java

@@ -1,47 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-public class Creator {
-	private String affiliation;
-	private String name;
-	private String orcid;
-
-	public String getAffiliation() {
-		return affiliation;
-	}
-
-	public void setAffiliation(String affiliation) {
-		this.affiliation = affiliation;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public void setName(String name) {
-		this.name = name;
-	}
-
-	public String getOrcid() {
-		return orcid;
-	}
-
-	public void setOrcid(String orcid) {
-		this.orcid = orcid;
-	}
-
-	public static Creator newInstance(String name, String affiliation, String orcid) {
-		Creator c = new Creator();
-		if (!(name == null)) {
-			c.name = name;
-		}
-		if (!(affiliation == null)) {
-			c.affiliation = affiliation;
-		}
-		if (!(orcid == null)) {
-			c.orcid = orcid;
-		}
-
-		return c;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java

@@ -1,58 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-import net.minidev.json.annotate.JsonIgnore;
-
-public class File implements Serializable {
-	private String checksum;
-	private String filename;
-	private long filesize;
-	private String id;
-
-	@JsonIgnore
-	// private Links links;
-
-	public String getChecksum() {
-		return checksum;
-	}
-
-	public void setChecksum(String checksum) {
-		this.checksum = checksum;
-	}
-
-	public String getFilename() {
-		return filename;
-	}
-
-	public void setFilename(String filename) {
-		this.filename = filename;
-	}
-
-	public long getFilesize() {
-		return filesize;
-	}
-
-	public void setFilesize(long filesize) {
-		this.filesize = filesize;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-//    @JsonIgnore
-//    public Links getLinks() {
-//        return links;
-//    }
-//
-//    @JsonIgnore
-//    public void setLinks(Links links) {
-//        this.links = links;
-//    }
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java

@@ -1,23 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class Grant implements Serializable {
-	private String id;
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public static Grant newInstance(String id) {
-		Grant g = new Grant();
-		g.id = id;
-
-		return g;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java

@@ -1,92 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class Links implements Serializable {
-
-	private String bucket;
-
-	private String discard;
-
-	private String edit;
-	private String files;
-	private String html;
-	private String latest_draft;
-	private String latest_draft_html;
-	private String publish;
-
-	private String self;
-
-	public String getBucket() {
-		return bucket;
-	}
-
-	public void setBucket(String bucket) {
-		this.bucket = bucket;
-	}
-
-	public String getDiscard() {
-		return discard;
-	}
-
-	public void setDiscard(String discard) {
-		this.discard = discard;
-	}
-
-	public String getEdit() {
-		return edit;
-	}
-
-	public void setEdit(String edit) {
-		this.edit = edit;
-	}
-
-	public String getFiles() {
-		return files;
-	}
-
-	public void setFiles(String files) {
-		this.files = files;
-	}
-
-	public String getHtml() {
-		return html;
-	}
-
-	public void setHtml(String html) {
-		this.html = html;
-	}
-
-	public String getLatest_draft() {
-		return latest_draft;
-	}
-
-	public void setLatest_draft(String latest_draft) {
-		this.latest_draft = latest_draft;
-	}
-
-	public String getLatest_draft_html() {
-		return latest_draft_html;
-	}
-
-	public void setLatest_draft_html(String latest_draft_html) {
-		this.latest_draft_html = latest_draft_html;
-	}
-
-	public String getPublish() {
-		return publish;
-	}
-
-	public void setPublish(String publish) {
-		this.publish = publish;
-	}
-
-	public String getSelf() {
-		return self;
-	}
-
-	public void setSelf(String self) {
-		this.self = self;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java

@@ -1,153 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class Metadata implements Serializable {
-
-	private String access_right;
-	private List<Community> communities;
-	private List<Creator> creators;
-	private String description;
-	private String doi;
-	private List<Grant> grants;
-	private List<String> keywords;
-	private String language;
-	private String license;
-	private PrereserveDoi prereserve_doi;
-	private String publication_date;
-	private List<String> references;
-	private List<RelatedIdentifier> related_identifiers;
-	private String title;
-	private String upload_type;
-	private String version;
-
-	public String getUpload_type() {
-		return upload_type;
-	}
-
-	public void setUpload_type(String upload_type) {
-		this.upload_type = upload_type;
-	}
-
-	public String getVersion() {
-		return version;
-	}
-
-	public void setVersion(String version) {
-		this.version = version;
-	}
-
-	public String getAccess_right() {
-		return access_right;
-	}
-
-	public void setAccess_right(String access_right) {
-		this.access_right = access_right;
-	}
-
-	public List<Community> getCommunities() {
-		return communities;
-	}
-
-	public void setCommunities(List<Community> communities) {
-		this.communities = communities;
-	}
-
-	public List<Creator> getCreators() {
-		return creators;
-	}
-
-	public void setCreators(List<Creator> creators) {
-		this.creators = creators;
-	}
-
-	public String getDescription() {
-		return description;
-	}
-
-	public void setDescription(String description) {
-		this.description = description;
-	}
-
-	public String getDoi() {
-		return doi;
-	}
-
-	public void setDoi(String doi) {
-		this.doi = doi;
-	}
-
-	public List<Grant> getGrants() {
-		return grants;
-	}
-
-	public void setGrants(List<Grant> grants) {
-		this.grants = grants;
-	}
-
-	public List<String> getKeywords() {
-		return keywords;
-	}
-
-	public void setKeywords(List<String> keywords) {
-		this.keywords = keywords;
-	}
-
-	public String getLanguage() {
-		return language;
-	}
-
-	public void setLanguage(String language) {
-		this.language = language;
-	}
-
-	public String getLicense() {
-		return license;
-	}
-
-	public void setLicense(String license) {
-		this.license = license;
-	}
-
-	public PrereserveDoi getPrereserve_doi() {
-		return prereserve_doi;
-	}
-
-	public void setPrereserve_doi(PrereserveDoi prereserve_doi) {
-		this.prereserve_doi = prereserve_doi;
-	}
-
-	public String getPublication_date() {
-		return publication_date;
-	}
-
-	public void setPublication_date(String publication_date) {
-		this.publication_date = publication_date;
-	}
-
-	public List<String> getReferences() {
-		return references;
-	}
-
-	public void setReferences(List<String> references) {
-		this.references = references;
-	}
-
-	public List<RelatedIdentifier> getRelated_identifiers() {
-		return related_identifiers;
-	}
-
-	public void setRelated_identifiers(List<RelatedIdentifier> related_identifiers) {
-		this.related_identifiers = related_identifiers;
-	}
-
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java

@@ -1,25 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class PrereserveDoi implements Serializable {
-	private String doi;
-	private String recid;
-
-	public String getDoi() {
-		return doi;
-	}
-
-	public void setDoi(String doi) {
-		this.doi = doi;
-	}
-
-	public String getRecid() {
-		return recid;
-	}
-
-	public void setRecid(String recid) {
-		this.recid = recid;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java

@@ -1,43 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-
-public class RelatedIdentifier implements Serializable {
-	private String identifier;
-	private String relation;
-	private String resource_type;
-	private String scheme;
-
-	public String getIdentifier() {
-		return identifier;
-	}
-
-	public void setIdentifier(String identifier) {
-		this.identifier = identifier;
-	}
-
-	public String getRelation() {
-		return relation;
-	}
-
-	public void setRelation(String relation) {
-		this.relation = relation;
-	}
-
-	public String getResource_type() {
-		return resource_type;
-	}
-
-	public void setResource_type(String resource_type) {
-		this.resource_type = resource_type;
-	}
-
-	public String getScheme() {
-		return scheme;
-	}
-
-	public void setScheme(String scheme) {
-		this.scheme = scheme;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java

@@ -1,118 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class ZenodoModel implements Serializable {
-
-	private String conceptrecid;
-	private String created;
-
-	private List<File> files;
-	private String id;
-	private Links links;
-	private Metadata metadata;
-	private String modified;
-	private String owner;
-	private String record_id;
-	private String state;
-	private boolean submitted;
-	private String title;
-
-	public String getConceptrecid() {
-		return conceptrecid;
-	}
-
-	public void setConceptrecid(String conceptrecid) {
-		this.conceptrecid = conceptrecid;
-	}
-
-	public String getCreated() {
-		return created;
-	}
-
-	public void setCreated(String created) {
-		this.created = created;
-	}
-
-	public List<File> getFiles() {
-		return files;
-	}
-
-	public void setFiles(List<File> files) {
-		this.files = files;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public Links getLinks() {
-		return links;
-	}
-
-	public void setLinks(Links links) {
-		this.links = links;
-	}
-
-	public Metadata getMetadata() {
-		return metadata;
-	}
-
-	public void setMetadata(Metadata metadata) {
-		this.metadata = metadata;
-	}
-
-	public String getModified() {
-		return modified;
-	}
-
-	public void setModified(String modified) {
-		this.modified = modified;
-	}
-
-	public String getOwner() {
-		return owner;
-	}
-
-	public void setOwner(String owner) {
-		this.owner = owner;
-	}
-
-	public String getRecord_id() {
-		return record_id;
-	}
-
-	public void setRecord_id(String record_id) {
-		this.record_id = record_id;
-	}
-
-	public String getState() {
-		return state;
-	}
-
-	public void setState(String state) {
-		this.state = state;
-	}
-
-	public boolean isSubmitted() {
-		return submitted;
-	}
-
-	public void setSubmitted(boolean submitted) {
-		this.submitted = submitted;
-	}
-
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java

@@ -1,7 +0,0 @@
-
-package eu.dnetlib.dhp.common.api.zenodo;
-
-import java.util.ArrayList;
-
-public class ZenodoModelList extends ArrayList<ZenodoModel> {
-}

dhp-common/src/main/java/eu/dnetlib/dhp/common/rest/DNetRestClient.java

@@ -1,72 +0,0 @@
-
-package eu.dnetlib.dhp.common.rest;
-
-import java.util.Arrays;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-public class DNetRestClient {
-
-	private static final Logger log = LoggerFactory.getLogger(DNetRestClient.class);
-
-	private static final ObjectMapper mapper = new ObjectMapper();
-
-	public static <T> T doGET(final String url, Class<T> clazz) throws Exception {
-		final HttpGet httpGet = new HttpGet(url);
-		return doHTTPRequest(httpGet, clazz);
-	}
-
-	public static String doGET(final String url) throws Exception {
-		final HttpGet httpGet = new HttpGet(url);
-		return doHTTPRequest(httpGet);
-	}
-
-	public static <V> String doPOST(final String url, V objParam) throws Exception {
-		final HttpPost httpPost = new HttpPost(url);
-
-		if (objParam != null) {
-			final StringEntity entity = new StringEntity(mapper.writeValueAsString(objParam));
-			httpPost.setEntity(entity);
-			httpPost.setHeader("Accept", "application/json");
-			httpPost.setHeader("Content-type", "application/json");
-		}
-		return doHTTPRequest(httpPost);
-	}
-
-	public static <T, V> T doPOST(final String url, V objParam, Class<T> clazz) throws Exception {
-		return mapper.readValue(doPOST(url, objParam), clazz);
-	}
-
-	private static String doHTTPRequest(final HttpUriRequest r) throws Exception {
-		CloseableHttpClient client = HttpClients.createDefault();
-
-		log.info("performing HTTP request, method {} on URI {}", r.getMethod(), r.getURI().toString());
-		log
-			.info(
-				"request headers: {}",
-				Arrays
-					.asList(r.getAllHeaders())
-					.stream()
-					.map(h -> h.getName() + ":" + h.getValue())
-					.collect(Collectors.joining(",")));
-
-		CloseableHttpResponse response = client.execute(r);
-		return IOUtils.toString(response.getEntity().getContent());
-	}
-
-	private static <T> T doHTTPRequest(final HttpUriRequest r, Class<T> clazz) throws Exception {
-		return mapper.readValue(doHTTPRequest(r), clazz);
-	}
-}

dhp-common/src/main/java/eu/dnetlib/dhp/message/Message.java

@@ -1,64 +0,0 @@
-
-package eu.dnetlib.dhp.message;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class Message implements Serializable {
-
-	private static final long serialVersionUID = 401753881204524893L;
-
-	public static String CURRENT_PARAM = "current";
-	public static String TOTAL_PARAM = "total";
-
-	private MessageType messageType;
-
-	private String workflowId;
-
-	private Map<String, String> body;
-
-	public Message() {
-	}
-
-	public Message(final MessageType messageType, final String workflowId) {
-		this(messageType, workflowId, new LinkedHashMap<>());
-	}
-
-	public Message(final MessageType messageType, final String workflowId, final Map<String, String> body) {
-		this.messageType = messageType;
-		this.workflowId = workflowId;
-		this.body = body;
-	}
-
-	public MessageType getMessageType() {
-		return messageType;
-	}
-
-	public void setMessageType(MessageType messageType) {
-		this.messageType = messageType;
-	}
-
-	public String getWorkflowId() {
-		return workflowId;
-	}
-
-	public void setWorkflowId(final String workflowId) {
-		this.workflowId = workflowId;
-	}
-
-	public Map<String, String> getBody() {
-		return body;
-	}
-
-	public void setBody(final Map<String, String> body) {
-		this.body = body;
-	}
-
-	@Override
-	public String toString() {
-		return String.format("Message [type=%s, workflowId=%s, body=%s]", messageType, workflowId, body);
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/message/MessageSender.java

@@ -1,94 +0,0 @@
-
-package eu.dnetlib.dhp.message;
-
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-public class MessageSender {
-
-	private static final Logger log = LoggerFactory.getLogger(MessageSender.class);
-
-	private static final int SOCKET_TIMEOUT_MS = 2000;
-
-	private static final int CONNECTION_REQUEST_TIMEOUT_MS = 2000;
-
-	private static final int CONNTECTION_TIMEOUT_MS = 2000;
-
-	private final ObjectMapper objectMapper = new ObjectMapper();
-
-	private final String dnetMessageEndpoint;
-
-	private final String workflowId;
-
-	private final ExecutorService executorService = Executors.newCachedThreadPool();
-
-	public MessageSender(final String dnetMessageEndpoint, final String workflowId) {
-		this.workflowId = workflowId;
-		this.dnetMessageEndpoint = dnetMessageEndpoint;
-	}
-
-	public void sendMessage(final Message message) {
-		executorService.submit(() -> _sendMessage(message));
-	}
-
-	public void sendMessage(final Long current, final Long total) {
-		sendMessage(createOngoingMessage(current, total));
-	}
-
-	public void sendReport(final Map<String, String> report) {
-		sendMessage(new Message(MessageType.REPORT, workflowId, report));
-	}
-
-	private Message createOngoingMessage(final Long current, final Long total) {
-		final Message m = new Message(MessageType.ONGOING, workflowId);
-		m.getBody().put(Message.CURRENT_PARAM, current.toString());
-		if (total != null) {
-			m.getBody().put(Message.TOTAL_PARAM, total.toString());
-		}
-		return m;
-	}
-
-	private void _sendMessage(final Message message) {
-		try {
-			final String json = objectMapper.writeValueAsString(message);
-
-			final HttpPut req = new HttpPut(dnetMessageEndpoint);
-			req.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
-
-			final RequestConfig requestConfig = RequestConfig
-				.custom()
-				.setConnectTimeout(CONNTECTION_TIMEOUT_MS)
-				.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MS)
-				.setSocketTimeout(SOCKET_TIMEOUT_MS)
-				.build();
-
-			try (final CloseableHttpClient client = HttpClients
-				.custom()
-				.setDefaultRequestConfig(requestConfig)
-				.build();
-				final CloseableHttpResponse response = client.execute(req)) {
-				log.debug("Sent Message to " + dnetMessageEndpoint);
-				log.debug("MESSAGE:" + message);
-			} catch (final Throwable e) {
-				log.error("Error sending message to " + dnetMessageEndpoint + ", message content: " + message, e);
-			}
-		} catch (final JsonProcessingException e) {
-			log.error("Error sending message to " + dnetMessageEndpoint + ", message content: " + message, e);
-		}
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/message/MessageType.java

@@ -1,21 +0,0 @@
-
-package eu.dnetlib.dhp.message;
-
-import java.io.Serializable;
-import java.util.Optional;
-
-import org.apache.commons.lang3.StringUtils;
-
-public enum MessageType implements Serializable {
-
-	ONGOING, REPORT;
-
-	public MessageType from(String value) {
-		return Optional
-			.ofNullable(value)
-			.map(StringUtils::upperCase)
-			.map(MessageType::valueOf)
-			.orElseThrow(() -> new IllegalArgumentException("unknown message type: " + value));
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/MetadataRecord.java

@@ -0,0 +1,121 @@
+
+package eu.dnetlib.dhp.model.mdstore;
+
+import java.io.Serializable;
+
+import eu.dnetlib.dhp.utils.DHPUtils;
+
+/** This class models a record inside the new Metadata store collection on HDFS * */
+public class MetadataRecord implements Serializable {
+
+	/** The D-Net Identifier associated to the record */
+	private String id;
+
+	/** The original Identifier of the record */
+	private String originalId;
+
+	/** The encoding of the record, should be JSON or XML */
+	private String encoding;
+
+	/**
+	 * The information about the provenance of the record see @{@link Provenance} for the model of this information
+	 */
+	private Provenance provenance;
+
+	/** The content of the metadata */
+	private String body;
+
+	/** the date when the record has been stored */
+	private long dateOfCollection;
+
+	/** the date when the record has been stored */
+	private long dateOfTransformation;
+
+	public MetadataRecord() {
+		this.dateOfCollection = System.currentTimeMillis();
+	}
+
+	public MetadataRecord(
+		String originalId,
+		String encoding,
+		Provenance provenance,
+		String body,
+		long dateOfCollection) {
+
+		this.originalId = originalId;
+		this.encoding = encoding;
+		this.provenance = provenance;
+		this.body = body;
+		this.dateOfCollection = dateOfCollection;
+		this.id = DHPUtils.generateIdentifier(originalId, this.provenance.getNsPrefix());
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public String getOriginalId() {
+		return originalId;
+	}
+
+	public void setOriginalId(String originalId) {
+		this.originalId = originalId;
+	}
+
+	public String getEncoding() {
+		return encoding;
+	}
+
+	public void setEncoding(String encoding) {
+		this.encoding = encoding;
+	}
+
+	public Provenance getProvenance() {
+		return provenance;
+	}
+
+	public void setProvenance(Provenance provenance) {
+		this.provenance = provenance;
+	}
+
+	public String getBody() {
+		return body;
+	}
+
+	public void setBody(String body) {
+		this.body = body;
+	}
+
+	public long getDateOfCollection() {
+		return dateOfCollection;
+	}
+
+	public void setDateOfCollection(long dateOfCollection) {
+		this.dateOfCollection = dateOfCollection;
+	}
+
+	public long getDateOfTransformation() {
+		return dateOfTransformation;
+	}
+
+	public void setDateOfTransformation(long dateOfTransformation) {
+		this.dateOfTransformation = dateOfTransformation;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (!(o instanceof MetadataRecord)) {
+			return false;
+		}
+		return ((MetadataRecord) o).getId().equalsIgnoreCase(id);
+	}
+
+	@Override
+	public int hashCode() {
+		return id.hashCode();
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/model/mdstore/Provenance.java

@@ -0,0 +1,52 @@
+
+package eu.dnetlib.dhp.model.mdstore;
+
+import java.io.Serializable;
+
+/**
+ * @author Sandro La Bruzzo
+ *         <p>
+ *         Provenace class models the provenance of the record in the metadataStore It contains the identifier and the
+ *         name of the datasource that gives the record
+ */
+public class Provenance implements Serializable {
+
+	private String datasourceId;
+
+	private String datasourceName;
+
+	private String nsPrefix;
+
+	public Provenance() {
+	}
+
+	public Provenance(String datasourceId, String datasourceName, String nsPrefix) {
+		this.datasourceId = datasourceId;
+		this.datasourceName = datasourceName;
+		this.nsPrefix = nsPrefix;
+	}
+
+	public String getDatasourceId() {
+		return datasourceId;
+	}
+
+	public void setDatasourceId(String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	public String getDatasourceName() {
+		return datasourceName;
+	}
+
+	public void setDatasourceName(String datasourceName) {
+		this.datasourceName = datasourceName;
+	}
+
+	public String getNsPrefix() {
+		return nsPrefix;
+	}
+
+	public void setNsPrefix(String nsPrefix) {
+		this.nsPrefix = nsPrefix;
+	}
+}

dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java

@@ -1,459 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-import java.time.LocalDate;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeParseException;
-import java.util.*;
-import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.commons.lang3.StringUtils;
-import org.jetbrains.annotations.NotNull;
-
-import com.github.sisyphsu.dateparser.DateParserUtils;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.common.ModelSupport;
-import eu.dnetlib.dhp.schema.oaf.*;
-
-public class GraphCleaningFunctions extends CleaningFunctions {
-
-	public static final String ORCID_CLEANING_REGEX = ".*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9x]{4})";
-	public static final int ORCID_LEN = 19;
-	public static final String CLEANING_REGEX = "(?:\\n|\\r|\\t)";
-	public static final String INVALID_AUTHOR_REGEX = ".*deactivated.*";
-	public static final String TITLE_FILTER_REGEX = "[.*test.*\\W\\d]";
-	public static final int TITLE_FILTER_RESIDUAL_LENGTH = 10;
-
-	public static <T extends Oaf> T fixVocabularyNames(T value) {
-		if (value instanceof Datasource) {
-			// nothing to clean here
-		} else if (value instanceof Project) {
-			// nothing to clean here
-		} else if (value instanceof Organization) {
-			Organization o = (Organization) value;
-			if (Objects.nonNull(o.getCountry())) {
-				fixVocabName(o.getCountry(), ModelConstants.DNET_COUNTRY_TYPE);
-			}
-		} else if (value instanceof Relation) {
-			// nothing to clean here
-		} else if (value instanceof Result) {
-
-			Result r = (Result) value;
-
-			fixVocabName(r.getLanguage(), ModelConstants.DNET_LANGUAGES);
-			fixVocabName(r.getResourcetype(), ModelConstants.DNET_DATA_CITE_RESOURCE);
-			fixVocabName(r.getBestaccessright(), ModelConstants.DNET_ACCESS_MODES);
-
-			if (Objects.nonNull(r.getSubject())) {
-				r.getSubject().forEach(s -> fixVocabName(s.getQualifier(), ModelConstants.DNET_SUBJECT_TYPOLOGIES));
-			}
-			if (Objects.nonNull(r.getInstance())) {
-				for (Instance i : r.getInstance()) {
-					fixVocabName(i.getAccessright(), ModelConstants.DNET_ACCESS_MODES);
-					fixVocabName(i.getRefereed(), ModelConstants.DNET_REVIEW_LEVELS);
-				}
-			}
-			if (Objects.nonNull(r.getAuthor())) {
-				r.getAuthor().stream().filter(Objects::nonNull).forEach(a -> {
-					if (Objects.nonNull(a.getPid())) {
-						a.getPid().stream().filter(Objects::nonNull).forEach(p -> {
-							fixVocabName(p.getQualifier(), ModelConstants.DNET_PID_TYPES);
-						});
-					}
-				});
-			}
-			if (value instanceof Publication) {
-
-			} else if (value instanceof Dataset) {
-
-			} else if (value instanceof OtherResearchProduct) {
-
-			} else if (value instanceof Software) {
-
-			}
-		}
-
-		return value;
-	}
-
-	public static <T extends Oaf> boolean filter(T value) {
-		if (value instanceof Datasource) {
-			// nothing to evaluate here
-		} else if (value instanceof Project) {
-			// nothing to evaluate here
-		} else if (value instanceof Organization) {
-			// nothing to evaluate here
-		} else if (value instanceof Relation) {
-			// nothing to clean here
-		} else if (value instanceof Result) {
-
-			Result r = (Result) value;
-
-			if (Objects.isNull(r.getTitle()) || r.getTitle().isEmpty()) {
-				return false;
-			}
-
-			if (value instanceof Publication) {
-
-			} else if (value instanceof Dataset) {
-
-			} else if (value instanceof OtherResearchProduct) {
-
-			} else if (value instanceof Software) {
-
-			}
-		}
-		return true;
-	}
-
-	public static <T extends Oaf> T cleanup(T value) {
-		if (value instanceof Datasource) {
-			// nothing to clean here
-		} else if (value instanceof Project) {
-			// nothing to clean here
-		} else if (value instanceof Organization) {
-			Organization o = (Organization) value;
-			if (Objects.isNull(o.getCountry()) || StringUtils.isBlank(o.getCountry().getClassid())) {
-				o.setCountry(ModelConstants.UNKNOWN_COUNTRY);
-			}
-		} else if (value instanceof Relation) {
-			Relation r = (Relation) value;
-
-			Optional<String> validationDate = doCleanDate(r.getValidationDate());
-			if (validationDate.isPresent()) {
-				r.setValidationDate(validationDate.get());
-				r.setValidated(true);
-			} else {
-				r.setValidationDate(null);
-				r.setValidated(false);
-			}
-		} else if (value instanceof Result) {
-
-			Result r = (Result) value;
-
-			if (Objects.nonNull(r.getDateofacceptance())) {
-				Optional<String> date = cleanDateField(r.getDateofacceptance());
-				if (date.isPresent()) {
-					r.getDateofacceptance().setValue(date.get());
-				} else {
-					r.setDateofacceptance(null);
-				}
-			}
-			if (Objects.nonNull(r.getRelevantdate())) {
-				r
-					.setRelevantdate(
-						r
-							.getRelevantdate()
-							.stream()
-							.filter(Objects::nonNull)
-							.filter(sp -> Objects.nonNull(sp.getQualifier()))
-							.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
-							.map(sp -> {
-								sp.setValue(GraphCleaningFunctions.cleanDate(sp.getValue()));
-								return sp;
-							})
-							.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
-							.collect(Collectors.toList()));
-			}
-			if (Objects.nonNull(r.getPublisher()) && StringUtils.isBlank(r.getPublisher().getValue())) {
-				r.setPublisher(null);
-			}
-			if (Objects.isNull(r.getLanguage()) || StringUtils.isBlank(r.getLanguage().getClassid())) {
-				r
-					.setLanguage(
-						qualifier("und", "Undetermined", ModelConstants.DNET_LANGUAGES));
-			}
-			if (Objects.nonNull(r.getSubject())) {
-				r
-					.setSubject(
-						r
-							.getSubject()
-							.stream()
-							.filter(Objects::nonNull)
-							.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
-							.filter(sp -> Objects.nonNull(sp.getQualifier()))
-							.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
-							.map(GraphCleaningFunctions::cleanValue)
-							.collect(Collectors.toList()));
-			}
-			if (Objects.nonNull(r.getTitle())) {
-				r
-					.setTitle(
-						r
-							.getTitle()
-							.stream()
-							.filter(Objects::nonNull)
-							.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
-							.filter(
-								sp -> sp
-									.getValue()
-									.toLowerCase()
-									.replaceAll(TITLE_FILTER_REGEX, "")
-									.length() > TITLE_FILTER_RESIDUAL_LENGTH)
-							.map(GraphCleaningFunctions::cleanValue)
-							.collect(Collectors.toList()));
-			}
-			if (Objects.nonNull(r.getDescription())) {
-				r
-					.setDescription(
-						r
-							.getDescription()
-							.stream()
-							.filter(Objects::nonNull)
-							.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
-							.map(GraphCleaningFunctions::cleanValue)
-							.collect(Collectors.toList()));
-			}
-			if (Objects.nonNull(r.getPid())) {
-				r.setPid(processPidCleaning(r.getPid()));
-			}
-			if (Objects.isNull(r.getResourcetype()) || StringUtils.isBlank(r.getResourcetype().getClassid())) {
-				r
-					.setResourcetype(
-						qualifier(ModelConstants.UNKNOWN, "Unknown", ModelConstants.DNET_DATA_CITE_RESOURCE));
-			}
-			if (Objects.nonNull(r.getInstance())) {
-
-				for (Instance i : r.getInstance()) {
-					if (Objects.nonNull(i.getPid())) {
-						i.setPid(processPidCleaning(i.getPid()));
-					}
-					if (Objects.nonNull(i.getAlternateIdentifier())) {
-						i.setAlternateIdentifier(processPidCleaning(i.getAlternateIdentifier()));
-					}
-					Optional
-						.ofNullable(i.getPid())
-						.ifPresent(pid -> {
-							final Set<StructuredProperty> pids = Sets.newHashSet(pid);
-							Optional
-								.ofNullable(i.getAlternateIdentifier())
-								.ifPresent(altId -> {
-									final Set<StructuredProperty> altIds = Sets.newHashSet(altId);
-									i.setAlternateIdentifier(Lists.newArrayList(Sets.difference(altIds, pids)));
-								});
-						});
-
-					if (Objects.isNull(i.getAccessright()) || StringUtils.isBlank(i.getAccessright().getClassid())) {
-						i
-							.setAccessright(
-								accessRight(
-									ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE,
-									ModelConstants.DNET_ACCESS_MODES));
-					}
-					if (Objects.isNull(i.getHostedby()) || StringUtils.isBlank(i.getHostedby().getKey())) {
-						i.setHostedby(ModelConstants.UNKNOWN_REPOSITORY);
-					}
-					if (Objects.isNull(i.getRefereed())) {
-						i.setRefereed(qualifier("0000", "Unknown", ModelConstants.DNET_REVIEW_LEVELS));
-					}
-					if (Objects.nonNull(i.getDateofacceptance())) {
-						Optional<String> date = cleanDateField(i.getDateofacceptance());
-						if (date.isPresent()) {
-							i.getDateofacceptance().setValue(date.get());
-						} else {
-							i.setDateofacceptance(null);
-						}
-					}
-				}
-			}
-			if (Objects.isNull(r.getBestaccessright()) || StringUtils.isBlank(r.getBestaccessright().getClassid())) {
-				Qualifier bestaccessrights = OafMapperUtils.createBestAccessRights(r.getInstance());
-				if (Objects.isNull(bestaccessrights)) {
-					r
-						.setBestaccessright(
-							qualifier(
-								ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE,
-								ModelConstants.DNET_ACCESS_MODES));
-				} else {
-					r.setBestaccessright(bestaccessrights);
-				}
-			}
-			if (Objects.nonNull(r.getAuthor())) {
-				r
-					.setAuthor(
-						r
-							.getAuthor()
-							.stream()
-							.filter(a -> Objects.nonNull(a))
-							.filter(a -> StringUtils.isNotBlank(a.getFullname()))
-							.filter(a -> StringUtils.isNotBlank(a.getFullname().replaceAll("[\\W]", "")))
-							.collect(Collectors.toList()));
-
-				boolean nullRank = r
-					.getAuthor()
-					.stream()
-					.anyMatch(a -> Objects.isNull(a.getRank()));
-				if (nullRank) {
-					int i = 1;
-					for (Author author : r.getAuthor()) {
-						author.setRank(i++);
-					}
-				}
-
-				for (Author a : r.getAuthor()) {
-					if (Objects.isNull(a.getPid())) {
-						a.setPid(Lists.newArrayList());
-					} else {
-						a
-							.setPid(
-								a
-									.getPid()
-									.stream()
-									.filter(Objects::nonNull)
-									.filter(p -> Objects.nonNull(p.getQualifier()))
-									.filter(p -> StringUtils.isNotBlank(p.getValue()))
-									.map(p -> {
-										// hack to distinguish orcid from orcid_pending
-										String pidProvenance = Optional
-											.ofNullable(p.getDataInfo())
-											.map(
-												d -> Optional
-													.ofNullable(d.getProvenanceaction())
-													.map(Qualifier::getClassid)
-													.orElse(""))
-											.orElse("");
-										if (p
-											.getQualifier()
-											.getClassid()
-											.toLowerCase()
-											.contains(ModelConstants.ORCID)) {
-											if (pidProvenance
-												.equals(ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY)) {
-												p.getQualifier().setClassid(ModelConstants.ORCID);
-											} else {
-												p.getQualifier().setClassid(ModelConstants.ORCID_PENDING);
-											}
-											final String orcid = p
-												.getValue()
-												.trim()
-												.toLowerCase()
-												.replaceAll(ORCID_CLEANING_REGEX, "$1-$2-$3-$4");
-											if (orcid.length() == ORCID_LEN) {
-												p.setValue(orcid);
-											} else {
-												p.setValue("");
-											}
-										}
-										return p;
-									})
-									.filter(p -> StringUtils.isNotBlank(p.getValue()))
-									.collect(
-										Collectors
-											.toMap(
-												p -> p.getQualifier().getClassid() + p.getValue(),
-												Function.identity(),
-												(p1, p2) -> p1,
-												LinkedHashMap::new))
-									.values()
-									.stream()
-									.collect(Collectors.toList()));
-					}
-				}
-			}
-			if (value instanceof Publication) {
-
-			} else if (value instanceof Dataset) {
-
-			} else if (value instanceof OtherResearchProduct) {
-
-			} else if (value instanceof Software) {
-
-			}
-		}
-
-		return value;
-	}
-
-	private static Optional<String> cleanDateField(Field<String> dateofacceptance) {
-		return Optional
-			.ofNullable(dateofacceptance)
-			.map(Field::getValue)
-			.map(GraphCleaningFunctions::cleanDate)
-			.filter(Objects::nonNull);
-	}
-
-	protected static Optional<String> doCleanDate(String date) {
-		return Optional.ofNullable(cleanDate(date));
-	}
-
-	public static String cleanDate(final String inputDate) {
-
-		if (StringUtils.isBlank(inputDate)) {
-			return null;
-		}
-
-		try {
-			final LocalDate date = DateParserUtils
-				.parseDate(inputDate.trim())
-				.toInstant()
-				.atZone(ZoneId.systemDefault())
-				.toLocalDate();
-			return DateTimeFormatter.ofPattern(ModelSupport.DATE_FORMAT).format(date);
-		} catch (DateTimeParseException e) {
-			return null;
-		}
-	}
-
-	// HELPERS
-
-	private static boolean isValidAuthorName(Author a) {
-		return !Stream
-			.of(a.getFullname(), a.getName(), a.getSurname())
-			.filter(s -> s != null && !s.isEmpty())
-			.collect(Collectors.joining(""))
-			.toLowerCase()
-			.matches(INVALID_AUTHOR_REGEX);
-	}
-
-	private static List<StructuredProperty> processPidCleaning(List<StructuredProperty> pids) {
-		return pids
-			.stream()
-			.filter(Objects::nonNull)
-			.filter(sp -> StringUtils.isNotBlank(StringUtils.trim(sp.getValue())))
-			.filter(sp -> !PID_BLACKLIST.contains(sp.getValue().trim().toLowerCase()))
-			.filter(sp -> Objects.nonNull(sp.getQualifier()))
-			.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
-			.map(CleaningFunctions::normalizePidValue)
-			.filter(CleaningFunctions::pidFilter)
-			.collect(Collectors.toList());
-	}
-
-	private static void fixVocabName(Qualifier q, String vocabularyName) {
-		if (Objects.nonNull(q) && StringUtils.isBlank(q.getSchemeid())) {
-			q.setSchemeid(vocabularyName);
-			q.setSchemename(vocabularyName);
-		}
-	}
-
-	private static AccessRight accessRight(String classid, String classname, String scheme) {
-		return OafMapperUtils
-			.accessRight(
-				classid, classname, scheme, scheme);
-	}
-
-	private static Qualifier qualifier(String classid, String classname, String scheme) {
-		return OafMapperUtils
-			.qualifier(
-				classid, classname, scheme, scheme);
-	}
-
-	protected static StructuredProperty cleanValue(StructuredProperty s) {
-		s.setValue(s.getValue().replaceAll(CLEANING_REGEX, " "));
-		return s;
-	}
-
-	protected static Field<String> cleanValue(Field<String> s) {
-		s.setValue(s.getValue().replaceAll(CLEANING_REGEX, " "));
-		return s;
-	}
-
-}

dhp-common/src/main/java/eu/dnetlib/dhp/utils/DHPUtils.java

@@ -1,43 +1,23 @@
 
 package eu.dnetlib.dhp.utils;
 
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Base64OutputStream;
 import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.SaveMode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Maps;
 import com.jayway.jsonpath.JsonPath;
 
 import net.minidev.json.JSONArray;
-import scala.collection.JavaConverters;
-import scala.collection.Seq;
 
 public class DHPUtils {
 
-	private static final Logger log = LoggerFactory.getLogger(DHPUtils.class);
-
-	public static Seq<String> toSeq(List<String> list) {
-		return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq();
-	}
-
 	public static String md5(final String s) {
 		try {
 			final MessageDigest md = MessageDigest.getInstance("MD5");
@@ -92,72 +72,4 @@ public class DHPUtils {
 			return "";
 		}
 	}
-
-	public static final ObjectMapper MAPPER = new ObjectMapper();
-
-	public static void writeHdfsFile(final Configuration conf, final String content, final String path)
-		throws IOException {
-
-		log.info("writing file {}, size {}", path, content.length());
-		try (FileSystem fs = FileSystem.get(conf);
-			BufferedOutputStream os = new BufferedOutputStream(fs.create(new Path(path)))) {
-			os.write(content.getBytes(StandardCharsets.UTF_8));
-			os.flush();
-		}
-	}
-
-	public static String readHdfsFile(Configuration conf, String path) throws IOException {
-		log.info("reading file {}", path);
-
-		try (FileSystem fs = FileSystem.get(conf)) {
-			final Path p = new Path(path);
-			if (!fs.exists(p)) {
-				throw new FileNotFoundException(path);
-			}
-			return IOUtils.toString(fs.open(p));
-		}
-	}
-
-	public static <T> T readHdfsFileAs(Configuration conf, String path, Class<T> clazz) throws IOException {
-		return MAPPER.readValue(readHdfsFile(conf, path), clazz);
-	}
-
-	public static <T> void saveDataset(final Dataset<T> mdstore, final String targetPath) {
-		log.info("saving dataset in: {}", targetPath);
-		mdstore
-			.write()
-			.mode(SaveMode.Overwrite)
-			.format("parquet")
-			.save(targetPath);
-	}
-
-	public static Configuration getHadoopConfiguration(String nameNode) {
-		// ====== Init HDFS File System Object
-		Configuration conf = new Configuration();
-		// Set FileSystem URI
-		conf.set("fs.defaultFS", nameNode);
-		// Because of Maven
-		conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
-		conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
-
-		System.setProperty("hadoop.home.dir", "/");
-		return conf;
-	}
-
-	public static void populateOOZIEEnv(final Map<String, String> report) throws IOException {
-		File file = new File(System.getProperty("oozie.action.output.properties"));
-		Properties props = new Properties();
-		report.forEach((k, v) -> props.setProperty(k, v));
-
-		try (OutputStream os = new FileOutputStream(file)) {
-			props.store(os, "");
-		}
-	}
-
-	public static void populateOOZIEEnv(final String paramName, String value) throws IOException {
-		Map<String, String> report = Maps.newHashMap();
-		report.put(paramName, value);
-
-		populateOOZIEEnv(report);
-	}
 }

dhp-common/src/main/java/eu/dnetlib/dhp/utils/ISLookupClientFactory.java

@@ -1,11 +1,11 @@
 
 package eu.dnetlib.dhp.utils;
 
-import org.apache.cxf.endpoint.Client;
-import org.apache.cxf.frontend.ClientProxy;
+import java.util.Map;
+
+import javax.xml.ws.BindingProvider;
+
 import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
-import org.apache.cxf.transport.http.HTTPConduit;
-import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -15,8 +15,8 @@ public class ISLookupClientFactory {
 
 	private static final Logger log = LoggerFactory.getLogger(ISLookupClientFactory.class);
 
-	private static final int requestTimeout = 60000 * 10;
-	private static final int connectTimeout = 60000 * 10;
+	private static int requestTimeout = 60000 * 10;
+	private static int connectTimeout = 60000 * 10;
 
 	public static ISLookUpService getLookUpService(final String isLookupUrl) {
 		return getServiceStub(ISLookUpService.class, isLookupUrl);
@@ -31,23 +31,20 @@ public class ISLookupClientFactory {
 
 		final T service = (T) jaxWsProxyFactory.create();
 
-		Client client = ClientProxy.getClient(service);
-		if (client != null) {
-			HTTPConduit conduit = (HTTPConduit) client.getConduit();
-			HTTPClientPolicy policy = new HTTPClientPolicy();
-
+		if (service instanceof BindingProvider) {
 			log
 				.info(
-					String
-						.format(
-							"setting connectTimeout to %s, requestTimeout to %s for service %s",
-							connectTimeout,
-							requestTimeout,
-							clazz.getCanonicalName()));
+					"setting timeouts for {} to requestTimeout: {}, connectTimeout: {}",
+					BindingProvider.class.getName(), requestTimeout, connectTimeout);
 
-			policy.setConnectionTimeout(connectTimeout);
-			policy.setReceiveTimeout(requestTimeout);
-			conduit.setClient(policy);
+			Map<String, Object> requestContext = ((BindingProvider) service).getRequestContext();
+
+			requestContext.put("com.sun.xml.internal.ws.request.timeout", requestTimeout);
+			requestContext.put("com.sun.xml.internal.ws.connect.timeout", connectTimeout);
+			requestContext.put("com.sun.xml.ws.request.timeout", requestTimeout);
+			requestContext.put("com.sun.xml.ws.connect.timeout", connectTimeout);
+			requestContext.put("javax.xml.ws.client.receiveTimeout", requestTimeout);
+			requestContext.put("javax.xml.ws.client.connectionTimeout", connectTimeout);
 		}
 
 		return service;

dhp-common/src/main/java/eu/dnetlib/dhp/utils/saxon/NormalizeDate.java

@@ -5,8 +5,6 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
-import org.apache.commons.lang3.StringUtils;
-
 import net.sf.saxon.expr.XPathContext;
 import net.sf.saxon.om.Sequence;
 import net.sf.saxon.trans.XPathException;
@@ -21,8 +19,6 @@ public class NormalizeDate extends AbstractExtensionFunction {
 
 	private static final String normalizeOutFormat = "yyyy-MM-dd'T'hh:mm:ss'Z'";
 
-	public static final String BLANK = "";
-
 	@Override
 	public String getName() {
 		return "normalizeDate";
@@ -31,10 +27,10 @@ public class NormalizeDate extends AbstractExtensionFunction {
 	@Override
 	public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
 		if (arguments == null | arguments.length == 0) {
-			return new StringValue(BLANK);
+			return new StringValue("");
 		}
 		String s = arguments[0].head().getStringValue();
-		return new StringValue(_normalizeDate(s));
+		return new StringValue(_year(s));
 	}
 
 	@Override
@@ -59,8 +55,8 @@ public class NormalizeDate extends AbstractExtensionFunction {
 		return SequenceType.SINGLE_STRING;
 	}
 
-	private String _normalizeDate(String s) {
-		final String date = StringUtils.isNotBlank(s) ? s.trim() : BLANK;
+	private String _year(String s) {
+		final String date = s != null ? s.trim() : "";
 
 		for (String format : normalizeDateFormats) {
 			try {
@@ -70,6 +66,6 @@ public class NormalizeDate extends AbstractExtensionFunction {
 			} catch (ParseException e) {
 			}
 		}
-		return BLANK;
+		return "";
 	}
 }

dhp-common/src/main/java/eu/dnetlib/message/Message.java

@@ -0,0 +1,76 @@
+
+package eu.dnetlib.message;
+
+import java.io.IOException;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class Message {
+
+	private String workflowId;
+
+	private String jobName;
+
+	private MessageType type;
+
+	private Map<String, String> body;
+
+	public static Message fromJson(final String json) throws IOException {
+		final ObjectMapper jsonMapper = new ObjectMapper();
+		return jsonMapper.readValue(json, Message.class);
+	}
+
+	public Message() {
+	}
+
+	public Message(String workflowId, String jobName, MessageType type, Map<String, String> body) {
+		this.workflowId = workflowId;
+		this.jobName = jobName;
+		this.type = type;
+		this.body = body;
+	}
+
+	public String getWorkflowId() {
+		return workflowId;
+	}
+
+	public void setWorkflowId(String workflowId) {
+		this.workflowId = workflowId;
+	}
+
+	public String getJobName() {
+		return jobName;
+	}
+
+	public void setJobName(String jobName) {
+		this.jobName = jobName;
+	}
+
+	public MessageType getType() {
+		return type;
+	}
+
+	public void setType(MessageType type) {
+		this.type = type;
+	}
+
+	public Map<String, String> getBody() {
+		return body;
+	}
+
+	public void setBody(Map<String, String> body) {
+		this.body = body;
+	}
+
+	@Override
+	public String toString() {
+		final ObjectMapper jsonMapper = new ObjectMapper();
+		try {
+			return jsonMapper.writeValueAsString(this);
+		} catch (JsonProcessingException e) {
+			return null;
+		}
+	}
+}

dhp-common/src/main/java/eu/dnetlib/message/MessageConsumer.java

@@ -0,0 +1,47 @@
+
+package eu.dnetlib.message;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+
+public class MessageConsumer extends DefaultConsumer {
+
+	final LinkedBlockingQueue<Message> queueMessages;
+
+	/**
+	 * Constructs a new instance and records its association to the passed-in channel.
+	 *
+	 * @param channel the channel to which this consumer is attached
+	 * @param queueMessages
+	 */
+	public MessageConsumer(Channel channel, LinkedBlockingQueue<Message> queueMessages) {
+		super(channel);
+		this.queueMessages = queueMessages;
+	}
+
+	@Override
+	public void handleDelivery(
+		String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body)
+		throws IOException {
+		final String json = new String(body, StandardCharsets.UTF_8);
+		Message message = Message.fromJson(json);
+		try {
+			this.queueMessages.put(message);
+			System.out.println("Receiving Message " + message);
+		} catch (InterruptedException e) {
+			if (message.getType() == MessageType.REPORT)
+				throw new RuntimeException("Error on sending message");
+			else {
+				// TODO LOGGING EXCEPTION
+			}
+		} finally {
+			getChannel().basicAck(envelope.getDeliveryTag(), false);
+		}
+	}
+}

dhp-common/src/main/java/eu/dnetlib/message/MessageManager.java

@@ -0,0 +1,136 @@
+
+package eu.dnetlib.message;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeoutException;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+
+public class MessageManager {
+
+	private final String messageHost;
+
+	private final String username;
+
+	private final String password;
+
+	private Connection connection;
+
+	private final Map<String, Channel> channels = new HashMap<>();
+
+	private boolean durable;
+
+	private boolean autodelete;
+
+	private final LinkedBlockingQueue<Message> queueMessages;
+
+	public MessageManager(
+		String messageHost,
+		String username,
+		String password,
+		final LinkedBlockingQueue<Message> queueMessages) {
+		this.queueMessages = queueMessages;
+		this.messageHost = messageHost;
+		this.username = username;
+		this.password = password;
+	}
+
+	public MessageManager(
+		String messageHost,
+		String username,
+		String password,
+		boolean durable,
+		boolean autodelete,
+		final LinkedBlockingQueue<Message> queueMessages) {
+		this.queueMessages = queueMessages;
+		this.messageHost = messageHost;
+		this.username = username;
+		this.password = password;
+
+		this.durable = durable;
+		this.autodelete = autodelete;
+	}
+
+	private Connection createConnection() throws IOException, TimeoutException {
+		ConnectionFactory factory = new ConnectionFactory();
+		factory.setHost(this.messageHost);
+		factory.setUsername(this.username);
+		factory.setPassword(this.password);
+		return factory.newConnection();
+	}
+
+	private Channel createChannel(
+		final Connection connection,
+		final String queueName,
+		final boolean durable,
+		final boolean autodelete)
+		throws Exception {
+		Map<String, Object> args = new HashMap<>();
+		args.put("x-message-ttl", 10000);
+		Channel channel = connection.createChannel();
+		channel.queueDeclare(queueName, durable, false, this.autodelete, args);
+		return channel;
+	}
+
+	private Channel getOrCreateChannel(final String queueName, boolean durable, boolean autodelete)
+		throws Exception {
+		if (channels.containsKey(queueName)) {
+			return channels.get(queueName);
+		}
+
+		if (this.connection == null) {
+			this.connection = createConnection();
+		}
+		channels.put(queueName, createChannel(this.connection, queueName, durable, autodelete));
+		return channels.get(queueName);
+	}
+
+	public void close() throws IOException {
+		channels
+			.values()
+			.forEach(
+				ch -> {
+					try {
+						ch.close();
+					} catch (Exception e) {
+						// TODO LOG
+					}
+				});
+
+		this.connection.close();
+	}
+
+	public boolean sendMessage(final Message message, String queueName) throws Exception {
+		try {
+			Channel channel = getOrCreateChannel(queueName, this.durable, this.autodelete);
+			channel.basicPublish("", queueName, null, message.toString().getBytes());
+			return true;
+		} catch (Throwable e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public boolean sendMessage(
+		final Message message, String queueName, boolean durable_var, boolean autodelete_var)
+		throws Exception {
+		try {
+			Channel channel = getOrCreateChannel(queueName, durable_var, autodelete_var);
+			channel.basicPublish("", queueName, null, message.toString().getBytes());
+			return true;
+		} catch (Throwable e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public void startConsumingMessage(
+		final String queueName, final boolean durable, final boolean autodelete) throws Exception {
+
+		Channel channel = createChannel(createConnection(), queueName, durable, autodelete);
+		channel.basicConsume(queueName, false, new MessageConsumer(channel, queueMessages));
+	}
+}

dhp-common/src/main/java/eu/dnetlib/message/MessageType.java

@@ -0,0 +1,6 @@
+
+package eu.dnetlib.message;
+
+public enum MessageType {
+	ONGOING, REPORT
+}

dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java

@@ -1,109 +0,0 @@
-
-package eu.dnetlib.dhp.common.api;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-@Disabled
-public class ZenodoAPIClientTest {
-
-	private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
-	private final String ACCESS_TOKEN = "";
-
-	private final String CONCEPT_REC_ID = "657113";
-
-	private final String depositionId = "674915";
-
-	@Test
-	public void testUploadOldDeposition() throws IOException, MissingConceptDoiException {
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-		Assertions.assertEquals(200, client.uploadOpenDeposition(depositionId));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
-
-		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
-		Assertions.assertEquals(200, client.sendMretadata(metadata));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	public void testNewDeposition() throws IOException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-		Assertions.assertEquals(201, client.newDeposition());
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
-
-		String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
-
-		Assertions.assertEquals(200, client.sendMretadata(metadata));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	public void testNewVersionNewName() throws IOException, MissingConceptDoiException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-
-		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/newVersion")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length()));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-	@Test
-	public void testNewVersionOldName() throws IOException, MissingConceptDoiException {
-
-		ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
-			ACCESS_TOKEN);
-
-		Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID));
-
-		File file = new File(getClass()
-			.getResource("/eu/dnetlib/dhp/common/api/newVersion2")
-			.getPath());
-
-		InputStream is = new FileInputStream(file);
-
-		Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length()));
-
-		Assertions.assertEquals(202, client.publish());
-
-	}
-
-}

dhp-common/src/test/java/eu/dnetlib/dhp/model/mdstore/MetadataRecordTest.java

@@ -0,0 +1,16 @@
+
+package eu.dnetlib.dhp.model.mdstore;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+public class MetadataRecordTest {
+
+	@Test
+	public void getTimestamp() {
+
+		MetadataRecord r = new MetadataRecord();
+		assertTrue(r.getDateOfCollection() > 0);
+	}
+}

dhp-common/src/test/java/eu/dnetlib/dhp/oa/merge/AuthorMergerTest.java

@@ -1,100 +0,0 @@
-
-package eu.dnetlib.dhp.oa.merge;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.dhp.schema.oaf.Author;
-import eu.dnetlib.dhp.schema.oaf.Publication;
-import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
-import eu.dnetlib.pace.util.MapDocumentUtil;
-import scala.Tuple2;
-
-public class AuthorMergerTest {
-
-	private String publicationsBasePath;
-
-	private List<List<Author>> authors;
-
-	@BeforeEach
-	public void setUp() throws Exception {
-
-		publicationsBasePath = Paths
-			.get(AuthorMergerTest.class.getResource("/eu/dnetlib/dhp/oa/merge").toURI())
-			.toFile()
-			.getAbsolutePath();
-
-		authors = readSample(publicationsBasePath + "/publications_with_authors.json", Publication.class)
-			.stream()
-			.map(p -> p._2().getAuthor())
-			.collect(Collectors.toList());
-
-	}
-
-	@Test
-	public void mergeTest() { // used in the dedup: threshold set to 0.95
-
-		for (List<Author> authors1 : authors) {
-			System.out.println("List " + (authors.indexOf(authors1) + 1));
-			for (Author author : authors1) {
-				System.out.println(authorToString(author));
-			}
-		}
-
-		List<Author> merge = AuthorMerger.merge(authors);
-
-		System.out.println("Merge ");
-		for (Author author : merge) {
-			System.out.println(authorToString(author));
-		}
-
-		Assertions.assertEquals(7, merge.size());
-
-	}
-
-	public <T> List<Tuple2<String, T>> readSample(String path, Class<T> clazz) {
-		List<Tuple2<String, T>> res = new ArrayList<>();
-		BufferedReader reader;
-		try {
-			reader = new BufferedReader(new FileReader(path));
-			String line = reader.readLine();
-			while (line != null) {
-				res
-					.add(
-						new Tuple2<>(
-							MapDocumentUtil.getJPathString("$.id", line),
-							new ObjectMapper().readValue(line, clazz)));
-				// read next line
-				line = reader.readLine();
-			}
-			reader.close();
-		} catch (IOException e) {
-			e.printStackTrace();
-		}
-
-		return res;
-	}
-
-	public String authorToString(Author a) {
-
-		String print = "Fullname = ";
-		print += a.getFullname() + " pid = [";
-		if (a.getPid() != null)
-			for (StructuredProperty sp : a.getPid()) {
-				print += sp.toComparableString() + " ";
-			}
-		print += "]";
-		return print;
-	}
-}

dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtilsTest.java

@@ -1,180 +0,0 @@
-
-package eu.dnetlib.dhp.schema.oaf.utils;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import java.io.IOException;
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import eu.dnetlib.dhp.schema.common.ModelConstants;
-import eu.dnetlib.dhp.schema.oaf.*;
-
-public class OafMapperUtilsTest {
-
-	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
-		.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
-	@Test
-	public void testDateValidation() {
-
-		assertTrue(GraphCleaningFunctions.doCleanDate("2016-05-07T12:41:19.202Z  ").isPresent());
-		assertTrue(GraphCleaningFunctions.doCleanDate("2020-09-10 11:08:52 ").isPresent());
-		assertTrue(GraphCleaningFunctions.doCleanDate("  2016-04-05").isPresent());
-
-		assertEquals("2016-04-05", GraphCleaningFunctions.doCleanDate("2016 Apr 05").get());
-
-		assertEquals("2009-05-08", GraphCleaningFunctions.doCleanDate("May 8, 2009 5:57:51 PM").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct 7, 1970").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct 7, '70").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct. 7, 1970").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct. 7, 70").get());
-		assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon Jan  2 15:04:05 2006").get());
-		assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon Jan  2 15:04:05 MST 2006").get());
-		assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon Jan 02 15:04:05 -0700 2006").get());
-		assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Monday, 02-Jan-06 15:04:05 MST").get());
-		assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon, 02 Jan 2006 15:04:05 MST").get());
-		assertEquals("2017-07-11", GraphCleaningFunctions.doCleanDate("Tue, 11 Jul 2017 16:28:13 +0200 (CEST)").get());
-		assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon, 02 Jan 2006 15:04:05 -0700").get());
-		assertEquals("2018-01-04", GraphCleaningFunctions.doCleanDate("Thu, 4 Jan 2018 17:53:36 +0000").get());
-		assertEquals("2015-08-10", GraphCleaningFunctions.doCleanDate("Mon Aug 10 15:44:11 UTC+0100 2015").get());
-		assertEquals(
-			"2015-07-03",
-			GraphCleaningFunctions.doCleanDate("Fri Jul 03 2015 18:04:07 GMT+0100 (GMT Daylight Time)").get());
-		assertEquals("2012-09-17", GraphCleaningFunctions.doCleanDate("September 17, 2012 10:09am").get());
-		assertEquals("2012-09-17", GraphCleaningFunctions.doCleanDate("September 17, 2012 at 10:09am PST-08").get());
-		assertEquals("2012-09-17", GraphCleaningFunctions.doCleanDate("September 17, 2012, 10:10:09").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("October 7, 1970").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("October 7th, 1970").get());
-		assertEquals("2006-02-12", GraphCleaningFunctions.doCleanDate("12 Feb 2006, 19:17").get());
-		assertEquals("2006-02-12", GraphCleaningFunctions.doCleanDate("12 Feb 2006 19:17").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("7 oct 70").get());
-		assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("7 oct 1970").get());
-		assertEquals("2013-02-03", GraphCleaningFunctions.doCleanDate("03 February 2013").get());
-		assertEquals("2013-07-01", GraphCleaningFunctions.doCleanDate("1 July 2013").get());
-		assertEquals("2013-02-03", GraphCleaningFunctions.doCleanDate("2013-Feb-03").get());
-		assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("3/31/2014").get());
-		assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("03/31/2014").get());
-		assertEquals("1971-08-21", GraphCleaningFunctions.doCleanDate("08/21/71").get());
-		assertEquals("1971-01-08", GraphCleaningFunctions.doCleanDate("8/1/71").get());
-		assertEquals("2014-08-04", GraphCleaningFunctions.doCleanDate("4/8/2014 22:05").get());
-		assertEquals("2014-08-04", GraphCleaningFunctions.doCleanDate("04/08/2014 22:05").get());
-		assertEquals("2014-08-04", GraphCleaningFunctions.doCleanDate("4/8/14 22:05").get());
-		assertEquals("2014-02-04", GraphCleaningFunctions.doCleanDate("04/2/2014 03:00:51").get());
-		assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 12:00:00 AM").get());
-		assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 01:00:01 PM").get());
-		assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 01:00 PM").get());
-		assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 1:00 PM").get());
-		assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 12:00 AM").get());
-		assertEquals("2014-02-04", GraphCleaningFunctions.doCleanDate("4/02/2014 03:00:51").get());
-		assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("03/19/2012 10:11:59").get());
-		assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("03/19/2012 10:11:59.3186369").get());
-		assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("2014/3/31").get());
-		assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("2014/03/31").get());
-		assertEquals("2014-04-08", GraphCleaningFunctions.doCleanDate("2014/4/8 22:05").get());
-		assertEquals("2014-04-08", GraphCleaningFunctions.doCleanDate("2014/04/08 22:05").get());
-		assertEquals("2014-04-02", GraphCleaningFunctions.doCleanDate("2014/04/2 03:00:51").get());
-		assertEquals("2014-04-02", GraphCleaningFunctions.doCleanDate("2014/4/02 03:00:51").get());
-		assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("2012/03/19 10:11:59").get());
-		assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("2012/03/19 10:11:59.3186369").get());
-		assertEquals("2014-04-08", GraphCleaningFunctions.doCleanDate("2014年04月08日").get());
-		assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("2006-01-02T15:04:05+0000").get());
-		assertEquals("2009-08-13", GraphCleaningFunctions.doCleanDate("2009-08-12T22:15:09-07:00").get());
-		assertEquals("2009-08-12", GraphCleaningFunctions.doCleanDate("2009-08-12T22:15:09").get());
-		assertEquals("2009-08-12", GraphCleaningFunctions.doCleanDate("2009-08-12T22:15:09Z").get());
-		assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 17:24:37.3186369").get());
-		assertEquals("2012-08-03", GraphCleaningFunctions.doCleanDate("2012-08-03 18:31:59.257000000").get());
-		assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 17:24:37.123").get());
-		assertEquals("2013-04-01", GraphCleaningFunctions.doCleanDate("2013-04-01 22:43").get());
-		assertEquals("2013-04-01", GraphCleaningFunctions.doCleanDate("2013-04-01 22:43:22").get());
-		assertEquals("2014-12-16", GraphCleaningFunctions.doCleanDate("2014-12-16 06:20:00 UTC").get());
-		assertEquals("2014-12-16", GraphCleaningFunctions.doCleanDate("2014-12-16 06:20:00 GMT").get());
-		assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 05:24:37 PM").get());
-		assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 13:13:43 +0800").get());
-		assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 13:13:43 +0800 +08").get());
-		assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 13:13:44 +09:00").get());
-		assertEquals("2012-08-03", GraphCleaningFunctions.doCleanDate("2012-08-03 18:31:59.257000000 +0000 UTC").get());
-		assertEquals("2015-09-30", GraphCleaningFunctions.doCleanDate("2015-09-30 18:48:56.35272715 +0000 UTC").get());
-		assertEquals("2015-02-18", GraphCleaningFunctions.doCleanDate("2015-02-18 00:12:00 +0000 GMT").get());
-		assertEquals("2015-02-18", GraphCleaningFunctions.doCleanDate("2015-02-18 00:12:00 +0000 UTC").get());
-		assertEquals(
-			"2015-02-08", GraphCleaningFunctions.doCleanDate("2015-02-08 03:02:00 +0300 MSK m=+0.000000001").get());
-		assertEquals(
-			"2015-02-08", GraphCleaningFunctions.doCleanDate("2015-02-08 03:02:00.001 +0300 MSK m=+0.000000001").get());
-		assertEquals("2017-07-19", GraphCleaningFunctions.doCleanDate("2017-07-19 03:21:51+00:00").get());
-		assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26").get());
-		assertEquals("2014-04-01", GraphCleaningFunctions.doCleanDate("2014-04").get());
-		assertEquals("2014-01-01", GraphCleaningFunctions.doCleanDate("2014").get());
-		assertEquals("2014-05-11", GraphCleaningFunctions.doCleanDate("2014-05-11 08:20:13,787").get());
-		assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("3.31.2014").get());
-		assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("03.31.2014").get());
-		assertEquals("1971-08-21", GraphCleaningFunctions.doCleanDate("08.21.71").get());
-		assertEquals("2014-03-01", GraphCleaningFunctions.doCleanDate("2014.03").get());
-		assertEquals("2014-03-30", GraphCleaningFunctions.doCleanDate("2014.03.30").get());
-		assertEquals("2014-06-01", GraphCleaningFunctions.doCleanDate("20140601").get());
-		assertEquals("2014-07-22", GraphCleaningFunctions.doCleanDate("20140722105203").get());
-		assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("1332151919").get());
-		assertEquals("2013-11-12", GraphCleaningFunctions.doCleanDate("1384216367189").get());
-		assertEquals("2013-11-12", GraphCleaningFunctions.doCleanDate("1384216367111222").get());
-		assertEquals("2013-11-12", GraphCleaningFunctions.doCleanDate("1384216367111222333").get());
-
-	}
-
-	@Test
-	public void testDate() {
-		System.out.println(GraphCleaningFunctions.cleanDate("23-FEB-1998"));
-	}
-
-	@Test
-	public void testMergePubs() throws IOException {
-		Publication p1 = read("publication_1.json", Publication.class);
-		Publication p2 = read("publication_2.json", Publication.class);
-		Dataset d1 = read("dataset_1.json", Dataset.class);
-		Dataset d2 = read("dataset_2.json", Dataset.class);
-
-		assertEquals(p1.getCollectedfrom().size(), 1);
-		assertEquals(p1.getCollectedfrom().get(0).getKey(), ModelConstants.CROSSREF_ID);
-		assertEquals(d2.getCollectedfrom().size(), 1);
-		assertFalse(cfId(d2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
-
-		assertTrue(
-			OafMapperUtils
-				.mergeResults(p1, d2)
-				.getResulttype()
-				.getClassid()
-				.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID));
-
-		assertEquals(p2.getCollectedfrom().size(), 1);
-		assertFalse(cfId(p2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
-		assertEquals(d1.getCollectedfrom().size(), 1);
-		assertTrue(cfId(d1.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
-
-		assertTrue(
-			OafMapperUtils
-				.mergeResults(p2, d1)
-				.getResulttype()
-				.getClassid()
-				.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID));
-	}
-
-	protected HashSet<String> cfId(List<KeyValue> collectedfrom) {
-		return collectedfrom.stream().map(c -> c.getKey()).collect(Collectors.toCollection(HashSet::new));
-	}
-
-	protected <T extends Result> T read(String filename, Class<T> clazz) throws IOException {
-		final String json = IOUtils.toString(getClass().getResourceAsStream(filename));
-		return OBJECT_MAPPER.readValue(json, clazz);
-	}
-
-}

dhp-common/src/test/java/eu/dnetlib/message/MessageTest.java

@@ -0,0 +1,51 @@
+
+package eu.dnetlib.message;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+
+public class MessageTest {
+
+	@Test
+	public void fromJsonTest() throws IOException {
+		Message m = new Message();
+		m.setWorkflowId("wId");
+		m.setType(MessageType.ONGOING);
+		m.setJobName("Collection");
+		Map<String, String> body = new HashMap<>();
+		body.put("parsedItem", "300");
+		body.put("ExecutionTime", "30s");
+
+		m.setBody(body);
+		System.out.println("m = " + m);
+		Message m1 = Message.fromJson(m.toString());
+		assertEquals(m1.getWorkflowId(), m.getWorkflowId());
+		assertEquals(m1.getType(), m.getType());
+		assertEquals(m1.getJobName(), m.getJobName());
+
+		assertNotNull(m1.getBody());
+		m1.getBody().keySet().forEach(it -> assertEquals(m1.getBody().get(it), m.getBody().get(it)));
+		assertEquals(m1.getJobName(), m.getJobName());
+	}
+
+	@Test
+	public void toStringTest() {
+		final String expectedJson = "{\"workflowId\":\"wId\",\"jobName\":\"Collection\",\"type\":\"ONGOING\",\"body\":{\"ExecutionTime\":\"30s\",\"parsedItem\":\"300\"}}";
+		Message m = new Message();
+		m.setWorkflowId("wId");
+		m.setType(MessageType.ONGOING);
+		m.setJobName("Collection");
+		Map<String, String> body = new HashMap<>();
+		body.put("parsedItem", "300");
+		body.put("ExecutionTime", "30s");
+
+		m.setBody(body);
+
+		assertEquals(expectedJson, m.toString());
+	}
+}

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/metadata.json

@@ -1 +0,0 @@
-{"metadata":{"access_right":"open","communities":[{"identifier":"openaire-research-graph"}],"creators":[{"affiliation":"ISTI - CNR","name":"Bardi, Alessia","orcid":"0000-0002-1112-1292"},{"affiliation":"eifl", "name":"Kuchma, Iryna"},{"affiliation":"BIH", "name":"Brobov, Evgeny"},{"affiliation":"GIDIF RBM", "name":"Truccolo, Ivana"},{"affiliation":"unesp", "name":"Monteiro, Elizabete"},{"affiliation":"und", "name":"Casalegno, Carlotta"},{"affiliation":"CARL ABRC", "name":"Clary, Erin"},{"affiliation":"The University of Edimburgh", "name":"Romanowski, Andrew"},{"affiliation":"ISTI - CNR", "name":"Pavone, Gina"},{"affiliation":"ISTI - CNR", "name":"Artini, Michele"},{"affiliation":"ISTI - CNR","name":"Atzori, Claudio","orcid":"0000-0001-9613-6639"},{"affiliation":"University of Bielefeld","name":"Bäcker, Amelie","orcid":"0000-0001-6015-2063"},{"affiliation":"ISTI - CNR","name":"Baglioni, Miriam","orcid":"0000-0002-2273-9004"},{"affiliation":"University of Bielefeld","name":"Czerniak, Andreas","orcid":"0000-0003-3883-4169"},{"affiliation":"ISTI - CNR","name":"De Bonis, Michele"},{"affiliation":"Athena Research and Innovation Centre","name":"Dimitropoulos, Harry"},{"affiliation":"Athena Research and Innovation Centre","name":"Foufoulas, Ioannis"},{"affiliation":"University of Warsaw","name":"Horst, Marek"},{"affiliation":"Athena Research and Innovation Centre","name":"Iatropoulou, Katerina"},{"affiliation":"University of Warsaw","name":"Jacewicz, Przemyslaw"},{"affiliation":"Athena Research and Innovation Centre","name":"Kokogiannaki, Argiro", "orcid":"0000-0002-3880-0244"},{"affiliation":"ISTI - CNR","name":"La Bruzzo, Sandro","orcid":"0000-0003-2855-1245"},{"affiliation":"ISTI - CNR","name":"Lazzeri, Emma"},{"affiliation":"University of Bielefeld","name":"Löhden, Aenne"},{"affiliation":"ISTI - CNR","name":"Manghi, Paolo","orcid":"0000-0001-7291-3210"},{"affiliation":"ISTI - CNR","name":"Mannocci, Andrea","orcid":"0000-0002-5193-7851"},{"affiliation":"Athena Research and Innovation Center","name":"Manola, Natalia"},{"affiliation":"ISTI - CNR","name":"Ottonello, Enrico"},{"affiliation":"University of Bielefeld","name":"Shirrwagen, Jochen"}],"description":"\\u003cp\\u003eThis dump provides access to the metadata records of publications, research data, software and projects that may be relevant to the Corona Virus Disease (COVID-19) fight. The dump contains records of the OpenAIRE COVID-19 Gateway (https://covid-19.openaire.eu/), identified via full-text mining and inference techniques applied to the OpenAIRE Research Graph (https://explore.openaire.eu/). The Graph is one of the largest Open Access collections of metadata records and links between publications, datasets, software, projects, funders, and organizations, aggregating 12,000+ scientific data sources world-wide, among which the Covid-19 data sources Zenodo COVID-19 Community, WHO (World Health Organization), BIP! FInder for COVID-19, Protein Data Bank, Dimensions, scienceOpen, and RSNA. \\u003cp\\u003eThe dump consists of a gzip file containing one json per line. Each json is compliant to the schema available at https://doi.org/10.5281/zenodo.3974226\\u003c/p\\u003e ","title":"OpenAIRE Covid-19 publications, datasets, software and projects metadata.","upload_type":"dataset","version":"1.0"}}

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newDeposition

@@ -1 +0,0 @@
-This is a test for a new deposition

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newVersion

@@ -1 +0,0 @@
-This is a test for a new version of an old deposition

dhp-common/src/test/resources/eu/dnetlib/dhp/common/api/newVersion2

@@ -1,2 +0,0 @@
-This is a test for a new version of an old deposition. This should replace the other new version. I expect to have only two
-files in the deposition

dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/dataset_1.json

@@ -1 +0,0 @@
-{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1g", "resuttype" : { "classid" : "dataset" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value" : "Crossref"} ]}

dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/dataset_2.json

@@ -1 +0,0 @@
-{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1g", "resuttype" : { "classid" : "dataset" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e3", "value" : "Repository B"} ]}

dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/publication_1.json

@@ -1 +0,0 @@
-{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f", "resuttype" : { "classid" : "publication" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value" : "Crossref"} ]}

dhp-common/src/test/resources/eu/dnetlib/dhp/schema/oaf/utils/publication_2.json

@@ -1 +0,0 @@
-{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f", "resuttype" : { "classid" : "publication" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e3", "value" : "Repository A"} ]}

dhp-schemas/README.md

@@ -0,0 +1,11 @@
+Description of the project
+--------------------------
+This project defines **object schemas** of the OpenAIRE main entities and the relationships that intercur among them. 
+Namely it defines the model for 
+  
+-  **research product (result)** which subclasses in publication, dataset, other research product, software
+-  **data source** object describing the data provider (institutional repository, aggregators, cris systems)
+-  **organization** research bodies managing a data source or participating to a research project
+-  **project** research project
+ 
+Te serialization of such objects (data store files) are used to pass data between workflow nodes in the processing pipeline.

dhp-schemas/pom.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>eu.dnetlib.dhp</groupId>
+        <artifactId>dhp</artifactId>
+        <version>1.2.4-SNAPSHOT</version>
+        <relativePath>../</relativePath>
+    </parent>
+
+    <artifactId>dhp-schemas</artifactId>
+    <packaging>jar</packaging>
+
+    <description>This module contains common schema classes meant to be used across the dnet-hadoop submodules</description>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>4.0.1</version>
+                <executions>
+                    <execution>
+                        <id>scala-compile-first</id>
+                        <phase>initialize</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                </configuration>
+            </plugin>
+
+        </plugins>
+    </build>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+    </dependencies>
+
+
+</project>

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/action/AtomicAction.java

@@ -0,0 +1,40 @@
+
+package eu.dnetlib.dhp.schema.action;
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+import eu.dnetlib.dhp.schema.oaf.Oaf;
+
+@JsonDeserialize(using = AtomicActionDeserializer.class)
+public class AtomicAction<T extends Oaf> implements Serializable {
+
+	private Class<T> clazz;
+
+	private T payload;
+
+	public AtomicAction() {
+	}
+
+	public AtomicAction(Class<T> clazz, T payload) {
+		this.clazz = clazz;
+		this.payload = payload;
+	}
+
+	public Class<T> getClazz() {
+		return clazz;
+	}
+
+	public void setClazz(Class<T> clazz) {
+		this.clazz = clazz;
+	}
+
+	public T getPayload() {
+		return payload;
+	}
+
+	public void setPayload(T payload) {
+		this.payload = payload;
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/action/AtomicActionDeserializer.java

@@ -0,0 +1,32 @@
+
+package eu.dnetlib.dhp.schema.action;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.dhp.schema.oaf.Oaf;
+
+public class AtomicActionDeserializer extends JsonDeserializer {
+
+	@Override
+	public Object deserialize(JsonParser jp, DeserializationContext ctxt)
+		throws IOException {
+		JsonNode node = jp.getCodec().readTree(jp);
+		String classTag = node.get("clazz").asText();
+		JsonNode payload = node.get("payload");
+		ObjectMapper mapper = new ObjectMapper();
+
+		try {
+			final Class<?> clazz = Class.forName(classTag);
+			return new AtomicAction(clazz, (Oaf) mapper.readValue(payload.toString(), clazz));
+		} catch (ClassNotFoundException e) {
+			throw new IOException(e);
+		}
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/EntityType.java

@@ -0,0 +1,21 @@
+
+package eu.dnetlib.dhp.schema.common;
+
+import eu.dnetlib.dhp.schema.oaf.OafEntity;
+
+/** Actual entity types in the Graph */
+public enum EntityType {
+	publication, dataset, otherresearchproduct, software, datasource, organization, project;
+
+	/**
+	 * Resolves the EntityType, given the relative class name
+	 *
+	 * @param clazz the given class name
+	 * @param <T> actual OafEntity subclass
+	 * @return the EntityType associated to the given class
+	 */
+	public static <T extends OafEntity> EntityType fromClass(Class<T> clazz) {
+
+		return EntityType.valueOf(clazz.getSimpleName().toLowerCase());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/LicenseComparator.java

@@ -0,0 +1,69 @@
+
+package eu.dnetlib.dhp.schema.common;
+
+import java.util.Comparator;
+
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
+
+public class LicenseComparator implements Comparator<Qualifier> {
+
+	@Override
+	public int compare(Qualifier left, Qualifier right) {
+
+		if (left == null && right == null)
+			return 0;
+		if (left == null)
+			return 1;
+		if (right == null)
+			return -1;
+
+		String lClass = left.getClassid();
+		String rClass = right.getClassid();
+
+		if (lClass.equals(rClass))
+			return 0;
+
+		if (lClass.equals("OPEN SOURCE"))
+			return -1;
+		if (rClass.equals("OPEN SOURCE"))
+			return 1;
+
+		if (lClass.equals("OPEN"))
+			return -1;
+		if (rClass.equals("OPEN"))
+			return 1;
+
+		if (lClass.equals("6MONTHS"))
+			return -1;
+		if (rClass.equals("6MONTHS"))
+			return 1;
+
+		if (lClass.equals("12MONTHS"))
+			return -1;
+		if (rClass.equals("12MONTHS"))
+			return 1;
+
+		if (lClass.equals("EMBARGO"))
+			return -1;
+		if (rClass.equals("EMBARGO"))
+			return 1;
+
+		if (lClass.equals("RESTRICTED"))
+			return -1;
+		if (rClass.equals("RESTRICTED"))
+			return 1;
+
+		if (lClass.equals("CLOSED"))
+			return -1;
+		if (rClass.equals("CLOSED"))
+			return 1;
+
+		if (lClass.equals("UNKNOWN"))
+			return -1;
+		if (rClass.equals("UNKNOWN"))
+			return 1;
+
+		// Else (but unlikely), lexicographical ordering will do.
+		return lClass.compareTo(rClass);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/MainEntityType.java

@@ -0,0 +1,7 @@
+
+package eu.dnetlib.dhp.schema.common;
+
+/** Main entity types in the Graph */
+public enum MainEntityType {
+	result, datasource, organization, project
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelConstants.java

@@ -0,0 +1,124 @@
+
+package eu.dnetlib.dhp.schema.common;
+
+import eu.dnetlib.dhp.schema.oaf.DataInfo;
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Qualifier;
+
+public class ModelConstants {
+
+	public static final String DNET_SUBJECT_TYPOLOGIES = "dnet:subject_classification_typologies";
+	public static final String DNET_RESULT_TYPOLOGIES = "dnet:result_typologies";
+	public static final String DNET_PUBLICATION_RESOURCE = "dnet:publication_resource";
+	public static final String DNET_ACCESS_MODES = "dnet:access_modes";
+	public static final String DNET_LANGUAGES = "dnet:languages";
+	public static final String DNET_PID_TYPES = "dnet:pid_types";
+	public static final String DNET_DATA_CITE_DATE = "dnet:dataCite_date";
+	public static final String DNET_DATA_CITE_RESOURCE = "dnet:dataCite_resource";
+	public static final String DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions";
+	public static final String DNET_COUNTRY_TYPE = "dnet:countries";
+	public static final String DNET_REVIEW_LEVELS = "dnet:review_levels";
+
+	public static final String SYSIMPORT_CROSSWALK_REPOSITORY = "sysimport:crosswalk:repository";
+	public static final String SYSIMPORT_CROSSWALK_ENTITYREGISTRY = "sysimport:crosswalk:entityregistry";
+	public static final String USER_CLAIM = "user:claim";
+
+	public static final String DATASET_RESULTTYPE_CLASSID = "dataset";
+	public static final String PUBLICATION_RESULTTYPE_CLASSID = "publication";
+	public static final String SOFTWARE_RESULTTYPE_CLASSID = "software";
+	public static final String ORP_RESULTTYPE_CLASSID = "other";
+
+	public static final String RESULT_RESULT = "resultResult";
+	/**
+	 * @deprecated Use {@link ModelConstants#RELATIONSHIP} instead.
+	 */
+	@Deprecated
+	public static final String PUBLICATION_DATASET = "publicationDataset";
+	public static final String IS_RELATED_TO = "isRelatedTo";
+	public static final String SUPPLEMENT = "supplement";
+	public static final String IS_SUPPLEMENT_TO = "isSupplementTo";
+	public static final String IS_SUPPLEMENTED_BY = "isSupplementedBy";
+	public static final String PART = "part";
+	public static final String IS_PART_OF = "IsPartOf";
+	public static final String HAS_PARTS = "HasParts";
+	public static final String RELATIONSHIP = "relationship";
+	public static final String CITATION = "citation";
+	public static final String CITES = "cites";
+	public static final String IS_CITED_BY = "IsCitedBy";
+	public static final String REVIEW = "review";
+	public static final String REVIEWS = "reviews";
+	public static final String IS_REVIEWED_BY = "IsReviewedBy";
+
+	public static final String RESULT_PROJECT = "resultProject";
+	public static final String OUTCOME = "outcome";
+	public static final String IS_PRODUCED_BY = "isProducedBy";
+	public static final String PRODUCES = "produces";
+
+	public static final String DATASOURCE_ORGANIZATION = "datasourceOrganization";
+	public static final String PROVISION = "provision";
+	public static final String IS_PROVIDED_BY = "isProvidedBy";
+	public static final String PROVIDES = "provides";
+
+	public static final String PROJECT_ORGANIZATION = "projectOrganization";
+	public static final String PARTICIPATION = "participation";
+	public static final String HAS_PARTICIPANT = "hasParticipant";
+	public static final String IS_PARTICIPANT = "isParticipant";
+
+	public static final String RESULT_ORGANIZATION = "resultOrganization";
+	public static final String AFFILIATION = "affiliation";
+	public static final String IS_AUTHOR_INSTITUTION_OF = "isAuthorInstitutionOf";
+	public static final String HAS_AUTHOR_INSTITUTION = "hasAuthorInstitution";
+
+	public static final String MERGES = "merges";
+
+	public static final String UNKNOWN = "UNKNOWN";
+	public static final String NOT_AVAILABLE = "not available";
+
+	public static final Qualifier PUBLICATION_DEFAULT_RESULTTYPE = qualifier(
+		PUBLICATION_RESULTTYPE_CLASSID, PUBLICATION_RESULTTYPE_CLASSID,
+		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
+
+	public static final Qualifier DATASET_DEFAULT_RESULTTYPE = qualifier(
+		DATASET_RESULTTYPE_CLASSID, DATASET_RESULTTYPE_CLASSID,
+		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
+
+	public static final Qualifier SOFTWARE_DEFAULT_RESULTTYPE = qualifier(
+		SOFTWARE_RESULTTYPE_CLASSID, SOFTWARE_RESULTTYPE_CLASSID,
+		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
+
+	public static final Qualifier ORP_DEFAULT_RESULTTYPE = qualifier(
+		ORP_RESULTTYPE_CLASSID, ORP_RESULTTYPE_CLASSID,
+		DNET_RESULT_TYPOLOGIES, DNET_RESULT_TYPOLOGIES);
+
+	public static final Qualifier REPOSITORY_PROVENANCE_ACTIONS = qualifier(
+		SYSIMPORT_CROSSWALK_REPOSITORY, SYSIMPORT_CROSSWALK_REPOSITORY,
+		DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS);
+
+	public static final Qualifier ENTITYREGISTRY_PROVENANCE_ACTION = qualifier(
+		SYSIMPORT_CROSSWALK_ENTITYREGISTRY, SYSIMPORT_CROSSWALK_ENTITYREGISTRY,
+		DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS);
+
+	public static final KeyValue UNKNOWN_REPOSITORY = keyValue(
+		"10|openaire____::55045bd2a65019fd8e6741a755395c8c", "Unknown Repository");
+
+	private static Qualifier qualifier(
+		final String classid,
+		final String classname,
+		final String schemeid,
+		final String schemename) {
+		final Qualifier q = new Qualifier();
+		q.setClassid(classid);
+		q.setClassname(classname);
+		q.setSchemeid(schemeid);
+		q.setSchemename(schemename);
+		return q;
+	}
+
+	private static KeyValue keyValue(String key, String value) {
+		KeyValue kv = new KeyValue();
+		kv.setKey(key);
+		kv.setValue(value);
+		kv.setDataInfo(new DataInfo());
+		return kv;
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/ModelSupport.java

@@ -0,0 +1,467 @@
+
+package eu.dnetlib.dhp.schema.common;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Function;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.google.common.collect.Maps;
+
+import eu.dnetlib.dhp.schema.oaf.*;
+
+/** Oaf model utility methods. */
+public class ModelSupport {
+
+	/** Defines the mapping between the actual entity type and the main entity type */
+	private static Map<EntityType, MainEntityType> entityMapping = Maps.newHashMap();
+
+	static {
+		entityMapping.put(EntityType.publication, MainEntityType.result);
+		entityMapping.put(EntityType.dataset, MainEntityType.result);
+		entityMapping.put(EntityType.otherresearchproduct, MainEntityType.result);
+		entityMapping.put(EntityType.software, MainEntityType.result);
+		entityMapping.put(EntityType.datasource, MainEntityType.datasource);
+		entityMapping.put(EntityType.organization, MainEntityType.organization);
+		entityMapping.put(EntityType.project, MainEntityType.project);
+	}
+
+	/**
+	 * Defines the mapping between the actual entity types and the relative classes implementing them
+	 */
+	public static final Map<EntityType, Class> entityTypes = Maps.newHashMap();
+
+	static {
+		entityTypes.put(EntityType.datasource, Datasource.class);
+		entityTypes.put(EntityType.organization, Organization.class);
+		entityTypes.put(EntityType.project, Project.class);
+		entityTypes.put(EntityType.dataset, Dataset.class);
+		entityTypes.put(EntityType.otherresearchproduct, OtherResearchProduct.class);
+		entityTypes.put(EntityType.software, Software.class);
+		entityTypes.put(EntityType.publication, Publication.class);
+	}
+
+	public static final Map<String, Class> oafTypes = Maps.newHashMap();
+
+	static {
+		oafTypes.put("datasource", Datasource.class);
+		oafTypes.put("organization", Organization.class);
+		oafTypes.put("project", Project.class);
+		oafTypes.put("dataset", Dataset.class);
+		oafTypes.put("otherresearchproduct", OtherResearchProduct.class);
+		oafTypes.put("software", Software.class);
+		oafTypes.put("publication", Publication.class);
+		oafTypes.put("relation", Relation.class);
+	}
+
+	public static final Map<Class, String> idPrefixMap = Maps.newHashMap();
+
+	static {
+		idPrefixMap.put(Datasource.class, "10");
+		idPrefixMap.put(Organization.class, "20");
+		idPrefixMap.put(Project.class, "40");
+		idPrefixMap.put(Dataset.class, "50");
+		idPrefixMap.put(OtherResearchProduct.class, "50");
+		idPrefixMap.put(Software.class, "50");
+		idPrefixMap.put(Publication.class, "50");
+	}
+
+	public static final Map<String, String> entityIdPrefix = Maps.newHashMap();
+
+	static {
+		entityIdPrefix.put("datasource", "10");
+		entityIdPrefix.put("organization", "20");
+		entityIdPrefix.put("project", "40");
+		entityIdPrefix.put("result", "50");
+	}
+
+	public static final Map<String, RelationInverse> relationInverseMap = Maps.newHashMap();
+
+	static {
+		relationInverseMap
+			.put(
+				"personResult_authorship_isAuthorOf", new RelationInverse()
+					.setRelation("isAuthorOf")
+					.setInverse("hasAuthor")
+					.setRelType("personResult")
+					.setSubReltype("authorship"));
+		relationInverseMap
+			.put(
+				"personResult_authorship_hasAuthor", new RelationInverse()
+					.setInverse("isAuthorOf")
+					.setRelation("hasAuthor")
+					.setRelType("personResult")
+					.setSubReltype("authorship"));
+		relationInverseMap
+			.put(
+				"projectOrganization_participation_isParticipant", new RelationInverse()
+					.setRelation("isParticipant")
+					.setInverse("hasParticipant")
+					.setRelType("projectOrganization")
+					.setSubReltype("participation"));
+		relationInverseMap
+			.put(
+				"projectOrganization_participation_hasParticipant", new RelationInverse()
+					.setInverse("isParticipant")
+					.setRelation("hasParticipant")
+					.setRelType("projectOrganization")
+					.setSubReltype("participation"));
+		relationInverseMap
+			.put(
+				"resultOrganization_affiliation_hasAuthorInstitution", new RelationInverse()
+					.setRelation("hasAuthorInstitution")
+					.setInverse("isAuthorInstitutionOf")
+					.setRelType("resultOrganization")
+					.setSubReltype("affiliation"));
+		relationInverseMap
+			.put(
+				"resultOrganization_affiliation_isAuthorInstitutionOf", new RelationInverse()
+					.setInverse("hasAuthorInstitution")
+					.setRelation("isAuthorInstitutionOf")
+					.setRelType("resultOrganization")
+					.setSubReltype("affiliation"));
+		relationInverseMap
+			.put(
+				"organizationOrganization_dedup_merges", new RelationInverse()
+					.setRelation("merges")
+					.setInverse("isMergedIn")
+					.setRelType("organizationOrganization")
+					.setSubReltype("dedup"));
+		relationInverseMap
+			.put(
+				"organizationOrganization_dedup_isMergedIn", new RelationInverse()
+					.setInverse("merges")
+					.setRelation("isMergedIn")
+					.setRelType("organizationOrganization")
+					.setSubReltype("dedup"));
+		relationInverseMap
+			.put(
+				"organizationOrganization_dedupSimilarity_isSimilarTo", new RelationInverse()
+					.setInverse("isSimilarTo")
+					.setRelation("isSimilarTo")
+					.setRelType("organizationOrganization")
+					.setSubReltype("dedupSimilarity"));
+
+		relationInverseMap
+			.put(
+				"resultProject_outcome_isProducedBy", new RelationInverse()
+					.setRelation("isProducedBy")
+					.setInverse("produces")
+					.setRelType("resultProject")
+					.setSubReltype("outcome"));
+		relationInverseMap
+			.put(
+				"resultProject_outcome_produces", new RelationInverse()
+					.setInverse("isProducedBy")
+					.setRelation("produces")
+					.setRelType("resultProject")
+					.setSubReltype("outcome"));
+		relationInverseMap
+			.put(
+				"projectPerson_contactPerson_isContact", new RelationInverse()
+					.setRelation("isContact")
+					.setInverse("hasContact")
+					.setRelType("projectPerson")
+					.setSubReltype("contactPerson"));
+		relationInverseMap
+			.put(
+				"projectPerson_contactPerson_hasContact", new RelationInverse()
+					.setInverse("isContact")
+					.setRelation("hasContact")
+					.setRelType("personPerson")
+					.setSubReltype("coAuthorship"));
+		relationInverseMap
+			.put(
+				"personPerson_coAuthorship_isCoauthorOf", new RelationInverse()
+					.setInverse("isCoAuthorOf")
+					.setRelation("isCoAuthorOf")
+					.setRelType("personPerson")
+					.setSubReltype("coAuthorship"));
+		relationInverseMap
+			.put(
+				"personPerson_dedup_merges", new RelationInverse()
+					.setInverse("isMergedIn")
+					.setRelation("merges")
+					.setRelType("personPerson")
+					.setSubReltype("dedup"));
+		relationInverseMap
+			.put(
+				"personPerson_dedup_isMergedIn", new RelationInverse()
+					.setInverse("merges")
+					.setRelation("isMergedIn")
+					.setRelType("personPerson")
+					.setSubReltype("dedup"));
+		relationInverseMap
+			.put(
+				"personPerson_dedupSimilarity_isSimilarTo", new RelationInverse()
+					.setInverse("isSimilarTo")
+					.setRelation("isSimilarTo")
+					.setRelType("personPerson")
+					.setSubReltype("dedupSimilarity"));
+		relationInverseMap
+			.put(
+				"datasourceOrganization_provision_isProvidedBy", new RelationInverse()
+					.setInverse("provides")
+					.setRelation("isProvidedBy")
+					.setRelType("datasourceOrganization")
+					.setSubReltype("provision"));
+		relationInverseMap
+			.put(
+				"datasourceOrganization_provision_provides", new RelationInverse()
+					.setInverse("isProvidedBy")
+					.setRelation("provides")
+					.setRelType("datasourceOrganization")
+					.setSubReltype("provision"));
+		relationInverseMap
+			.put(
+				"resultResult_similarity_hasAmongTopNSimilarDocuments", new RelationInverse()
+					.setInverse("isAmongTopNSimilarDocuments")
+					.setRelation("hasAmongTopNSimilarDocuments")
+					.setRelType("resultResult")
+					.setSubReltype("similarity"));
+		relationInverseMap
+			.put(
+				"resultResult_similarity_isAmongTopNSimilarDocuments", new RelationInverse()
+					.setInverse("hasAmongTopNSimilarDocuments")
+					.setRelation("isAmongTopNSimilarDocuments")
+					.setRelType("resultResult")
+					.setSubReltype("similarity"));
+		relationInverseMap
+			.put(
+				"resultResult_relationship_isRelatedTo", new RelationInverse()
+					.setInverse("isRelatedTo")
+					.setRelation("isRelatedTo")
+					.setRelType("resultResult")
+					.setSubReltype("relationship"));
+		relationInverseMap
+			.put(
+				"resultResult_similarity_isAmongTopNSimilarDocuments", new RelationInverse()
+					.setInverse("hasAmongTopNSimilarDocuments")
+					.setRelation("isAmongTopNSimilarDocuments")
+					.setRelType("resultResult")
+					.setSubReltype("similarity"));
+		relationInverseMap
+			.put(
+				"resultResult_supplement_isSupplementTo", new RelationInverse()
+					.setInverse("isSupplementedBy")
+					.setRelation("isSupplementTo")
+					.setRelType("resultResult")
+					.setSubReltype("supplement"));
+		relationInverseMap
+			.put(
+				"resultResult_supplement_isSupplementedBy", new RelationInverse()
+					.setInverse("isSupplementTo")
+					.setRelation("isSupplementedBy")
+					.setRelType("resultResult")
+					.setSubReltype("supplement"));
+		relationInverseMap
+			.put(
+				"resultResult_part_isPartOf", new RelationInverse()
+					.setInverse("hasPart")
+					.setRelation("isPartOf")
+					.setRelType("resultResult")
+					.setSubReltype("part"));
+		relationInverseMap
+			.put(
+				"resultResult_part_hasPart", new RelationInverse()
+					.setInverse("isPartOf")
+					.setRelation("hasPart")
+					.setRelType("resultResult")
+					.setSubReltype("part"));
+		relationInverseMap
+			.put(
+				"resultResult_dedup_merges", new RelationInverse()
+					.setInverse("isMergedIn")
+					.setRelation("merges")
+					.setRelType("resultResult")
+					.setSubReltype("dedup"));
+		relationInverseMap
+			.put(
+				"resultResult_dedup_isMergedIn", new RelationInverse()
+					.setInverse("merges")
+					.setRelation("isMergedIn")
+					.setRelType("resultResult")
+					.setSubReltype("dedup"));
+		relationInverseMap
+			.put(
+				"resultResult_dedupSimilarity_isSimilarTo", new RelationInverse()
+					.setInverse("isSimilarTo")
+					.setRelation("isSimilarTo")
+					.setRelType("resultResult")
+					.setSubReltype("dedupSimilarity"));
+
+	}
+
+	private static final String schemeTemplate = "dnet:%s_%s_relations";
+
+	private ModelSupport() {
+	}
+
+	public static <E extends OafEntity> String getIdPrefix(Class<E> clazz) {
+		return idPrefixMap.get(clazz);
+	}
+
+	/**
+	 * Checks subclass-superclass relationship.
+	 *
+	 * @param subClazzObject Subclass object instance
+	 * @param superClazzObject Superclass object instance
+	 * @param <X> Subclass type
+	 * @param <Y> Superclass type
+	 * @return True if X is a subclass of Y
+	 */
+	public static <X extends Oaf, Y extends Oaf> Boolean isSubClass(
+		X subClazzObject, Y superClazzObject) {
+		return isSubClass(subClazzObject.getClass(), superClazzObject.getClass());
+	}
+
+	/**
+	 * Checks subclass-superclass relationship.
+	 *
+	 * @param subClazzObject Subclass object instance
+	 * @param superClazz Superclass class
+	 * @param <X> Subclass type
+	 * @param <Y> Superclass type
+	 * @return True if X is a subclass of Y
+	 */
+	public static <X extends Oaf, Y extends Oaf> Boolean isSubClass(
+		X subClazzObject, Class<Y> superClazz) {
+		return isSubClass(subClazzObject.getClass(), superClazz);
+	}
+
+	/**
+	 * Checks subclass-superclass relationship.
+	 *
+	 * @param subClazz Subclass class
+	 * @param superClazz Superclass class
+	 * @param <X> Subclass type
+	 * @param <Y> Superclass type
+	 * @return True if X is a subclass of Y
+	 */
+	public static <X extends Oaf, Y extends Oaf> Boolean isSubClass(
+		Class<X> subClazz, Class<Y> superClazz) {
+		return superClazz.isAssignableFrom(subClazz);
+	}
+
+	/**
+	 * Lists all the OAF model classes
+	 *
+	 * @param <T>
+	 * @return
+	 */
+	public static <T extends Oaf> Class<T>[] getOafModelClasses() {
+		return new Class[] {
+			Author.class,
+			Context.class,
+			Country.class,
+			DataInfo.class,
+			Dataset.class,
+			Datasource.class,
+			ExternalReference.class,
+			ExtraInfo.class,
+			Field.class,
+			GeoLocation.class,
+			Instance.class,
+			Journal.class,
+			KeyValue.class,
+			Oaf.class,
+			OafEntity.class,
+			OAIProvenance.class,
+			Organization.class,
+			OriginDescription.class,
+			OtherResearchProduct.class,
+			Project.class,
+			Publication.class,
+			Qualifier.class,
+			Relation.class,
+			Result.class,
+			Software.class,
+			StructuredProperty.class
+		};
+	}
+
+	public static String getMainType(final EntityType type) {
+		return entityMapping.get(type).name();
+	}
+
+	public static boolean isResult(EntityType type) {
+		return MainEntityType.result.name().equals(getMainType(type));
+	}
+
+	public static String getScheme(final String sourceType, final String targetType) {
+		return String
+			.format(
+				schemeTemplate,
+				entityMapping.get(EntityType.valueOf(sourceType)).name(),
+				entityMapping.get(EntityType.valueOf(targetType)).name());
+	}
+
+	public static <T extends Oaf> String tableIdentifier(String dbName, String tableName) {
+
+		checkArgument(StringUtils.isNotBlank(dbName), "DB name cannot be empty");
+		checkArgument(StringUtils.isNotBlank(tableName), "table name cannot be empty");
+
+		return String.format("%s.%s", dbName, tableName);
+	}
+
+	public static <T extends Oaf> String tableIdentifier(String dbName, Class<T> clazz) {
+
+		checkArgument(Objects.nonNull(clazz), "clazz is needed to derive the table name, thus cannot be null");
+
+		return tableIdentifier(dbName, clazz.getSimpleName().toLowerCase());
+	}
+
+	public static <T extends Oaf> Function<T, String> idFn() {
+		return x -> {
+			if (isSubClass(x, Relation.class)) {
+				return idFnForRelation(x);
+			}
+			return idFnForOafEntity(x);
+		};
+	}
+
+	private static <T extends Oaf> String idFnForRelation(T t) {
+		Relation r = (Relation) t;
+		return Optional
+			.ofNullable(r.getSource())
+			.map(
+				source -> Optional
+					.ofNullable(r.getTarget())
+					.map(
+						target -> Optional
+							.ofNullable(r.getRelType())
+							.map(
+								relType -> Optional
+									.ofNullable(r.getSubRelType())
+									.map(
+										subRelType -> Optional
+											.ofNullable(r.getRelClass())
+											.map(
+												relClass -> String
+													.join(
+														source,
+														target,
+														relType,
+														subRelType,
+														relClass))
+											.orElse(
+												String
+													.join(
+														source,
+														target,
+														relType,
+														subRelType)))
+									.orElse(String.join(source, target, relType)))
+							.orElse(String.join(source, target)))
+					.orElse(source))
+			.orElse(null);
+	}
+
+	private static <T extends Oaf> String idFnForOafEntity(T t) {
+		return ((OafEntity) t).getId();
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/common/RelationInverse.java

@@ -0,0 +1,46 @@
+
+package eu.dnetlib.dhp.schema.common;
+
+public class RelationInverse {
+	private String relation;
+	private String inverse;
+	private String relType;
+	private String subReltype;
+
+	public String getRelType() {
+		return relType;
+	}
+
+	public RelationInverse setRelType(String relType) {
+		this.relType = relType;
+		return this;
+	}
+
+	public String getSubReltype() {
+		return subReltype;
+	}
+
+	public RelationInverse setSubReltype(String subReltype) {
+		this.subReltype = subReltype;
+		return this;
+	}
+
+	public String getRelation() {
+		return relation;
+	}
+
+	public RelationInverse setRelation(String relation) {
+		this.relation = relation;
+		return this;
+	}
+
+	public String getInverse() {
+		return inverse;
+	}
+
+	public RelationInverse setInverse(String inverse) {
+		this.inverse = inverse;
+		return this;
+	}
+
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Author.java

@@ -0,0 +1,89 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.*;
+
+public class Author implements Serializable {
+
+	private String fullname;
+
+	private String name;
+
+	private String surname;
+
+	private Integer rank;
+
+	private List<StructuredProperty> pid;
+
+	private List<Field<String>> affiliation;
+
+	public String getFullname() {
+		return fullname;
+	}
+
+	public void setFullname(String fullname) {
+		this.fullname = fullname;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getSurname() {
+		return surname;
+	}
+
+	public void setSurname(String surname) {
+		this.surname = surname;
+	}
+
+	public Integer getRank() {
+		return rank;
+	}
+
+	public void setRank(Integer rank) {
+		this.rank = rank;
+	}
+
+	public List<StructuredProperty> getPid() {
+		return pid;
+	}
+
+	public void setPid(List<StructuredProperty> pid) {
+		this.pid = pid;
+	}
+
+	public List<Field<String>> getAffiliation() {
+		return affiliation;
+	}
+
+	public void setAffiliation(List<Field<String>> affiliation) {
+		this.affiliation = affiliation;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		Author author = (Author) o;
+		return Objects.equals(fullname, author.fullname)
+			&& Objects.equals(name, author.name)
+			&& Objects.equals(surname, author.surname)
+			&& Objects.equals(rank, author.rank)
+			&& Objects.equals(pid, author.pid)
+			&& Objects.equals(affiliation, author.affiliation);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(fullname, name, surname, rank, pid, affiliation);
+	}
+
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Context.java

@@ -0,0 +1,46 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class Context implements Serializable {
+	private String id;
+
+	private List<DataInfo> dataInfo;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public List<DataInfo> getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(List<DataInfo> dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	@Override
+	public int hashCode() {
+		return id == null ? 0 : id.hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+
+		Context other = (Context) obj;
+
+		return id.equals(other.getId());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Country.java

@@ -0,0 +1,34 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.util.Objects;
+
+public class Country extends Qualifier {
+
+	private DataInfo dataInfo;
+
+	public DataInfo getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(DataInfo dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		if (!super.equals(o))
+			return false;
+		Country country = (Country) o;
+		return Objects.equals(dataInfo, country.dataInfo);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(super.hashCode(), dataInfo);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/DataInfo.java

@@ -0,0 +1,85 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class DataInfo implements Serializable {
+
+	private Boolean invisible = false;
+	private Boolean inferred;
+	private Boolean deletedbyinference = false;
+	private String trust;
+	private String inferenceprovenance;
+	private Qualifier provenanceaction;
+
+	public Boolean getInvisible() {
+		return invisible;
+	}
+
+	public void setInvisible(Boolean invisible) {
+		this.invisible = invisible;
+	}
+
+	public Boolean getInferred() {
+		return inferred;
+	}
+
+	public void setInferred(Boolean inferred) {
+		this.inferred = inferred;
+	}
+
+	public Boolean getDeletedbyinference() {
+		return deletedbyinference;
+	}
+
+	public void setDeletedbyinference(Boolean deletedbyinference) {
+		this.deletedbyinference = deletedbyinference;
+	}
+
+	public String getTrust() {
+		return trust;
+	}
+
+	public void setTrust(String trust) {
+		this.trust = trust;
+	}
+
+	public String getInferenceprovenance() {
+		return inferenceprovenance;
+	}
+
+	public void setInferenceprovenance(String inferenceprovenance) {
+		this.inferenceprovenance = inferenceprovenance;
+	}
+
+	public Qualifier getProvenanceaction() {
+		return provenanceaction;
+	}
+
+	public void setProvenanceaction(Qualifier provenanceaction) {
+		this.provenanceaction = provenanceaction;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		DataInfo dataInfo = (DataInfo) o;
+		return Objects.equals(invisible, dataInfo.invisible)
+			&& Objects.equals(inferred, dataInfo.inferred)
+			&& Objects.equals(deletedbyinference, dataInfo.deletedbyinference)
+			&& Objects.equals(trust, dataInfo.trust)
+			&& Objects.equals(inferenceprovenance, dataInfo.inferenceprovenance)
+			&& Objects.equals(provenanceaction, dataInfo.provenanceaction);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects
+			.hash(
+				invisible, inferred, deletedbyinference, trust, inferenceprovenance, provenanceaction);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Dataset.java

@@ -0,0 +1,116 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+
+public class Dataset extends Result implements Serializable {
+
+	private Field<String> storagedate;
+
+	// candidate for removal
+	private Field<String> device;
+
+	private Field<String> size;
+
+	private Field<String> version;
+
+	private Field<String> lastmetadataupdate;
+
+	private Field<String> metadataversionnumber;
+
+	private List<GeoLocation> geolocation;
+
+	public Dataset() {
+		setResulttype(ModelConstants.DATASET_DEFAULT_RESULTTYPE);
+	}
+
+	public Field<String> getStoragedate() {
+		return storagedate;
+	}
+
+	public void setStoragedate(Field<String> storagedate) {
+		this.storagedate = storagedate;
+	}
+
+	public Field<String> getDevice() {
+		return device;
+	}
+
+	public void setDevice(Field<String> device) {
+		this.device = device;
+	}
+
+	public Field<String> getSize() {
+		return size;
+	}
+
+	public void setSize(Field<String> size) {
+		this.size = size;
+	}
+
+	public Field<String> getVersion() {
+		return version;
+	}
+
+	public void setVersion(Field<String> version) {
+		this.version = version;
+	}
+
+	public Field<String> getLastmetadataupdate() {
+		return lastmetadataupdate;
+	}
+
+	public void setLastmetadataupdate(Field<String> lastmetadataupdate) {
+		this.lastmetadataupdate = lastmetadataupdate;
+	}
+
+	public Field<String> getMetadataversionnumber() {
+		return metadataversionnumber;
+	}
+
+	public void setMetadataversionnumber(Field<String> metadataversionnumber) {
+		this.metadataversionnumber = metadataversionnumber;
+	}
+
+	public List<GeoLocation> getGeolocation() {
+		return geolocation;
+	}
+
+	public void setGeolocation(List<GeoLocation> geolocation) {
+		this.geolocation = geolocation;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!Dataset.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		final Dataset d = (Dataset) e;
+
+		storagedate = d.getStoragedate() != null && compareTrust(this, e) < 0 ? d.getStoragedate() : storagedate;
+
+		device = d.getDevice() != null && compareTrust(this, e) < 0 ? d.getDevice() : device;
+
+		size = d.getSize() != null && compareTrust(this, e) < 0 ? d.getSize() : size;
+
+		version = d.getVersion() != null && compareTrust(this, e) < 0 ? d.getVersion() : version;
+
+		lastmetadataupdate = d.getLastmetadataupdate() != null && compareTrust(this, e) < 0
+			? d.getLastmetadataupdate()
+			: lastmetadataupdate;
+
+		metadataversionnumber = d.getMetadataversionnumber() != null && compareTrust(this, e) < 0
+			? d.getMetadataversionnumber()
+			: metadataversionnumber;
+
+		geolocation = mergeLists(geolocation, d.getGeolocation());
+
+		mergeOAFDataInfo(d);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Datasource.java

@@ -0,0 +1,472 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class Datasource extends OafEntity implements Serializable {
+
+	private Qualifier datasourcetype;
+
+	private Qualifier openairecompatibility;
+
+	private Field<String> officialname;
+
+	private Field<String> englishname;
+
+	private Field<String> websiteurl;
+
+	private Field<String> logourl;
+
+	private Field<String> contactemail;
+
+	private Field<String> namespaceprefix;
+
+	private Field<String> latitude;
+
+	private Field<String> longitude;
+
+	private Field<String> dateofvalidation;
+
+	private Field<String> description;
+
+	private List<StructuredProperty> subjects;
+
+	// opendoar specific fields (od*)
+	private Field<String> odnumberofitems;
+
+	private Field<String> odnumberofitemsdate;
+
+	private Field<String> odpolicies;
+
+	private List<Field<String>> odlanguages;
+
+	private List<Field<String>> odcontenttypes;
+
+	private List<Field<String>> accessinfopackage;
+
+	// re3data fields
+	private Field<String> releasestartdate;
+
+	private Field<String> releaseenddate;
+
+	private Field<String> missionstatementurl;
+
+	private Field<Boolean> dataprovider;
+
+	private Field<Boolean> serviceprovider;
+
+	// {open, restricted or closed}
+	private Field<String> databaseaccesstype;
+
+	// {open, restricted or closed}
+	private Field<String> datauploadtype;
+
+	// {feeRequired, registration, other}
+	private Field<String> databaseaccessrestriction;
+
+	// {feeRequired, registration, other}
+	private Field<String> datauploadrestriction;
+
+	private Field<Boolean> versioning;
+
+	private Field<String> citationguidelineurl;
+
+	// {yes, no, uknown}
+	private Field<String> qualitymanagementkind;
+
+	private Field<String> pidsystems;
+
+	private Field<String> certificates;
+
+	private List<KeyValue> policies;
+
+	private Journal journal;
+
+	public Qualifier getDatasourcetype() {
+		return datasourcetype;
+	}
+
+	public void setDatasourcetype(Qualifier datasourcetype) {
+		this.datasourcetype = datasourcetype;
+	}
+
+	public Qualifier getOpenairecompatibility() {
+		return openairecompatibility;
+	}
+
+	public void setOpenairecompatibility(Qualifier openairecompatibility) {
+		this.openairecompatibility = openairecompatibility;
+	}
+
+	public Field<String> getOfficialname() {
+		return officialname;
+	}
+
+	public void setOfficialname(Field<String> officialname) {
+		this.officialname = officialname;
+	}
+
+	public Field<String> getEnglishname() {
+		return englishname;
+	}
+
+	public void setEnglishname(Field<String> englishname) {
+		this.englishname = englishname;
+	}
+
+	public Field<String> getWebsiteurl() {
+		return websiteurl;
+	}
+
+	public void setWebsiteurl(Field<String> websiteurl) {
+		this.websiteurl = websiteurl;
+	}
+
+	public Field<String> getLogourl() {
+		return logourl;
+	}
+
+	public void setLogourl(Field<String> logourl) {
+		this.logourl = logourl;
+	}
+
+	public Field<String> getContactemail() {
+		return contactemail;
+	}
+
+	public void setContactemail(Field<String> contactemail) {
+		this.contactemail = contactemail;
+	}
+
+	public Field<String> getNamespaceprefix() {
+		return namespaceprefix;
+	}
+
+	public void setNamespaceprefix(Field<String> namespaceprefix) {
+		this.namespaceprefix = namespaceprefix;
+	}
+
+	public Field<String> getLatitude() {
+		return latitude;
+	}
+
+	public void setLatitude(Field<String> latitude) {
+		this.latitude = latitude;
+	}
+
+	public Field<String> getLongitude() {
+		return longitude;
+	}
+
+	public void setLongitude(Field<String> longitude) {
+		this.longitude = longitude;
+	}
+
+	public Field<String> getDateofvalidation() {
+		return dateofvalidation;
+	}
+
+	public void setDateofvalidation(Field<String> dateofvalidation) {
+		this.dateofvalidation = dateofvalidation;
+	}
+
+	public Field<String> getDescription() {
+		return description;
+	}
+
+	public void setDescription(Field<String> description) {
+		this.description = description;
+	}
+
+	public List<StructuredProperty> getSubjects() {
+		return subjects;
+	}
+
+	public void setSubjects(List<StructuredProperty> subjects) {
+		this.subjects = subjects;
+	}
+
+	public Field<String> getOdnumberofitems() {
+		return odnumberofitems;
+	}
+
+	public void setOdnumberofitems(Field<String> odnumberofitems) {
+		this.odnumberofitems = odnumberofitems;
+	}
+
+	public Field<String> getOdnumberofitemsdate() {
+		return odnumberofitemsdate;
+	}
+
+	public void setOdnumberofitemsdate(Field<String> odnumberofitemsdate) {
+		this.odnumberofitemsdate = odnumberofitemsdate;
+	}
+
+	public Field<String> getOdpolicies() {
+		return odpolicies;
+	}
+
+	public void setOdpolicies(Field<String> odpolicies) {
+		this.odpolicies = odpolicies;
+	}
+
+	public List<Field<String>> getOdlanguages() {
+		return odlanguages;
+	}
+
+	public void setOdlanguages(List<Field<String>> odlanguages) {
+		this.odlanguages = odlanguages;
+	}
+
+	public List<Field<String>> getOdcontenttypes() {
+		return odcontenttypes;
+	}
+
+	public void setOdcontenttypes(List<Field<String>> odcontenttypes) {
+		this.odcontenttypes = odcontenttypes;
+	}
+
+	public List<Field<String>> getAccessinfopackage() {
+		return accessinfopackage;
+	}
+
+	public void setAccessinfopackage(List<Field<String>> accessinfopackage) {
+		this.accessinfopackage = accessinfopackage;
+	}
+
+	public Field<String> getReleasestartdate() {
+		return releasestartdate;
+	}
+
+	public void setReleasestartdate(Field<String> releasestartdate) {
+		this.releasestartdate = releasestartdate;
+	}
+
+	public Field<String> getReleaseenddate() {
+		return releaseenddate;
+	}
+
+	public void setReleaseenddate(Field<String> releaseenddate) {
+		this.releaseenddate = releaseenddate;
+	}
+
+	public Field<String> getMissionstatementurl() {
+		return missionstatementurl;
+	}
+
+	public void setMissionstatementurl(Field<String> missionstatementurl) {
+		this.missionstatementurl = missionstatementurl;
+	}
+
+	public Field<Boolean> getDataprovider() {
+		return dataprovider;
+	}
+
+	public void setDataprovider(Field<Boolean> dataprovider) {
+		this.dataprovider = dataprovider;
+	}
+
+	public Field<Boolean> getServiceprovider() {
+		return serviceprovider;
+	}
+
+	public void setServiceprovider(Field<Boolean> serviceprovider) {
+		this.serviceprovider = serviceprovider;
+	}
+
+	public Field<String> getDatabaseaccesstype() {
+		return databaseaccesstype;
+	}
+
+	public void setDatabaseaccesstype(Field<String> databaseaccesstype) {
+		this.databaseaccesstype = databaseaccesstype;
+	}
+
+	public Field<String> getDatauploadtype() {
+		return datauploadtype;
+	}
+
+	public void setDatauploadtype(Field<String> datauploadtype) {
+		this.datauploadtype = datauploadtype;
+	}
+
+	public Field<String> getDatabaseaccessrestriction() {
+		return databaseaccessrestriction;
+	}
+
+	public void setDatabaseaccessrestriction(Field<String> databaseaccessrestriction) {
+		this.databaseaccessrestriction = databaseaccessrestriction;
+	}
+
+	public Field<String> getDatauploadrestriction() {
+		return datauploadrestriction;
+	}
+
+	public void setDatauploadrestriction(Field<String> datauploadrestriction) {
+		this.datauploadrestriction = datauploadrestriction;
+	}
+
+	public Field<Boolean> getVersioning() {
+		return versioning;
+	}
+
+	public void setVersioning(Field<Boolean> versioning) {
+		this.versioning = versioning;
+	}
+
+	public Field<String> getCitationguidelineurl() {
+		return citationguidelineurl;
+	}
+
+	public void setCitationguidelineurl(Field<String> citationguidelineurl) {
+		this.citationguidelineurl = citationguidelineurl;
+	}
+
+	public Field<String> getQualitymanagementkind() {
+		return qualitymanagementkind;
+	}
+
+	public void setQualitymanagementkind(Field<String> qualitymanagementkind) {
+		this.qualitymanagementkind = qualitymanagementkind;
+	}
+
+	public Field<String> getPidsystems() {
+		return pidsystems;
+	}
+
+	public void setPidsystems(Field<String> pidsystems) {
+		this.pidsystems = pidsystems;
+	}
+
+	public Field<String> getCertificates() {
+		return certificates;
+	}
+
+	public void setCertificates(Field<String> certificates) {
+		this.certificates = certificates;
+	}
+
+	public List<KeyValue> getPolicies() {
+		return policies;
+	}
+
+	public void setPolicies(List<KeyValue> policies) {
+		this.policies = policies;
+	}
+
+	public Journal getJournal() {
+		return journal;
+	}
+
+	public void setJournal(Journal journal) {
+		this.journal = journal;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!Datasource.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		Datasource d = (Datasource) e;
+
+		datasourcetype = d.getDatasourcetype() != null && compareTrust(this, e) < 0
+			? d.getDatasourcetype()
+			: datasourcetype;
+		openairecompatibility = d.getOpenairecompatibility() != null && compareTrust(this, e) < 0
+			? d.getOpenairecompatibility()
+			: openairecompatibility;
+		officialname = d.getOfficialname() != null && compareTrust(this, e) < 0
+			? d.getOfficialname()
+			: officialname;
+		englishname = d.getEnglishname() != null && compareTrust(this, e) < 0 ? d.getEnglishname() : officialname;
+		websiteurl = d.getWebsiteurl() != null && compareTrust(this, e) < 0 ? d.getWebsiteurl() : websiteurl;
+		logourl = d.getLogourl() != null && compareTrust(this, e) < 0 ? d.getLogourl() : getLogourl();
+		contactemail = d.getContactemail() != null && compareTrust(this, e) < 0
+			? d.getContactemail()
+			: contactemail;
+		namespaceprefix = d.getNamespaceprefix() != null && compareTrust(this, e) < 0
+			? d.getNamespaceprefix()
+			: namespaceprefix;
+		latitude = d.getLatitude() != null && compareTrust(this, e) < 0 ? d.getLatitude() : latitude;
+		longitude = d.getLongitude() != null && compareTrust(this, e) < 0 ? d.getLongitude() : longitude;
+		dateofvalidation = d.getDateofvalidation() != null && compareTrust(this, e) < 0
+			? d.getDateofvalidation()
+			: dateofvalidation;
+		description = d.getDescription() != null && compareTrust(this, e) < 0 ? d.getDescription() : description;
+		subjects = mergeLists(subjects, d.getSubjects());
+
+		// opendoar specific fields (od*)
+		odnumberofitems = d.getOdnumberofitems() != null && compareTrust(this, e) < 0
+			? d.getOdnumberofitems()
+			: odnumberofitems;
+		odnumberofitemsdate = d.getOdnumberofitemsdate() != null && compareTrust(this, e) < 0
+			? d.getOdnumberofitemsdate()
+			: odnumberofitemsdate;
+		odpolicies = d.getOdpolicies() != null && compareTrust(this, e) < 0 ? d.getOdpolicies() : odpolicies;
+		odlanguages = mergeLists(odlanguages, d.getOdlanguages());
+		odcontenttypes = mergeLists(odcontenttypes, d.getOdcontenttypes());
+		accessinfopackage = mergeLists(accessinfopackage, d.getAccessinfopackage());
+
+		// re3data fields
+		releasestartdate = d.getReleasestartdate() != null && compareTrust(this, e) < 0
+			? d.getReleasestartdate()
+			: releasestartdate;
+		releaseenddate = d.getReleaseenddate() != null && compareTrust(this, e) < 0
+			? d.getReleaseenddate()
+			: releaseenddate;
+		missionstatementurl = d.getMissionstatementurl() != null && compareTrust(this, e) < 0
+			? d.getMissionstatementurl()
+			: missionstatementurl;
+		dataprovider = d.getDataprovider() != null && compareTrust(this, e) < 0
+			? d.getDataprovider()
+			: dataprovider;
+		serviceprovider = d.getServiceprovider() != null && compareTrust(this, e) < 0
+			? d.getServiceprovider()
+			: serviceprovider;
+
+		// {open, restricted or closed}
+		databaseaccesstype = d.getDatabaseaccesstype() != null && compareTrust(this, e) < 0
+			? d.getDatabaseaccesstype()
+			: databaseaccesstype;
+
+		// {open, restricted or closed}
+		datauploadtype = d.getDatauploadtype() != null && compareTrust(this, e) < 0
+			? d.getDatauploadtype()
+			: datauploadtype;
+
+		// {feeRequired, registration, other}
+		databaseaccessrestriction = d.getDatabaseaccessrestriction() != null && compareTrust(this, e) < 0
+			? d.getDatabaseaccessrestriction()
+			: databaseaccessrestriction;
+
+		// {feeRequired, registration, other}
+		datauploadrestriction = d.getDatauploadrestriction() != null && compareTrust(this, e) < 0
+			? d.getDatauploadrestriction()
+			: datauploadrestriction;
+
+		versioning = d.getVersioning() != null && compareTrust(this, e) < 0 ? d.getVersioning() : versioning;
+		citationguidelineurl = d.getCitationguidelineurl() != null && compareTrust(this, e) < 0
+			? d.getCitationguidelineurl()
+			: citationguidelineurl;
+
+		// {yes, no, unknown}
+		qualitymanagementkind = d.getQualitymanagementkind() != null && compareTrust(this, e) < 0
+			? d.getQualitymanagementkind()
+			: qualitymanagementkind;
+		pidsystems = d.getPidsystems() != null && compareTrust(this, e) < 0 ? d.getPidsystems() : pidsystems;
+
+		certificates = d.getCertificates() != null && compareTrust(this, e) < 0
+			? d.getCertificates()
+			: certificates;
+
+		policies = mergeLists(policies, d.getPolicies());
+
+		journal = d.getJournal() != null && compareTrust(this, e) < 0 ? d.getJournal() : journal;
+
+		mergeOAFDataInfo(e);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/ExternalReference.java

@@ -0,0 +1,119 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class ExternalReference implements Serializable {
+	// source
+	private String sitename;
+
+	// title
+	private String label;
+
+	// text()
+	private String url;
+
+	// ?? not mapped yet ??
+	private String description;
+
+	// type
+	private Qualifier qualifier;
+
+	// site internal identifier
+	private String refidentifier;
+
+	// maps the oaf:reference/@query attribute
+	private String query;
+
+	// ExternalReferences might be also inferred
+	private DataInfo dataInfo;
+
+	public String getSitename() {
+		return sitename;
+	}
+
+	public void setSitename(String sitename) {
+		this.sitename = sitename;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(String label) {
+		this.label = label;
+	}
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(String url) {
+		this.url = url;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(String description) {
+		this.description = description;
+	}
+
+	public Qualifier getQualifier() {
+		return qualifier;
+	}
+
+	public void setQualifier(Qualifier qualifier) {
+		this.qualifier = qualifier;
+	}
+
+	public String getRefidentifier() {
+		return refidentifier;
+	}
+
+	public void setRefidentifier(String refidentifier) {
+		this.refidentifier = refidentifier;
+	}
+
+	public String getQuery() {
+		return query;
+	}
+
+	public void setQuery(String query) {
+		this.query = query;
+	}
+
+	public DataInfo getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(DataInfo dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		ExternalReference that = (ExternalReference) o;
+		return Objects.equals(sitename, that.sitename)
+			&& Objects.equals(label, that.label)
+			&& Objects.equals(url, that.url)
+			&& Objects.equals(description, that.description)
+			&& Objects.equals(qualifier, that.qualifier)
+			&& Objects.equals(refidentifier, that.refidentifier)
+			&& Objects.equals(query, that.query)
+			&& Objects.equals(dataInfo, that.dataInfo);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects
+			.hash(
+				sitename, label, url, description, qualifier, refidentifier, query, dataInfo);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/ExtraInfo.java

@@ -0,0 +1,77 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class ExtraInfo implements Serializable {
+	private String name;
+
+	private String typology;
+
+	private String provenance;
+
+	private String trust;
+
+	// json containing a Citation or Statistics
+	private String value;
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getTypology() {
+		return typology;
+	}
+
+	public void setTypology(String typology) {
+		this.typology = typology;
+	}
+
+	public String getProvenance() {
+		return provenance;
+	}
+
+	public void setProvenance(String provenance) {
+		this.provenance = provenance;
+	}
+
+	public String getTrust() {
+		return trust;
+	}
+
+	public void setTrust(String trust) {
+		this.trust = trust;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(String value) {
+		this.value = value;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		ExtraInfo extraInfo = (ExtraInfo) o;
+		return Objects.equals(name, extraInfo.name)
+			&& Objects.equals(typology, extraInfo.typology)
+			&& Objects.equals(provenance, extraInfo.provenance)
+			&& Objects.equals(trust, extraInfo.trust)
+			&& Objects.equals(value, extraInfo.value);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(name, typology, provenance, trust, value);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Field.java

@@ -0,0 +1,45 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class Field<T> implements Serializable {
+
+	private T value;
+
+	private DataInfo dataInfo;
+
+	public T getValue() {
+		return value;
+	}
+
+	public void setValue(T value) {
+		this.value = value;
+	}
+
+	public DataInfo getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(DataInfo dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	@Override
+	public int hashCode() {
+		return getValue() == null ? 0 : getValue().hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		Field<T> other = (Field<T>) obj;
+		return Objects.equals(getValue(), other.getValue());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/GeoLocation.java

@@ -0,0 +1,76 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public class GeoLocation implements Serializable {
+
+	private String point;
+
+	private String box;
+
+	private String place;
+
+	public String getPoint() {
+		return point;
+	}
+
+	public void setPoint(String point) {
+		this.point = point;
+	}
+
+	public String getBox() {
+		return box;
+	}
+
+	public void setBox(String box) {
+		this.box = box;
+	}
+
+	public String getPlace() {
+		return place;
+	}
+
+	public void setPlace(String place) {
+		this.place = place;
+	}
+
+	@JsonIgnore
+	public boolean isBlank() {
+		return StringUtils.isBlank(point) && StringUtils.isBlank(box) && StringUtils.isBlank(place);
+	}
+
+	public String toComparableString() {
+		return isBlank()
+			? ""
+			: String
+				.format(
+					"%s::%s%s",
+					point != null ? point.toLowerCase() : "",
+					box != null ? box.toLowerCase() : "",
+					place != null ? place.toLowerCase() : "");
+	}
+
+	@Override
+	public int hashCode() {
+		return toComparableString().hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+
+		GeoLocation other = (GeoLocation) obj;
+
+		return toComparableString().equals(other.toComparableString());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Instance.java

@@ -0,0 +1,152 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class Instance implements Serializable {
+
+	private Field<String> license;
+
+	private Qualifier accessright;
+
+	private Qualifier instancetype;
+
+	private KeyValue hostedby;
+
+	private List<String> url;
+
+	// other research products specifc
+	private String distributionlocation;
+
+	private KeyValue collectedfrom;
+
+	private Field<String> dateofacceptance;
+
+	// ( article | book ) processing charges. Defined here to cope with possible wrongly typed
+	// results
+	private Field<String> processingchargeamount;
+
+	// currency - alphabetic code describe in ISO-4217. Defined here to cope with possible wrongly
+	// typed results
+	private Field<String> processingchargecurrency;
+
+	private Qualifier refereed; // peer-review status
+
+	public Field<String> getLicense() {
+		return license;
+	}
+
+	public void setLicense(Field<String> license) {
+		this.license = license;
+	}
+
+	public Qualifier getAccessright() {
+		return accessright;
+	}
+
+	public void setAccessright(Qualifier accessright) {
+		this.accessright = accessright;
+	}
+
+	public Qualifier getInstancetype() {
+		return instancetype;
+	}
+
+	public void setInstancetype(Qualifier instancetype) {
+		this.instancetype = instancetype;
+	}
+
+	public KeyValue getHostedby() {
+		return hostedby;
+	}
+
+	public void setHostedby(KeyValue hostedby) {
+		this.hostedby = hostedby;
+	}
+
+	public List<String> getUrl() {
+		return url;
+	}
+
+	public void setUrl(List<String> url) {
+		this.url = url;
+	}
+
+	public String getDistributionlocation() {
+		return distributionlocation;
+	}
+
+	public void setDistributionlocation(String distributionlocation) {
+		this.distributionlocation = distributionlocation;
+	}
+
+	public KeyValue getCollectedfrom() {
+		return collectedfrom;
+	}
+
+	public void setCollectedfrom(KeyValue collectedfrom) {
+		this.collectedfrom = collectedfrom;
+	}
+
+	public Field<String> getDateofacceptance() {
+		return dateofacceptance;
+	}
+
+	public void setDateofacceptance(Field<String> dateofacceptance) {
+		this.dateofacceptance = dateofacceptance;
+	}
+
+	public Field<String> getProcessingchargeamount() {
+		return processingchargeamount;
+	}
+
+	public void setProcessingchargeamount(Field<String> processingchargeamount) {
+		this.processingchargeamount = processingchargeamount;
+	}
+
+	public Field<String> getProcessingchargecurrency() {
+		return processingchargecurrency;
+	}
+
+	public void setProcessingchargecurrency(Field<String> processingchargecurrency) {
+		this.processingchargecurrency = processingchargecurrency;
+	}
+
+	public Qualifier getRefereed() {
+		return refereed;
+	}
+
+	public void setRefereed(Qualifier refereed) {
+		this.refereed = refereed;
+	}
+
+	public String toComparableString() {
+		return String
+			.format(
+				"%s::%s::%s::%s",
+				hostedby != null && hostedby.getKey() != null ? hostedby.getKey().toLowerCase() : "",
+				accessright != null && accessright.getClassid() != null ? accessright.getClassid() : "",
+				instancetype != null && instancetype.getClassid() != null ? instancetype.getClassid() : "",
+				url != null ? url : "");
+	}
+
+	@Override
+	public int hashCode() {
+		return toComparableString().hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+
+		Instance other = (Instance) obj;
+
+		return toComparableString().equals(other.toComparableString());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Journal.java

@@ -0,0 +1,167 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class Journal implements Serializable {
+
+	private String name;
+
+	private String issnPrinted;
+
+	private String issnOnline;
+
+	private String issnLinking;
+
+	private String ep;
+
+	private String iss;
+
+	private String sp;
+
+	private String vol;
+
+	private String edition;
+
+	private String conferenceplace;
+
+	private String conferencedate;
+
+	private DataInfo dataInfo;
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getIssnPrinted() {
+		return issnPrinted;
+	}
+
+	public void setIssnPrinted(String issnPrinted) {
+		this.issnPrinted = issnPrinted;
+	}
+
+	public String getIssnOnline() {
+		return issnOnline;
+	}
+
+	public void setIssnOnline(String issnOnline) {
+		this.issnOnline = issnOnline;
+	}
+
+	public String getIssnLinking() {
+		return issnLinking;
+	}
+
+	public void setIssnLinking(String issnLinking) {
+		this.issnLinking = issnLinking;
+	}
+
+	public String getEp() {
+		return ep;
+	}
+
+	public void setEp(String ep) {
+		this.ep = ep;
+	}
+
+	public String getIss() {
+		return iss;
+	}
+
+	public void setIss(String iss) {
+		this.iss = iss;
+	}
+
+	public String getSp() {
+		return sp;
+	}
+
+	public void setSp(String sp) {
+		this.sp = sp;
+	}
+
+	public String getVol() {
+		return vol;
+	}
+
+	public void setVol(String vol) {
+		this.vol = vol;
+	}
+
+	public String getEdition() {
+		return edition;
+	}
+
+	public void setEdition(String edition) {
+		this.edition = edition;
+	}
+
+	public String getConferenceplace() {
+		return conferenceplace;
+	}
+
+	public void setConferenceplace(String conferenceplace) {
+		this.conferenceplace = conferenceplace;
+	}
+
+	public String getConferencedate() {
+		return conferencedate;
+	}
+
+	public void setConferencedate(String conferencedate) {
+		this.conferencedate = conferencedate;
+	}
+
+	public DataInfo getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(DataInfo dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		Journal journal = (Journal) o;
+		return Objects.equals(name, journal.name)
+			&& Objects.equals(issnPrinted, journal.issnPrinted)
+			&& Objects.equals(issnOnline, journal.issnOnline)
+			&& Objects.equals(issnLinking, journal.issnLinking)
+			&& Objects.equals(ep, journal.ep)
+			&& Objects.equals(iss, journal.iss)
+			&& Objects.equals(sp, journal.sp)
+			&& Objects.equals(vol, journal.vol)
+			&& Objects.equals(edition, journal.edition)
+			&& Objects.equals(conferenceplace, journal.conferenceplace)
+			&& Objects.equals(conferencedate, journal.conferencedate)
+			&& Objects.equals(dataInfo, journal.dataInfo);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects
+			.hash(
+				name,
+				issnPrinted,
+				issnOnline,
+				issnLinking,
+				ep,
+				iss,
+				sp,
+				vol,
+				edition,
+				conferenceplace,
+				conferencedate,
+				dataInfo);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/KeyValue.java

@@ -0,0 +1,74 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public class KeyValue implements Serializable {
+
+	private String key;
+
+	private String value;
+
+	private DataInfo dataInfo;
+
+	public String getKey() {
+		return key;
+	}
+
+	public void setKey(String key) {
+		this.key = key;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(String value) {
+		this.value = value;
+	}
+
+	public DataInfo getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(DataInfo dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	public String toComparableString() {
+		return isBlank()
+			? ""
+			: String
+				.format(
+					"%s::%s",
+					key != null ? key.toLowerCase() : "", value != null ? value.toLowerCase() : "");
+	}
+
+	@JsonIgnore
+	public boolean isBlank() {
+		return StringUtils.isBlank(key) && StringUtils.isBlank(value);
+	}
+
+	@Override
+	public int hashCode() {
+		return toComparableString().hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+
+		KeyValue other = (KeyValue) obj;
+
+		return toComparableString().equals(other.toComparableString());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Measure.java

@@ -0,0 +1,59 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.util.List;
+
+import com.google.common.base.Objects;
+
+/**
+ * Represent a measure, must be further described by a system available resource providing name and descriptions.
+ */
+public class Measure {
+
+	/**
+	 * Unique measure identifier.
+	 */
+	private String id;
+
+	/**
+	 * List of units associated with this measure. KeyValue provides a pair to store the laber (key) and the value, plus
+	 * common provenance information.
+	 */
+	private List<KeyValue> unit;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public List<KeyValue> getUnit() {
+		return unit;
+	}
+
+	public void setUnit(List<KeyValue> unit) {
+		this.unit = unit;
+	}
+
+	public void mergeFrom(Measure m) {
+		// TODO
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		Measure measure = (Measure) o;
+		return Objects.equal(id, measure.id) &&
+			Objects.equal(unit, measure.unit);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hashCode(id, unit);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OAIProvenance.java

@@ -0,0 +1,33 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class OAIProvenance implements Serializable {
+
+	private OriginDescription originDescription;
+
+	public OriginDescription getOriginDescription() {
+		return originDescription;
+	}
+
+	public void setOriginDescription(OriginDescription originDescription) {
+		this.originDescription = originDescription;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		OAIProvenance that = (OAIProvenance) o;
+		return Objects.equals(originDescription, that.originDescription);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(originDescription);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Oaf.java

@@ -0,0 +1,73 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+
+public abstract class Oaf implements Serializable {
+
+	/**
+	 * The list of datasource id/name pairs providing this relationship.
+	 */
+	protected List<KeyValue> collectedfrom;
+
+	private DataInfo dataInfo;
+
+	private Long lastupdatetimestamp;
+
+	public List<KeyValue> getCollectedfrom() {
+		return collectedfrom;
+	}
+
+	public void setCollectedfrom(List<KeyValue> collectedfrom) {
+		this.collectedfrom = collectedfrom;
+	}
+
+	public DataInfo getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(DataInfo dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	public Long getLastupdatetimestamp() {
+		return lastupdatetimestamp;
+	}
+
+	public void setLastupdatetimestamp(Long lastupdatetimestamp) {
+		this.lastupdatetimestamp = lastupdatetimestamp;
+	}
+
+	public void mergeOAFDataInfo(Oaf e) {
+		if (e.getDataInfo() != null && compareTrust(this, e) < 0)
+			dataInfo = e.getDataInfo();
+	}
+
+	protected String extractTrust(Oaf e) {
+		if (e == null || e.getDataInfo() == null || e.getDataInfo().getTrust() == null)
+			return "0.0";
+		return e.getDataInfo().getTrust();
+	}
+
+	protected int compareTrust(Oaf a, Oaf b) {
+		return extractTrust(a).compareTo(extractTrust(b));
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		Oaf oaf = (Oaf) o;
+		return Objects.equals(dataInfo, oaf.dataInfo)
+			&& Objects.equals(lastupdatetimestamp, oaf.lastupdatetimestamp);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(dataInfo, lastupdatetimestamp);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OafEntity.java

@@ -0,0 +1,130 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public abstract class OafEntity extends Oaf implements Serializable {
+
+	private String id;
+
+	private List<String> originalId;
+
+	private List<StructuredProperty> pid;
+
+	private String dateofcollection;
+
+	private String dateoftransformation;
+
+	private List<ExtraInfo> extraInfo;
+
+	private OAIProvenance oaiprovenance;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public List<String> getOriginalId() {
+		return originalId;
+	}
+
+	public void setOriginalId(List<String> originalId) {
+		this.originalId = originalId;
+	}
+
+	public List<StructuredProperty> getPid() {
+		return pid;
+	}
+
+	public void setPid(List<StructuredProperty> pid) {
+		this.pid = pid;
+	}
+
+	public String getDateofcollection() {
+		return dateofcollection;
+	}
+
+	public void setDateofcollection(String dateofcollection) {
+		this.dateofcollection = dateofcollection;
+	}
+
+	public String getDateoftransformation() {
+		return dateoftransformation;
+	}
+
+	public void setDateoftransformation(String dateoftransformation) {
+		this.dateoftransformation = dateoftransformation;
+	}
+
+	public List<ExtraInfo> getExtraInfo() {
+		return extraInfo;
+	}
+
+	public void setExtraInfo(List<ExtraInfo> extraInfo) {
+		this.extraInfo = extraInfo;
+	}
+
+	public OAIProvenance getOaiprovenance() {
+		return oaiprovenance;
+	}
+
+	public void setOaiprovenance(OAIProvenance oaiprovenance) {
+		this.oaiprovenance = oaiprovenance;
+	}
+
+	public void mergeFrom(OafEntity e) {
+
+		if (e == null)
+			return;
+
+		originalId = mergeLists(originalId, e.getOriginalId());
+
+		collectedfrom = mergeLists(collectedfrom, e.getCollectedfrom());
+
+		pid = mergeLists(pid, e.getPid());
+
+		if (e.getDateofcollection() != null && compareTrust(this, e) < 0)
+			dateofcollection = e.getDateofcollection();
+
+		if (e.getDateoftransformation() != null && compareTrust(this, e) < 0)
+			dateoftransformation = e.getDateoftransformation();
+
+		extraInfo = mergeLists(extraInfo, e.getExtraInfo());
+
+		if (e.getOaiprovenance() != null && compareTrust(this, e) < 0)
+			oaiprovenance = e.getOaiprovenance();
+	}
+
+	protected <T> List<T> mergeLists(final List<T>... lists) {
+
+		return Arrays
+			.stream(lists)
+			.filter(Objects::nonNull)
+			.flatMap(List::stream)
+			.filter(Objects::nonNull)
+			.distinct()
+			.collect(Collectors.toList());
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		if (!super.equals(o))
+			return false;
+		OafEntity oafEntity = (OafEntity) o;
+		return Objects.equals(id, oafEntity.id);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(super.hashCode(), id);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Organization.java

@@ -0,0 +1,214 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class Organization extends OafEntity implements Serializable {
+
+	private Field<String> legalshortname;
+
+	private Field<String> legalname;
+
+	private List<Field<String>> alternativeNames;
+
+	private Field<String> websiteurl;
+
+	private Field<String> logourl;
+
+	private Field<String> eclegalbody;
+
+	private Field<String> eclegalperson;
+
+	private Field<String> ecnonprofit;
+
+	private Field<String> ecresearchorganization;
+
+	private Field<String> echighereducation;
+
+	private Field<String> ecinternationalorganizationeurinterests;
+
+	private Field<String> ecinternationalorganization;
+
+	private Field<String> ecenterprise;
+
+	private Field<String> ecsmevalidated;
+
+	private Field<String> ecnutscode;
+
+	private Qualifier country;
+
+	public Field<String> getLegalshortname() {
+		return legalshortname;
+	}
+
+	public void setLegalshortname(Field<String> legalshortname) {
+		this.legalshortname = legalshortname;
+	}
+
+	public Field<String> getLegalname() {
+		return legalname;
+	}
+
+	public void setLegalname(Field<String> legalname) {
+		this.legalname = legalname;
+	}
+
+	public List<Field<String>> getAlternativeNames() {
+		return alternativeNames;
+	}
+
+	public void setAlternativeNames(List<Field<String>> alternativeNames) {
+		this.alternativeNames = alternativeNames;
+	}
+
+	public Field<String> getWebsiteurl() {
+		return websiteurl;
+	}
+
+	public void setWebsiteurl(Field<String> websiteurl) {
+		this.websiteurl = websiteurl;
+	}
+
+	public Field<String> getLogourl() {
+		return logourl;
+	}
+
+	public void setLogourl(Field<String> logourl) {
+		this.logourl = logourl;
+	}
+
+	public Field<String> getEclegalbody() {
+		return eclegalbody;
+	}
+
+	public void setEclegalbody(Field<String> eclegalbody) {
+		this.eclegalbody = eclegalbody;
+	}
+
+	public Field<String> getEclegalperson() {
+		return eclegalperson;
+	}
+
+	public void setEclegalperson(Field<String> eclegalperson) {
+		this.eclegalperson = eclegalperson;
+	}
+
+	public Field<String> getEcnonprofit() {
+		return ecnonprofit;
+	}
+
+	public void setEcnonprofit(Field<String> ecnonprofit) {
+		this.ecnonprofit = ecnonprofit;
+	}
+
+	public Field<String> getEcresearchorganization() {
+		return ecresearchorganization;
+	}
+
+	public void setEcresearchorganization(Field<String> ecresearchorganization) {
+		this.ecresearchorganization = ecresearchorganization;
+	}
+
+	public Field<String> getEchighereducation() {
+		return echighereducation;
+	}
+
+	public void setEchighereducation(Field<String> echighereducation) {
+		this.echighereducation = echighereducation;
+	}
+
+	public Field<String> getEcinternationalorganizationeurinterests() {
+		return ecinternationalorganizationeurinterests;
+	}
+
+	public void setEcinternationalorganizationeurinterests(
+		Field<String> ecinternationalorganizationeurinterests) {
+		this.ecinternationalorganizationeurinterests = ecinternationalorganizationeurinterests;
+	}
+
+	public Field<String> getEcinternationalorganization() {
+		return ecinternationalorganization;
+	}
+
+	public void setEcinternationalorganization(Field<String> ecinternationalorganization) {
+		this.ecinternationalorganization = ecinternationalorganization;
+	}
+
+	public Field<String> getEcenterprise() {
+		return ecenterprise;
+	}
+
+	public void setEcenterprise(Field<String> ecenterprise) {
+		this.ecenterprise = ecenterprise;
+	}
+
+	public Field<String> getEcsmevalidated() {
+		return ecsmevalidated;
+	}
+
+	public void setEcsmevalidated(Field<String> ecsmevalidated) {
+		this.ecsmevalidated = ecsmevalidated;
+	}
+
+	public Field<String> getEcnutscode() {
+		return ecnutscode;
+	}
+
+	public void setEcnutscode(Field<String> ecnutscode) {
+		this.ecnutscode = ecnutscode;
+	}
+
+	public Qualifier getCountry() {
+		return country;
+	}
+
+	public void setCountry(Qualifier country) {
+		this.country = country;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!Organization.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		final Organization o = (Organization) e;
+		legalshortname = o.getLegalshortname() != null && compareTrust(this, e) < 0
+			? o.getLegalshortname()
+			: legalshortname;
+		legalname = o.getLegalname() != null && compareTrust(this, e) < 0 ? o.getLegalname() : legalname;
+		alternativeNames = mergeLists(o.getAlternativeNames(), alternativeNames);
+		websiteurl = o.getWebsiteurl() != null && compareTrust(this, e) < 0 ? o.getWebsiteurl() : websiteurl;
+		logourl = o.getLogourl() != null && compareTrust(this, e) < 0 ? o.getLogourl() : logourl;
+		eclegalbody = o.getEclegalbody() != null && compareTrust(this, e) < 0 ? o.getEclegalbody() : eclegalbody;
+		eclegalperson = o.getEclegalperson() != null && compareTrust(this, e) < 0
+			? o.getEclegalperson()
+			: eclegalperson;
+		ecnonprofit = o.getEcnonprofit() != null && compareTrust(this, e) < 0 ? o.getEcnonprofit() : ecnonprofit;
+		ecresearchorganization = o.getEcresearchorganization() != null && compareTrust(this, e) < 0
+			? o.getEcresearchorganization()
+			: ecresearchorganization;
+		echighereducation = o.getEchighereducation() != null && compareTrust(this, e) < 0
+			? o.getEchighereducation()
+			: echighereducation;
+		ecinternationalorganizationeurinterests = o.getEcinternationalorganizationeurinterests() != null
+			&& compareTrust(this, e) < 0
+				? o.getEcinternationalorganizationeurinterests()
+				: ecinternationalorganizationeurinterests;
+		ecinternationalorganization = o.getEcinternationalorganization() != null && compareTrust(this, e) < 0
+			? o.getEcinternationalorganization()
+			: ecinternationalorganization;
+		ecenterprise = o.getEcenterprise() != null && compareTrust(this, e) < 0
+			? o.getEcenterprise()
+			: ecenterprise;
+		ecsmevalidated = o.getEcsmevalidated() != null && compareTrust(this, e) < 0
+			? o.getEcsmevalidated()
+			: ecsmevalidated;
+		ecnutscode = o.getEcnutscode() != null && compareTrust(this, e) < 0 ? o.getEcnutscode() : ecnutscode;
+		country = o.getCountry() != null && compareTrust(this, e) < 0 ? o.getCountry() : country;
+		mergeOAFDataInfo(o);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OriginDescription.java

@@ -0,0 +1,88 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class OriginDescription implements Serializable {
+
+	private String harvestDate;
+
+	private Boolean altered = true;
+
+	private String baseURL;
+
+	private String identifier;
+
+	private String datestamp;
+
+	private String metadataNamespace;
+
+	public String getHarvestDate() {
+		return harvestDate;
+	}
+
+	public void setHarvestDate(String harvestDate) {
+		this.harvestDate = harvestDate;
+	}
+
+	public Boolean getAltered() {
+		return altered;
+	}
+
+	public void setAltered(Boolean altered) {
+		this.altered = altered;
+	}
+
+	public String getBaseURL() {
+		return baseURL;
+	}
+
+	public void setBaseURL(String baseURL) {
+		this.baseURL = baseURL;
+	}
+
+	public String getIdentifier() {
+		return identifier;
+	}
+
+	public void setIdentifier(String identifier) {
+		this.identifier = identifier;
+	}
+
+	public String getDatestamp() {
+		return datestamp;
+	}
+
+	public void setDatestamp(String datestamp) {
+		this.datestamp = datestamp;
+	}
+
+	public String getMetadataNamespace() {
+		return metadataNamespace;
+	}
+
+	public void setMetadataNamespace(String metadataNamespace) {
+		this.metadataNamespace = metadataNamespace;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		OriginDescription that = (OriginDescription) o;
+		return Objects.equals(harvestDate, that.harvestDate)
+			&& Objects.equals(altered, that.altered)
+			&& Objects.equals(baseURL, that.baseURL)
+			&& Objects.equals(identifier, that.identifier)
+			&& Objects.equals(datestamp, that.datestamp)
+			&& Objects.equals(metadataNamespace, that.metadataNamespace);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(harvestDate, altered, baseURL, identifier, datestamp, metadataNamespace);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/OtherResearchProduct.java

@@ -0,0 +1,60 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+
+public class OtherResearchProduct extends Result implements Serializable {
+
+	private List<Field<String>> contactperson;
+
+	private List<Field<String>> contactgroup;
+
+	private List<Field<String>> tool;
+
+	public OtherResearchProduct() {
+		setResulttype(ModelConstants.ORP_DEFAULT_RESULTTYPE);
+	}
+
+	public List<Field<String>> getContactperson() {
+		return contactperson;
+	}
+
+	public void setContactperson(List<Field<String>> contactperson) {
+		this.contactperson = contactperson;
+	}
+
+	public List<Field<String>> getContactgroup() {
+		return contactgroup;
+	}
+
+	public void setContactgroup(List<Field<String>> contactgroup) {
+		this.contactgroup = contactgroup;
+	}
+
+	public List<Field<String>> getTool() {
+		return tool;
+	}
+
+	public void setTool(List<Field<String>> tool) {
+		this.tool = tool;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!OtherResearchProduct.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		OtherResearchProduct o = (OtherResearchProduct) e;
+
+		contactperson = mergeLists(contactperson, o.getContactperson());
+		contactgroup = mergeLists(contactgroup, o.getContactgroup());
+		tool = mergeLists(tool, o.getTool());
+		mergeOAFDataInfo(e);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Programme.java

@@ -0,0 +1,38 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+public class Programme implements Serializable {
+	private String code;
+	private String description;
+
+	public String getCode() {
+		return code;
+	}
+
+	public void setCode(String code) {
+		this.code = code;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(String description) {
+		this.description = description;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+
+		Programme programme = (Programme) o;
+		return Objects.equals(code, programme.code);
+	}
+
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Project.java

@@ -0,0 +1,338 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class Project extends OafEntity implements Serializable {
+
+	private Field<String> websiteurl;
+
+	private Field<String> code;
+
+	private Field<String> acronym;
+
+	private Field<String> title;
+
+	private Field<String> startdate;
+
+	private Field<String> enddate;
+
+	private Field<String> callidentifier;
+
+	private Field<String> keywords;
+
+	private Field<String> duration;
+
+	private Field<String> ecsc39;
+
+	private Field<String> oamandatepublications;
+
+	private Field<String> ecarticle29_3;
+
+	private List<StructuredProperty> subjects;
+
+	private List<Field<String>> fundingtree;
+
+	private Qualifier contracttype;
+
+	private Field<String> optional1;
+
+	private Field<String> optional2;
+
+	private Field<String> jsonextrainfo;
+
+	private Field<String> contactfullname;
+
+	private Field<String> contactfax;
+
+	private Field<String> contactphone;
+
+	private Field<String> contactemail;
+
+	private Field<String> summary;
+
+	private Field<String> currency;
+
+	private Float totalcost;
+
+	private Float fundedamount;
+
+	private List<Programme> programme;
+
+	public Field<String> getWebsiteurl() {
+		return websiteurl;
+	}
+
+	public void setWebsiteurl(Field<String> websiteurl) {
+		this.websiteurl = websiteurl;
+	}
+
+	public Field<String> getCode() {
+		return code;
+	}
+
+	public void setCode(Field<String> code) {
+		this.code = code;
+	}
+
+	public Field<String> getAcronym() {
+		return acronym;
+	}
+
+	public void setAcronym(Field<String> acronym) {
+		this.acronym = acronym;
+	}
+
+	public Field<String> getTitle() {
+		return title;
+	}
+
+	public void setTitle(Field<String> title) {
+		this.title = title;
+	}
+
+	public Field<String> getStartdate() {
+		return startdate;
+	}
+
+	public void setStartdate(Field<String> startdate) {
+		this.startdate = startdate;
+	}
+
+	public Field<String> getEnddate() {
+		return enddate;
+	}
+
+	public void setEnddate(Field<String> enddate) {
+		this.enddate = enddate;
+	}
+
+	public Field<String> getCallidentifier() {
+		return callidentifier;
+	}
+
+	public void setCallidentifier(Field<String> callidentifier) {
+		this.callidentifier = callidentifier;
+	}
+
+	public Field<String> getKeywords() {
+		return keywords;
+	}
+
+	public void setKeywords(Field<String> keywords) {
+		this.keywords = keywords;
+	}
+
+	public Field<String> getDuration() {
+		return duration;
+	}
+
+	public void setDuration(Field<String> duration) {
+		this.duration = duration;
+	}
+
+	public Field<String> getEcsc39() {
+		return ecsc39;
+	}
+
+	public void setEcsc39(Field<String> ecsc39) {
+		this.ecsc39 = ecsc39;
+	}
+
+	public Field<String> getOamandatepublications() {
+		return oamandatepublications;
+	}
+
+	public void setOamandatepublications(Field<String> oamandatepublications) {
+		this.oamandatepublications = oamandatepublications;
+	}
+
+	public Field<String> getEcarticle29_3() {
+		return ecarticle29_3;
+	}
+
+	public void setEcarticle29_3(Field<String> ecarticle29_3) {
+		this.ecarticle29_3 = ecarticle29_3;
+	}
+
+	public List<StructuredProperty> getSubjects() {
+		return subjects;
+	}
+
+	public void setSubjects(List<StructuredProperty> subjects) {
+		this.subjects = subjects;
+	}
+
+	public List<Field<String>> getFundingtree() {
+		return fundingtree;
+	}
+
+	public void setFundingtree(List<Field<String>> fundingtree) {
+		this.fundingtree = fundingtree;
+	}
+
+	public Qualifier getContracttype() {
+		return contracttype;
+	}
+
+	public void setContracttype(Qualifier contracttype) {
+		this.contracttype = contracttype;
+	}
+
+	public Field<String> getOptional1() {
+		return optional1;
+	}
+
+	public void setOptional1(Field<String> optional1) {
+		this.optional1 = optional1;
+	}
+
+	public Field<String> getOptional2() {
+		return optional2;
+	}
+
+	public void setOptional2(Field<String> optional2) {
+		this.optional2 = optional2;
+	}
+
+	public Field<String> getJsonextrainfo() {
+		return jsonextrainfo;
+	}
+
+	public void setJsonextrainfo(Field<String> jsonextrainfo) {
+		this.jsonextrainfo = jsonextrainfo;
+	}
+
+	public Field<String> getContactfullname() {
+		return contactfullname;
+	}
+
+	public void setContactfullname(Field<String> contactfullname) {
+		this.contactfullname = contactfullname;
+	}
+
+	public Field<String> getContactfax() {
+		return contactfax;
+	}
+
+	public void setContactfax(Field<String> contactfax) {
+		this.contactfax = contactfax;
+	}
+
+	public Field<String> getContactphone() {
+		return contactphone;
+	}
+
+	public void setContactphone(Field<String> contactphone) {
+		this.contactphone = contactphone;
+	}
+
+	public Field<String> getContactemail() {
+		return contactemail;
+	}
+
+	public void setContactemail(Field<String> contactemail) {
+		this.contactemail = contactemail;
+	}
+
+	public Field<String> getSummary() {
+		return summary;
+	}
+
+	public void setSummary(Field<String> summary) {
+		this.summary = summary;
+	}
+
+	public Field<String> getCurrency() {
+		return currency;
+	}
+
+	public void setCurrency(Field<String> currency) {
+		this.currency = currency;
+	}
+
+	public Float getTotalcost() {
+		return totalcost;
+	}
+
+	public void setTotalcost(Float totalcost) {
+		this.totalcost = totalcost;
+	}
+
+	public Float getFundedamount() {
+		return fundedamount;
+	}
+
+	public void setFundedamount(Float fundedamount) {
+		this.fundedamount = fundedamount;
+	}
+
+	public List<Programme> getProgramme() {
+		return programme;
+	}
+
+	public void setProgramme(List<Programme> programme) {
+		this.programme = programme;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!Project.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		Project p = (Project) e;
+
+		websiteurl = p.getWebsiteurl() != null && compareTrust(this, e) < 0 ? p.getWebsiteurl() : websiteurl;
+		code = p.getCode() != null && compareTrust(this, e) < 0 ? p.getCode() : code;
+		acronym = p.getAcronym() != null && compareTrust(this, e) < 0 ? p.getAcronym() : acronym;
+		title = p.getTitle() != null && compareTrust(this, e) < 0 ? p.getTitle() : title;
+		startdate = p.getStartdate() != null && compareTrust(this, e) < 0 ? p.getStartdate() : startdate;
+		enddate = p.getEnddate() != null && compareTrust(this, e) < 0 ? p.getEnddate() : enddate;
+		callidentifier = p.getCallidentifier() != null && compareTrust(this, e) < 0
+			? p.getCallidentifier()
+			: callidentifier;
+		keywords = p.getKeywords() != null && compareTrust(this, e) < 0 ? p.getKeywords() : keywords;
+		duration = p.getDuration() != null && compareTrust(this, e) < 0 ? p.getDuration() : duration;
+		ecsc39 = p.getEcsc39() != null && compareTrust(this, e) < 0 ? p.getEcsc39() : ecsc39;
+		oamandatepublications = p.getOamandatepublications() != null && compareTrust(this, e) < 0
+			? p.getOamandatepublications()
+			: oamandatepublications;
+		ecarticle29_3 = p.getEcarticle29_3() != null && compareTrust(this, e) < 0
+			? p.getEcarticle29_3()
+			: ecarticle29_3;
+		subjects = mergeLists(subjects, p.getSubjects());
+		fundingtree = mergeLists(fundingtree, p.getFundingtree());
+		contracttype = p.getContracttype() != null && compareTrust(this, e) < 0
+			? p.getContracttype()
+			: contracttype;
+		optional1 = p.getOptional1() != null && compareTrust(this, e) < 0 ? p.getOptional1() : optional1;
+		optional2 = p.getOptional2() != null && compareTrust(this, e) < 0 ? p.getOptional2() : optional2;
+		jsonextrainfo = p.getJsonextrainfo() != null && compareTrust(this, e) < 0
+			? p.getJsonextrainfo()
+			: jsonextrainfo;
+		contactfullname = p.getContactfullname() != null && compareTrust(this, e) < 0
+			? p.getContactfullname()
+			: contactfullname;
+		contactfax = p.getContactfax() != null && compareTrust(this, e) < 0 ? p.getContactfax() : contactfax;
+		contactphone = p.getContactphone() != null && compareTrust(this, e) < 0
+			? p.getContactphone()
+			: contactphone;
+		contactemail = p.getContactemail() != null && compareTrust(this, e) < 0
+			? p.getContactemail()
+			: contactemail;
+		summary = p.getSummary() != null && compareTrust(this, e) < 0 ? p.getSummary() : summary;
+		currency = p.getCurrency() != null && compareTrust(this, e) < 0 ? p.getCurrency() : currency;
+		totalcost = p.getTotalcost() != null && compareTrust(this, e) < 0 ? p.getTotalcost() : totalcost;
+		fundedamount = p.getFundedamount() != null && compareTrust(this, e) < 0
+			? p.getFundedamount()
+			: fundedamount;
+
+		programme = mergeLists(programme, p.getProgramme());
+
+		mergeOAFDataInfo(e);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Publication.java

@@ -0,0 +1,39 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+
+public class Publication extends Result implements Serializable {
+
+	// publication specific
+	private Journal journal;
+
+	public Publication() {
+		setResulttype(ModelConstants.PUBLICATION_DEFAULT_RESULTTYPE);
+	}
+
+	public Journal getJournal() {
+		return journal;
+	}
+
+	public void setJournal(Journal journal) {
+		this.journal = journal;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!Publication.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		Publication p = (Publication) e;
+
+		if (p.getJournal() != null && compareTrust(this, e) < 0)
+			journal = p.getJournal();
+		mergeOAFDataInfo(e);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Qualifier.java

@@ -0,0 +1,87 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public class Qualifier implements Serializable {
+
+	private String classid;
+	private String classname;
+	private String schemeid;
+	private String schemename;
+
+	public String getClassid() {
+		return classid;
+	}
+
+	public void setClassid(String classid) {
+		this.classid = classid;
+	}
+
+	public String getClassname() {
+		return classname;
+	}
+
+	public void setClassname(String classname) {
+		this.classname = classname;
+	}
+
+	public String getSchemeid() {
+		return schemeid;
+	}
+
+	public void setSchemeid(String schemeid) {
+		this.schemeid = schemeid;
+	}
+
+	public String getSchemename() {
+		return schemename;
+	}
+
+	public void setSchemename(String schemename) {
+		this.schemename = schemename;
+	}
+
+	public String toComparableString() {
+		return isBlank()
+			? ""
+			: String
+				.format(
+					"%s::%s::%s::%s",
+					classid != null ? classid : "",
+					classname != null ? classname : "",
+					schemeid != null ? schemeid : "",
+					schemename != null ? schemename : "");
+	}
+
+	@JsonIgnore
+	public boolean isBlank() {
+		return StringUtils.isBlank(classid)
+			&& StringUtils.isBlank(classname)
+			&& StringUtils.isBlank(schemeid)
+			&& StringUtils.isBlank(schemename);
+	}
+
+	@Override
+	public int hashCode() {
+		return toComparableString().hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+
+		Qualifier other = (Qualifier) obj;
+
+		return toComparableString().equals(other.toComparableString());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Relation.java

@@ -0,0 +1,167 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Relation models any edge between two nodes in the OpenAIRE graph. It has a source id and a target id pointing to
+ * graph node identifiers and it is further characterised by the semantic of the link through the fields relType,
+ * subRelType and relClass. Provenance information is modeled according to the dataInfo element and collectedFrom, while
+ * individual relationship types can provide extra information via the properties field.
+ */
+public class Relation extends Oaf {
+
+	/**
+	 * Main relationship classifier, values include 'resultResult', 'resultProject', 'resultOrganization', etc.
+	 */
+	private String relType;
+
+	/**
+	 * Further classifies a relationship, values include 'affiliation', 'similarity', 'supplement', etc.
+	 */
+	private String subRelType;
+
+	/**
+	 * Indicates the direction of the relationship, values include 'isSupplementTo', 'isSupplementedBy', 'merges,
+	 * 'isMergedIn'.
+	 */
+	private String relClass;
+
+	/**
+	 * The source entity id.
+	 */
+	private String source;
+
+	/**
+	 * The target entity id.
+	 */
+	private String target;
+
+	/**
+	 * Was this relationship authoritatively validated?
+	 */
+	private Boolean validated;
+
+	/**
+	 * When was this relationship authoritatively validated.
+	 */
+	private String validationDate;
+
+	/**
+	 * List of relation specific properties. Values include 'similarityLevel', indicating the similarity score between a
+	 * pair of publications.
+	 */
+	private List<KeyValue> properties = new ArrayList<>();
+
+	public String getRelType() {
+		return relType;
+	}
+
+	public void setRelType(final String relType) {
+		this.relType = relType;
+	}
+
+	public String getSubRelType() {
+		return subRelType;
+	}
+
+	public void setSubRelType(final String subRelType) {
+		this.subRelType = subRelType;
+	}
+
+	public String getRelClass() {
+		return relClass;
+	}
+
+	public void setRelClass(final String relClass) {
+		this.relClass = relClass;
+	}
+
+	public String getSource() {
+		return source;
+	}
+
+	public void setSource(final String source) {
+		this.source = source;
+	}
+
+	public String getTarget() {
+		return target;
+	}
+
+	public void setTarget(final String target) {
+		this.target = target;
+	}
+
+	public List<KeyValue> getProperties() {
+		return properties;
+	}
+
+	public void setProperties(List<KeyValue> properties) {
+		this.properties = properties;
+	}
+
+	public Boolean getValidated() {
+		return validated;
+	}
+
+	public void setValidated(Boolean validated) {
+		this.validated = validated;
+	}
+
+	public String getValidationDate() {
+		return validationDate;
+	}
+
+	public void setValidationDate(String validationDate) {
+		this.validationDate = validationDate;
+	}
+
+	public void mergeFrom(final Relation r) {
+
+		checkArgument(Objects.equals(getSource(), r.getSource()), "source ids must be equal");
+		checkArgument(Objects.equals(getTarget(), r.getTarget()), "target ids must be equal");
+		checkArgument(Objects.equals(getRelType(), r.getRelType()), "relType(s) must be equal");
+		checkArgument(
+			Objects.equals(getSubRelType(), r.getSubRelType()), "subRelType(s) must be equal");
+		checkArgument(Objects.equals(getRelClass(), r.getRelClass()), "relClass(es) must be equal");
+
+		setCollectedfrom(
+			Stream
+				.concat(
+					Optional
+						.ofNullable(getCollectedfrom())
+						.map(Collection::stream)
+						.orElse(Stream.empty()),
+					Optional
+						.ofNullable(r.getCollectedfrom())
+						.map(Collection::stream)
+						.orElse(Stream.empty()))
+				.distinct() // relies on KeyValue.equals
+				.collect(Collectors.toList()));
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		Relation relation = (Relation) o;
+		return relType.equals(relation.relType)
+			&& subRelType.equals(relation.subRelType)
+			&& relClass.equals(relation.relClass)
+			&& source.equals(relation.source)
+			&& target.equals(relation.target);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(relType, subRelType, relClass, source, target, collectedfrom);
+	}
+
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Result.java

@@ -0,0 +1,348 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class Result extends OafEntity implements Serializable {
+
+	private List<Measure> measures;
+
+	private List<Author> author;
+
+	// resulttype allows subclassing results into publications | datasets | software
+	private Qualifier resulttype;
+
+	// common fields
+	private Qualifier language;
+
+	private List<Country> country;
+
+	private List<StructuredProperty> subject;
+
+	private List<StructuredProperty> title;
+
+	private List<StructuredProperty> relevantdate;
+
+	private List<Field<String>> description;
+
+	private Field<String> dateofacceptance;
+
+	private Field<String> publisher;
+
+	private Field<String> embargoenddate;
+
+	private List<Field<String>> source;
+
+	private List<Field<String>> fulltext; // remove candidate
+
+	private List<Field<String>> format;
+
+	private List<Field<String>> contributor;
+
+	private Qualifier resourcetype;
+
+	private List<Field<String>> coverage;
+
+	private Qualifier bestaccessright;
+
+	private List<Context> context;
+
+	private List<ExternalReference> externalReference;
+
+	private List<Instance> instance;
+
+	public List<Measure> getMeasures() {
+		return measures;
+	}
+
+	public void setMeasures(List<Measure> measures) {
+		this.measures = measures;
+	}
+
+	public List<Author> getAuthor() {
+		return author;
+	}
+
+	public void setAuthor(List<Author> author) {
+		this.author = author;
+	}
+
+	public Qualifier getResulttype() {
+		return resulttype;
+	}
+
+	public void setResulttype(Qualifier resulttype) {
+		this.resulttype = resulttype;
+	}
+
+	public Qualifier getLanguage() {
+		return language;
+	}
+
+	public void setLanguage(Qualifier language) {
+		this.language = language;
+	}
+
+	public List<Country> getCountry() {
+		return country;
+	}
+
+	public void setCountry(List<Country> country) {
+		this.country = country;
+	}
+
+	public List<StructuredProperty> getSubject() {
+		return subject;
+	}
+
+	public void setSubject(List<StructuredProperty> subject) {
+		this.subject = subject;
+	}
+
+	public List<StructuredProperty> getTitle() {
+		return title;
+	}
+
+	public void setTitle(List<StructuredProperty> title) {
+		this.title = title;
+	}
+
+	public List<StructuredProperty> getRelevantdate() {
+		return relevantdate;
+	}
+
+	public void setRelevantdate(List<StructuredProperty> relevantdate) {
+		this.relevantdate = relevantdate;
+	}
+
+	public List<Field<String>> getDescription() {
+		return description;
+	}
+
+	public void setDescription(List<Field<String>> description) {
+		this.description = description;
+	}
+
+	public Field<String> getDateofacceptance() {
+		return dateofacceptance;
+	}
+
+	public void setDateofacceptance(Field<String> dateofacceptance) {
+		this.dateofacceptance = dateofacceptance;
+	}
+
+	public Field<String> getPublisher() {
+		return publisher;
+	}
+
+	public void setPublisher(Field<String> publisher) {
+		this.publisher = publisher;
+	}
+
+	public Field<String> getEmbargoenddate() {
+		return embargoenddate;
+	}
+
+	public void setEmbargoenddate(Field<String> embargoenddate) {
+		this.embargoenddate = embargoenddate;
+	}
+
+	public List<Field<String>> getSource() {
+		return source;
+	}
+
+	public void setSource(List<Field<String>> source) {
+		this.source = source;
+	}
+
+	public List<Field<String>> getFulltext() {
+		return fulltext;
+	}
+
+	public void setFulltext(List<Field<String>> fulltext) {
+		this.fulltext = fulltext;
+	}
+
+	public List<Field<String>> getFormat() {
+		return format;
+	}
+
+	public void setFormat(List<Field<String>> format) {
+		this.format = format;
+	}
+
+	public List<Field<String>> getContributor() {
+		return contributor;
+	}
+
+	public void setContributor(List<Field<String>> contributor) {
+		this.contributor = contributor;
+	}
+
+	public Qualifier getResourcetype() {
+		return resourcetype;
+	}
+
+	public void setResourcetype(Qualifier resourcetype) {
+		this.resourcetype = resourcetype;
+	}
+
+	public List<Field<String>> getCoverage() {
+		return coverage;
+	}
+
+	public void setCoverage(List<Field<String>> coverage) {
+		this.coverage = coverage;
+	}
+
+	public Qualifier getBestaccessright() {
+		return bestaccessright;
+	}
+
+	public void setBestaccessright(Qualifier bestaccessright) {
+		this.bestaccessright = bestaccessright;
+	}
+
+	public List<Context> getContext() {
+		return context;
+	}
+
+	public void setContext(List<Context> context) {
+		this.context = context;
+	}
+
+	public List<ExternalReference> getExternalReference() {
+		return externalReference;
+	}
+
+	public void setExternalReference(List<ExternalReference> externalReference) {
+		this.externalReference = externalReference;
+	}
+
+	public List<Instance> getInstance() {
+		return instance;
+	}
+
+	public void setInstance(List<Instance> instance) {
+		this.instance = instance;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!Result.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		Result r = (Result) e;
+
+		// TODO consider merging also Measures
+
+		instance = mergeLists(instance, r.getInstance());
+
+		if (r.getBestaccessright() != null && compareTrust(this, r) < 0)
+			bestaccessright = r.getBestaccessright();
+
+		if (r.getResulttype() != null && compareTrust(this, r) < 0)
+			resulttype = r.getResulttype();
+
+		if (r.getLanguage() != null && compareTrust(this, r) < 0)
+			language = r.getLanguage();
+
+		country = mergeLists(country, r.getCountry());
+
+		subject = mergeLists(subject, r.getSubject());
+
+		// merge title lists: main title with higher trust and distinct between the others
+		StructuredProperty baseMainTitle = null;
+		if (title != null) {
+			baseMainTitle = getMainTitle(title);
+			if (baseMainTitle != null) {
+				final StructuredProperty p = baseMainTitle;
+				title = title.stream().filter(t -> t != p).collect(Collectors.toList());
+			}
+		}
+
+		StructuredProperty newMainTitle = null;
+		if (r.getTitle() != null) {
+			newMainTitle = getMainTitle(r.getTitle());
+			if (newMainTitle != null) {
+				final StructuredProperty p = newMainTitle;
+				r.setTitle(r.getTitle().stream().filter(t -> t != p).collect(Collectors.toList()));
+			}
+		}
+
+		if (newMainTitle != null && compareTrust(this, r) < 0) {
+			baseMainTitle = newMainTitle;
+		}
+
+		title = mergeLists(title, r.getTitle());
+		if (title != null && baseMainTitle != null) {
+			title.add(baseMainTitle);
+		}
+
+		relevantdate = mergeLists(relevantdate, r.getRelevantdate());
+
+		description = longestLists(description, r.getDescription());
+
+		if (r.getPublisher() != null && compareTrust(this, r) < 0)
+			publisher = r.getPublisher();
+
+		if (r.getEmbargoenddate() != null && compareTrust(this, r) < 0)
+			embargoenddate = r.getEmbargoenddate();
+
+		source = mergeLists(source, r.getSource());
+
+		fulltext = mergeLists(fulltext, r.getFulltext());
+
+		format = mergeLists(format, r.getFormat());
+
+		contributor = mergeLists(contributor, r.getContributor());
+
+		if (r.getResourcetype() != null)
+			resourcetype = r.getResourcetype();
+
+		coverage = mergeLists(coverage, r.getCoverage());
+
+		context = mergeLists(context, r.getContext());
+
+		externalReference = mergeLists(externalReference, r.getExternalReference());
+	}
+
+	private List<Field<String>> longestLists(List<Field<String>> a, List<Field<String>> b) {
+		if (a == null || b == null)
+			return a == null ? b : a;
+		if (a.size() == b.size()) {
+			int msa = a
+				.stream()
+				.filter(i -> i.getValue() != null)
+				.map(i -> i.getValue().length())
+				.max(Comparator.naturalOrder())
+				.orElse(0);
+			int msb = b
+				.stream()
+				.filter(i -> i.getValue() != null)
+				.map(i -> i.getValue().length())
+				.max(Comparator.naturalOrder())
+				.orElse(0);
+			return msa > msb ? a : b;
+		}
+		return a.size() > b.size() ? a : b;
+	}
+
+	private StructuredProperty getMainTitle(List<StructuredProperty> titles) {
+		// need to check if the list of titles contains more than 1 main title? (in that case, we should chose which
+		// main title select in the list)
+		for (StructuredProperty title : titles) {
+			if (title.getQualifier() != null && title.getQualifier().getClassid() != null)
+				if (title.getQualifier().getClassid().equals("main title"))
+					return title;
+		}
+		return null;
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/Software.java

@@ -0,0 +1,80 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+import java.util.List;
+
+import eu.dnetlib.dhp.schema.common.ModelConstants;
+
+public class Software extends Result implements Serializable {
+
+	private List<Field<String>> documentationUrl;
+
+	// candidate for removal
+	private List<StructuredProperty> license;
+
+	// candidate for removal
+	private Field<String> codeRepositoryUrl;
+
+	private Qualifier programmingLanguage;
+
+	public Software() {
+		setResulttype(ModelConstants.SOFTWARE_DEFAULT_RESULTTYPE);
+	}
+
+	public List<Field<String>> getDocumentationUrl() {
+		return documentationUrl;
+	}
+
+	public void setDocumentationUrl(List<Field<String>> documentationUrl) {
+		this.documentationUrl = documentationUrl;
+	}
+
+	public List<StructuredProperty> getLicense() {
+		return license;
+	}
+
+	public void setLicense(List<StructuredProperty> license) {
+		this.license = license;
+	}
+
+	public Field<String> getCodeRepositoryUrl() {
+		return codeRepositoryUrl;
+	}
+
+	public void setCodeRepositoryUrl(Field<String> codeRepositoryUrl) {
+		this.codeRepositoryUrl = codeRepositoryUrl;
+	}
+
+	public Qualifier getProgrammingLanguage() {
+		return programmingLanguage;
+	}
+
+	public void setProgrammingLanguage(Qualifier programmingLanguage) {
+		this.programmingLanguage = programmingLanguage;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+
+		if (!Software.class.isAssignableFrom(e.getClass())) {
+			return;
+		}
+
+		final Software s = (Software) e;
+		documentationUrl = mergeLists(documentationUrl, s.getDocumentationUrl());
+
+		license = mergeLists(license, s.getLicense());
+
+		codeRepositoryUrl = s.getCodeRepositoryUrl() != null && compareTrust(this, s) < 0
+			? s.getCodeRepositoryUrl()
+			: codeRepositoryUrl;
+
+		programmingLanguage = s.getProgrammingLanguage() != null && compareTrust(this, s) < 0
+			? s.getProgrammingLanguage()
+			: programmingLanguage;
+
+		mergeOAFDataInfo(e);
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/oaf/StructuredProperty.java

@@ -0,0 +1,60 @@
+
+package eu.dnetlib.dhp.schema.oaf;
+
+import java.io.Serializable;
+
+public class StructuredProperty implements Serializable {
+
+	private String value;
+
+	private Qualifier qualifier;
+
+	private DataInfo dataInfo;
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(String value) {
+		this.value = value;
+	}
+
+	public Qualifier getQualifier() {
+		return qualifier;
+	}
+
+	public void setQualifier(Qualifier qualifier) {
+		this.qualifier = qualifier;
+	}
+
+	public DataInfo getDataInfo() {
+		return dataInfo;
+	}
+
+	public void setDataInfo(DataInfo dataInfo) {
+		this.dataInfo = dataInfo;
+	}
+
+	public String toComparableString() {
+		return value != null ? value.toLowerCase() : "";
+	}
+
+	@Override
+	public int hashCode() {
+		return toComparableString().hashCode();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+
+		StructuredProperty other = (StructuredProperty) obj;
+
+		return toComparableString().equals(other.toComparableString());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIDataset.java

@@ -0,0 +1,89 @@
+
+package eu.dnetlib.dhp.schema.scholexplorer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
+import eu.dnetlib.dhp.schema.oaf.Dataset;
+import eu.dnetlib.dhp.schema.oaf.OafEntity;
+
+public class DLIDataset extends Dataset {
+
+	private String originalObjIdentifier;
+
+	private List<ProvenaceInfo> dlicollectedfrom;
+
+	private String completionStatus;
+
+	public String getCompletionStatus() {
+		return completionStatus;
+	}
+
+	public void setCompletionStatus(String completionStatus) {
+		this.completionStatus = completionStatus;
+	}
+
+	public List<ProvenaceInfo> getDlicollectedfrom() {
+		return dlicollectedfrom;
+	}
+
+	public void setDlicollectedfrom(List<ProvenaceInfo> dlicollectedfrom) {
+		this.dlicollectedfrom = dlicollectedfrom;
+	}
+
+	public String getOriginalObjIdentifier() {
+		return originalObjIdentifier;
+	}
+
+	public void setOriginalObjIdentifier(String originalObjIdentifier) {
+		this.originalObjIdentifier = originalObjIdentifier;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+		DLIDataset p = (DLIDataset) e;
+		if (StringUtils.isBlank(completionStatus) && StringUtils.isNotBlank(p.completionStatus))
+			completionStatus = p.completionStatus;
+		if ("complete".equalsIgnoreCase(p.completionStatus))
+			completionStatus = "complete";
+		dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
+	}
+
+	private List<ProvenaceInfo> mergeProvenance(
+		final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
+		Map<String, ProvenaceInfo> result = new HashMap<>();
+		if (a != null)
+			a
+				.forEach(
+					p -> {
+						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
+							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
+								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
+								result.put(p.getId(), p);
+							}
+
+						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
+							result.put(p.getId(), p);
+					});
+		if (b != null)
+			b
+				.forEach(
+					p -> {
+						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
+							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
+								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
+								result.put(p.getId(), p);
+							}
+
+						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
+							result.put(p.getId(), p);
+					});
+
+		return new ArrayList<>(result.values());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIPublication.java

@@ -0,0 +1,87 @@
+
+package eu.dnetlib.dhp.schema.scholexplorer;
+
+import java.io.Serializable;
+import java.util.*;
+
+import org.apache.commons.lang3.StringUtils;
+
+import eu.dnetlib.dhp.schema.oaf.OafEntity;
+import eu.dnetlib.dhp.schema.oaf.Publication;
+
+public class DLIPublication extends Publication implements Serializable {
+
+	private String originalObjIdentifier;
+
+	private List<ProvenaceInfo> dlicollectedfrom;
+
+	private String completionStatus;
+
+	public String getCompletionStatus() {
+		return completionStatus;
+	}
+
+	public void setCompletionStatus(String completionStatus) {
+		this.completionStatus = completionStatus;
+	}
+
+	public List<ProvenaceInfo> getDlicollectedfrom() {
+		return dlicollectedfrom;
+	}
+
+	public void setDlicollectedfrom(List<ProvenaceInfo> dlicollectedfrom) {
+		this.dlicollectedfrom = dlicollectedfrom;
+	}
+
+	public String getOriginalObjIdentifier() {
+		return originalObjIdentifier;
+	}
+
+	public void setOriginalObjIdentifier(String originalObjIdentifier) {
+		this.originalObjIdentifier = originalObjIdentifier;
+	}
+
+	@Override
+	public void mergeFrom(OafEntity e) {
+		super.mergeFrom(e);
+		DLIPublication p = (DLIPublication) e;
+		if (StringUtils.isBlank(completionStatus) && StringUtils.isNotBlank(p.completionStatus))
+			completionStatus = p.completionStatus;
+		if ("complete".equalsIgnoreCase(p.completionStatus))
+			completionStatus = "complete";
+		dlicollectedfrom = mergeProvenance(dlicollectedfrom, p.getDlicollectedfrom());
+	}
+
+	private List<ProvenaceInfo> mergeProvenance(
+		final List<ProvenaceInfo> a, final List<ProvenaceInfo> b) {
+		Map<String, ProvenaceInfo> result = new HashMap<>();
+		if (a != null)
+			a
+				.forEach(
+					p -> {
+						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
+							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
+								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
+								result.put(p.getId(), p);
+							}
+
+						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
+							result.put(p.getId(), p);
+					});
+		if (b != null)
+			b
+				.forEach(
+					p -> {
+						if (p != null && StringUtils.isNotBlank(p.getId()) && result.containsKey(p.getId())) {
+							if ("incomplete".equalsIgnoreCase(result.get(p.getId()).getCompletionStatus())
+								&& StringUtils.isNotBlank(p.getCompletionStatus())) {
+								result.put(p.getId(), p);
+							}
+
+						} else if (p != null && p.getId() != null && !result.containsKey(p.getId()))
+							result.put(p.getId(), p);
+					});
+
+		return new ArrayList<>(result.values());
+	}
+}

dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/scholexplorer/DLIRelation.java

@@ -0,0 +1,30 @@
+
+package eu.dnetlib.dhp.schema.scholexplorer;
+
+import java.util.List;
+
+import eu.dnetlib.dhp.schema.oaf.KeyValue;
+import eu.dnetlib.dhp.schema.oaf.Relation;
+
+public class DLIRelation extends Relation {
+
+	private String dateOfCollection;
+
+	private List<KeyValue> collectedFrom;
+
+	public List<KeyValue> getCollectedFrom() {
+		return collectedFrom;
+	}
+
+	public void setCollectedFrom(List<KeyValue> collectedFrom) {
+		this.collectedFrom = collectedFrom;
+	}
+
+	public String getDateOfCollection() {
+		return dateOfCollection;
+	}
+
+	public void setDateOfCollection(String dateOfCollection) {
+		this.dateOfCollection = dateOfCollection;
+	}
+}