moved some mdstore componenets under dnet-data-services

apps/dnet-is-application/pom.xml

@@ -19,6 +19,12 @@
 			<artifactId>dnet-is-common</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+		
+		<dependency>
+			<groupId>eu.dnetlib.dhp</groupId>
+			<artifactId>dnet-data-services</artifactId>
+			<version>${project.version}</version>
+		</dependency>
 
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
@@ -29,28 +35,6 @@
 			<groupId>jaxen</groupId>
 			<artifactId>jaxen</artifactId>
 		</dependency>
-		
-		<!-- Hadoop -->
-		<dependency>
-			<groupId>org.apache.hadoop</groupId>
-			<artifactId>hadoop-client</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>com.twitter</groupId>
-			<artifactId>parquet-hadoop</artifactId>
-			<version>1.5.0-cdh5.13.3</version>
-		</dependency>
-		<dependency>
-			<groupId>com.twitter</groupId>
-			<artifactId>parquet-avro</artifactId>
-			<version>1.5.0-cdh5.13.3</version>
-		</dependency>
-		
-		<!-- Hadoop D-Net Schemas -->
-		<dependency>
-			<groupId>eu.dnetlib.dhp</groupId>
-            <artifactId>dhp-schemas</artifactId>
-		</dependency>
 
 		<!-- hot swapping, disable cache for template, enable live reload -->
 		<dependency>

apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java

@@ -38,7 +38,7 @@ public class MainDBConfig {
 		@Qualifier("mainDataSource") final DataSource ds) {
 		return builder
 			.dataSource(ds)
-			.packages("eu.dnetlib.is.model", "eu.dnetlib.msro.model", "eu.dnetlib.dhp.schema.mdstore")
+			.packages("eu.dnetlib.is.model", "eu.dnetlib.msro.model", "eu.dnetlib.data.mdstore.model")
 			.persistenceUnit("is")
 			.build();
 	}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java

@@ -8,15 +8,16 @@ import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
 import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
+import eu.dnetlib.data.mdstore.hadoop.HadoopMDStoreService;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.data.mdstore.model.MDStoreVersion;
+import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
 import eu.dnetlib.errors.MDStoreManagerException;
 import io.swagger.v3.oas.annotations.Operation;
 
 public class AbstractMDStoreController extends AbstractDnetController {
 
 	@Autowired
-	protected MDStoreService service;
+	protected HadoopMDStoreService service;
 
 	@Operation(summary = "Return all the mdstores")
 	@GetMapping("/")

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java

@@ -10,7 +10,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
-import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
+import eu.dnetlib.data.mdstore.model.MDStoreVersion;
 import eu.dnetlib.errors.MDStoreManagerException;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.tags.Tag;
@@ -44,7 +44,8 @@ public class MDStoreApiController extends AbstractMDStoreController {
 	@Operation(summary = "Delete expired versions")
 	@DeleteMapping("/versions/expired")
 	public StatusResponse deleteExpiredVersions() {
-		return service.deleteExpiredVersions();
+		new Thread(service::deleteExpiredVersions).start();
+		return StatusResponse.DELETING;
 	}
 
 	@Operation(summary = "list the file inside the path of a mdstore version")

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinController.java

@@ -6,7 +6,7 @@ import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 
 import eu.dnetlib.data.mdstore.hadoop.ZeppelinClient;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
 import eu.dnetlib.errors.MDStoreManagerException;
 
 @Controller

apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java

@@ -17,10 +17,10 @@ import eu.dnetlib.is.model.resource.SimpleResource;
 import eu.dnetlib.is.model.vocabulary.Synonym;
 import eu.dnetlib.is.model.vocabulary.Vocabulary;
 import eu.dnetlib.is.model.vocabulary.VocabularyTerm;
+import eu.dnetlib.is.resource.ResourceValidator;
 import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
 import eu.dnetlib.is.vocabulary.repository.VocabularyRepository;
 import eu.dnetlib.is.vocabulary.repository.VocabularyTermRepository;
-import eu.dnetlib.utils.ResourceValidator;
 import eu.dnetlib.utils.XmlIndenter;
 
 @Service

apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceValidator.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.utils;
+package eu.dnetlib.is.resource;
 
 import java.io.IOException;
 import java.io.InputStream;

apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java

@@ -17,7 +17,6 @@ import org.springframework.web.bind.annotation.PathVariable;
 import eu.dnetlib.errors.InformationServiceException;
 import eu.dnetlib.is.model.resource.SimpleResource;
 import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
-import eu.dnetlib.utils.ResourceValidator;
 
 @Service
 public class SimpleResourceService {

libs/dnet-data-services/pom.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<parent>
+		<groupId>eu.dnetlib.dhp</groupId>
+		<artifactId>libs</artifactId>
+		<version>3.3.3-SNAPSHOT</version>
+		<relativePath>../</relativePath>
+	</parent>
+
+	<modelVersion>4.0.0</modelVersion>
+	<artifactId>dnet-data-services</artifactId>
+	<packaging>jar</packaging>
+
+	<dependencies>
+		<dependency>
+			<groupId>eu.dnetlib.dhp</groupId>
+			<artifactId>dnet-is-common</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>commons-codec</groupId>
+			<artifactId>commons-codec</artifactId>
+		</dependency>
+
+		<!-- Hadoop -->
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-client</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.twitter</groupId>
+			<artifactId>parquet-hadoop</artifactId>
+			<version>1.5.0-cdh5.13.3</version>
+		</dependency>
+		<dependency>
+			<groupId>com.twitter</groupId>
+			<artifactId>parquet-avro</artifactId>
+			<version>1.5.0-cdh5.13.3</version>
+		</dependency>
+		
+		<!-- Tests -->
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.mockito</groupId>
+			<artifactId>mockito-core</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.mockito</groupId>
+			<artifactId>mockito-junit-jupiter</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-test</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+</project>

libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java

@@ -2,11 +2,8 @@ package eu.dnetlib.data.mdstore;
 
 import java.util.Comparator;
 import java.util.Date;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -16,25 +13,19 @@ import javax.transaction.Transactional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.stereotype.Service;
 
-import com.google.common.collect.Sets;
+import eu.dnetlib.data.mdstore.model.MDStore;
-
+import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
-import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
+import eu.dnetlib.data.mdstore.model.MDStoreVersion;
+import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
 import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository;
 import eu.dnetlib.data.mdstore.repository.MDStoreRepository;
 import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository;
 import eu.dnetlib.data.mdstore.repository.MDStoreWithInfoRepository;
-import eu.dnetlib.dhp.schema.mdstore.MDStore;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
 import eu.dnetlib.errors.MDStoreManagerException;
 
-@Service
+public abstract class MDStoreService {
-public class MDStoreService {
 
 	@Autowired
 	private MDStoreRepository mdstoreRepository;
@@ -45,12 +36,7 @@ public class MDStoreService {
 	@Autowired
 	private MDStoreWithInfoRepository mdstoreWithInfoRepository;
 	@Autowired
-	private JdbcTemplate jdbcTemplate;
+	protected JdbcTemplate jdbcTemplate;
-	@Autowired
-	protected HdfsClient hdfsClient;
-
-	@Value("${dhp.mdstore-manager.hdfs.base-path}")
-	private String hdfsBasePath;
 
 	private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
 
@@ -94,17 +80,27 @@ public class MDStoreService {
 		final String dsId,
 		final String apiId) {
 
-		final MDStore md = MDStore.newInstance(format, layout, interpretation, dsName, dsId, apiId, hdfsBasePath);
+		final MDStore md = newMDStore(format, layout, interpretation, dsName, dsId, apiId);
 		mdstoreRepository.save(md);
 
-		final MDStoreVersion v = MDStoreVersion.newInstance(md.getId(), false, hdfsBasePath);
+		final MDStoreVersion v = newMDStoreVersion(md.getId(), false);
 		v.setLastUpdate(new Date());
+
 		mdstoreVersionRepository.save(v);
 		mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v));
 
 		return md.getId();
 	}
 
+	protected abstract MDStore newMDStore(final String format,
+		final String layout,
+		final String interpretation,
+		final String dsName,
+		final String dsId,
+		final String apiId);
+
+	protected abstract MDStoreVersion newMDStoreVersion(final String mdId, final boolean writing);
+
 	@Transactional
 	public void deleteMdStore(final String mdId) throws MDStoreManagerException {
 
@@ -129,7 +125,7 @@ public class MDStoreService {
 		mdstoreVersionRepository.deleteByMdstore(mdId);
 		mdstoreRepository.deleteById(mdId);
 
-		hdfsClient.deletePath(md.get().getHdfsPath());
+		deleteFromBackend(md.get().getHdfsPath());
 	}
 
 	@Transactional
@@ -159,7 +155,7 @@ public class MDStoreService {
 
 	@Transactional
 	public MDStoreVersion prepareMdStoreVersion(final String mdId) {
-		final MDStoreVersion v = MDStoreVersion.newInstance(mdId, true, hdfsBasePath);
+		final MDStoreVersion v = newMDStoreVersion(mdId, true);
 		mdstoreVersionRepository.save(v);
 		return v;
 	}
@@ -176,12 +172,7 @@ public class MDStoreService {
 		return v;
 	}
 
-	public StatusResponse deleteExpiredVersions() {
+	public synchronized void deleteExpiredVersions() {
-		new Thread(this::performDeleteOfExpiredVersions).start();
-		return StatusResponse.DELETING;
-	}
-
-	private synchronized void performDeleteOfExpiredVersions() {
 		log.info("Deleting expired version...");
 		for (final String versionId : listExpiredVersions()) {
 			try {
@@ -210,37 +201,8 @@ public class MDStoreService {
 
 		mdstoreVersionRepository.delete(v);
 
-		hdfsClient.deletePath(v.getHdfsPath());
+		deleteFromBackend(v.getHdfsPath());
-	}
-
-	public Set<String> fixHdfsInconsistencies(final boolean delete) throws MDStoreManagerException {
-		final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
-		final Set<String> validDirs = new HashSet<>(jdbcTemplate
-			.queryForList("select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class));
-
-		final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
-		log.info("Found " + toDelete.size() + " hdfs paths to remove");
-
-		if (delete) {
-			for (final String p : toDelete) {
-				hdfsClient.deletePath(p);
-			}
-		}
-		return toDelete;
-	}
-
-	public List<Map<String, String>> listMdstoreParquet(final String mdId, final long limit) throws MDStoreManagerException {
-		return listVersionParquet(findMdStore(mdId).getCurrentVersion(), limit);
-	}
-
-	public List<Map<String, String>> listVersionParquet(final String versionId, final long limit) throws MDStoreManagerException {
-		final String path = findVersion(versionId).getHdfsPath();
-		return hdfsClient.readParquetFiles(path + "/store", limit);
-	}
-
-	public Set<String> listVersionFiles(final String versionId) throws MDStoreManagerException {
-		final String path = findVersion(versionId).getHdfsPath();
-		return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
 	}
 
+	protected abstract void deleteFromBackend(String path) throws MDStoreManagerException;
 }

libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HadoopMDStoreService.java

@@ -0,0 +1,82 @@
+package eu.dnetlib.data.mdstore.hadoop;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import com.google.common.collect.Sets;
+
+import eu.dnetlib.data.mdstore.MDStoreService;
+import eu.dnetlib.data.mdstore.model.MDStore;
+import eu.dnetlib.data.mdstore.model.MDStoreVersion;
+import eu.dnetlib.errors.MDStoreManagerException;
+
+@Service
+public class HadoopMDStoreService extends MDStoreService {
+
+	private static final Logger log = LoggerFactory.getLogger(HadoopMDStoreService.class);
+
+	@Autowired
+	protected HdfsClient hdfsClient;
+
+	@Value("${dhp.mdstore-manager.hdfs.base-path}")
+	private String hdfsBasePath;
+
+	@Override
+	protected void deleteFromBackend(final String path) throws MDStoreManagerException {
+		hdfsClient.deletePath(path);
+	}
+
+	@Override
+	protected MDStore newMDStore(final String format,
+		final String layout,
+		final String interpretation,
+		final String dsName,
+		final String dsId,
+		final String apiId) {
+		return MDStore.newInstance(format, layout, interpretation, dsName, dsId, apiId, hdfsBasePath);
+	}
+
+	@Override
+	protected MDStoreVersion newMDStoreVersion(final String mdId, final boolean writing) {
+		return MDStoreVersion.newInstance(mdId, writing, hdfsBasePath);
+	}
+
+	public Set<String> fixHdfsInconsistencies(final boolean delete) throws MDStoreManagerException {
+		final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
+		final Set<String> validDirs = new HashSet<>(jdbcTemplate
+			.queryForList("select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class));
+
+		final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
+		log.info("Found " + toDelete.size() + " hdfs paths to remove");
+
+		if (delete) {
+			for (final String p : toDelete) {
+				deleteFromBackend(p);
+			}
+		}
+		return toDelete;
+	}
+
+	public List<Map<String, String>> listMdstoreParquet(final String mdId, final long limit) throws MDStoreManagerException {
+		return listVersionParquet(findMdStore(mdId).getCurrentVersion(), limit);
+	}
+
+	public List<Map<String, String>> listVersionParquet(final String versionId, final long limit) throws MDStoreManagerException {
+		final String path = findVersion(versionId).getHdfsPath();
+		return hdfsClient.readParquetFiles(path + "/store", limit);
+	}
+
+	public Set<String> listVersionFiles(final String versionId) throws MDStoreManagerException {
+		final String path = findVersion(versionId).getHdfsPath();
+		return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
+	}
+
+}

libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/ZeppelinClient.java

@@ -33,7 +33,7 @@ import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.util.MultiValueMap;
 import org.springframework.web.client.RestTemplate;
 
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
 import eu.dnetlib.errors.MDStoreManagerException;
 
 @Component

libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreCurrentVersionRepository.java

@@ -3,7 +3,7 @@ package eu.dnetlib.data.mdstore.repository;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
-import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
+import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
 
 @Repository
 public interface MDStoreCurrentVersionRepository extends JpaRepository<MDStoreCurrentVersion, String> {

libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreRepository.java

@@ -3,7 +3,7 @@ package eu.dnetlib.data.mdstore.repository;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
-import eu.dnetlib.dhp.schema.mdstore.MDStore;
+import eu.dnetlib.data.mdstore.model.MDStore;
 
 @Repository
 public interface MDStoreRepository extends JpaRepository<MDStore, String> {

libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreVersionRepository.java

@@ -3,7 +3,7 @@ package eu.dnetlib.data.mdstore.repository;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.stereotype.Repository;
 
-import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
+import eu.dnetlib.data.mdstore.model.MDStoreVersion;
 
 @Repository
 public interface MDStoreVersionRepository extends JpaRepository<MDStoreVersion, String> {

libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreWithInfoRepository.java

@@ -2,7 +2,7 @@ package eu.dnetlib.data.mdstore.repository;
 
 import org.springframework.stereotype.Repository;
 
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
 import eu.dnetlib.utils.ReadOnlyRepository;
 
 @Repository

libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java

@@ -0,0 +1,175 @@
+package eu.dnetlib.data.mdstore.model;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.Objects;
+import java.util.UUID;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+@Entity
+@Table(name = "mdstores")
+public class MDStore implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = 3160530489149700055L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "format")
+	private String format;
+
+	@Column(name = "layout")
+	private String layout;
+
+	@Column(name = "interpretation")
+	private String interpretation;
+
+	@Column(name = "datasource_name")
+	private String datasourceName;
+
+	@Column(name = "datasource_id")
+	private String datasourceId;
+
+	@Column(name = "api_id")
+	private String apiId;
+
+	@Column(name = "hdfs_path")
+	private String hdfsPath;
+
+	@Column(name = "creation_date")
+	@Temporal(TemporalType.TIMESTAMP)
+	private Date creationDate;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+	public String getDatasourceName() {
+		return datasourceName;
+	}
+
+	public void setDatasourceName(final String datasourceName) {
+		this.datasourceName = datasourceName;
+	}
+
+	public String getDatasourceId() {
+		return datasourceId;
+	}
+
+	public void setDatasourceId(final String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	public String getApiId() {
+		return apiId;
+	}
+
+	public void setApiId(final String apiId) {
+		this.apiId = apiId;
+	}
+
+	public String getHdfsPath() {
+		return hdfsPath;
+	}
+
+	public void setHdfsPath(final String hdfsPath) {
+		this.hdfsPath = hdfsPath;
+	}
+
+	public Date getCreationDate() {
+		return creationDate;
+	}
+
+	public void setCreationDate(final Date creationDate) {
+		this.creationDate = creationDate;
+	}
+
+	public static MDStore newInstance(
+		final String format,
+		final String layout,
+		final String interpretation,
+		final String hdfsBasePath) {
+		return newInstance(format, layout, interpretation, null, null, null, hdfsBasePath);
+	}
+
+	public static MDStore newInstance(
+		final String format,
+		final String layout,
+		final String interpretation,
+		final String dsName,
+		final String dsId,
+		final String apiId,
+		final String hdfsBasePath) {
+
+		final String mdId = "md-" + UUID.randomUUID();
+
+		final MDStore md = new MDStore();
+		md.setId(mdId);
+		md.setFormat(format);
+		md.setLayout(layout);
+		md.setInterpretation(interpretation);
+		md.setCreationDate(new Date());
+		md.setDatasourceName(dsName);
+		md.setDatasourceId(dsId);
+		md.setApiId(apiId);
+		md.setHdfsPath(String.format("%s/%s", hdfsBasePath, mdId));
+
+		return md;
+	}
+
+	@Override
+	public String toString() {
+		return String
+			.format("MDStore [id=%s, format=%s, layout=%s, interpretation=%s, datasourceName=%s, datasourceId=%s, apiId=%s, hdfsPath=%s, creationDate=%s]", id, format, layout, interpretation, datasourceName, datasourceId, apiId, hdfsPath, creationDate);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(id);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof MDStore)) { return false; }
+		final MDStore other = (MDStore) obj;
+		return Objects.equals(id, other.id);
+	}
+
+}

libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreCurrentVersion.java

@@ -0,0 +1,69 @@
+package eu.dnetlib.data.mdstore.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "mdstore_current_versions")
+public class MDStoreCurrentVersion implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = -4757725888593745773L;
+
+	@Id
+	@Column(name = "mdstore")
+	private String mdstore;
+
+	@Column(name = "current_version")
+	private String currentVersion;
+
+	public String getMdstore() {
+		return mdstore;
+	}
+
+	public void setMdstore(final String mdstore) {
+		this.mdstore = mdstore;
+	}
+
+	public String getCurrentVersion() {
+		return currentVersion;
+	}
+
+	public void setCurrentVersion(final String currentVersion) {
+		this.currentVersion = currentVersion;
+	}
+
+	public static MDStoreCurrentVersion newInstance(final String mdId, final String versionId) {
+		final MDStoreCurrentVersion cv = new MDStoreCurrentVersion();
+		cv.setMdstore(mdId);
+		cv.setCurrentVersion(versionId);
+		return cv;
+	}
+
+	public static MDStoreCurrentVersion newInstance(final MDStoreVersion v) {
+		return newInstance(v.getMdstore(), v.getId());
+	}
+
+	@Override
+	public String toString() {
+		return String.format("MDStoreCurrentVersion [mdstore=%s, currentVersion=%s]", mdstore, currentVersion);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(currentVersion, mdstore);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof MDStoreCurrentVersion)) { return false; }
+		final MDStoreCurrentVersion other = (MDStoreCurrentVersion) obj;
+		return Objects.equals(currentVersion, other.currentVersion) && Objects.equals(mdstore, other.mdstore);
+	}
+}

libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java

@@ -0,0 +1,133 @@
+package eu.dnetlib.data.mdstore.model;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.Objects;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+@Entity
+@Table(name = "mdstore_versions")
+public class MDStoreVersion implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = -4763494442274298339L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "mdstore")
+	private String mdstore;
+
+	@Column(name = "writing")
+	private boolean writing;
+
+	@Column(name = "readcount")
+	private int readCount = 0;
+
+	@Column(name = "lastupdate")
+	@Temporal(TemporalType.TIMESTAMP)
+	private Date lastUpdate;
+
+	@Column(name = "size")
+	private long size = 0;
+
+	@Column(name = "hdfs_path")
+	private String hdfsPath;
+
+	public static MDStoreVersion newInstance(final String mdId, final boolean writing, final String hdfsBasePath) {
+		final MDStoreVersion v = new MDStoreVersion();
+
+		final String versionId = mdId + "-" + new Date().getTime();
+		v.setId(versionId);
+		v.setMdstore(mdId);
+		v.setLastUpdate(null);
+		v.setWriting(writing);
+		v.setReadCount(0);
+		v.setSize(0);
+		v.setHdfsPath(String.format("%s/%s/%s", hdfsBasePath, mdId, versionId));
+
+		return v;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getMdstore() {
+		return mdstore;
+	}
+
+	public void setMdstore(final String mdstore) {
+		this.mdstore = mdstore;
+	}
+
+	public boolean isWriting() {
+		return writing;
+	}
+
+	public void setWriting(final boolean writing) {
+		this.writing = writing;
+	}
+
+	public int getReadCount() {
+		return readCount;
+	}
+
+	public void setReadCount(final int readCount) {
+		this.readCount = readCount;
+	}
+
+	public Date getLastUpdate() {
+		return lastUpdate;
+	}
+
+	public void setLastUpdate(final Date lastUpdate) {
+		this.lastUpdate = lastUpdate;
+	}
+
+	public long getSize() {
+		return size;
+	}
+
+	public void setSize(final long size) {
+		this.size = size;
+	}
+
+	public String getHdfsPath() {
+		return hdfsPath;
+	}
+
+	public void setHdfsPath(final String hdfsPath) {
+		this.hdfsPath = hdfsPath;
+	}
+
+	@Override
+	public String toString() {
+		return String
+			.format("MDStoreVersion [id=%s, mdstore=%s, writing=%s, readCount=%s, lastUpdate=%s, size=%s, hdfsPath=%s]", id, mdstore, writing, readCount, lastUpdate, size, hdfsPath);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(id);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof MDStoreVersion)) { return false; }
+		final MDStoreVersion other = (MDStoreVersion) obj;
+		return Objects.equals(id, other.id);
+	}
+}

libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java

@@ -0,0 +1,186 @@
+package eu.dnetlib.data.mdstore.model;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.Objects;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+@Entity
+@Table(name = "mdstores_with_info")
+public class MDStoreWithInfo implements Serializable {
+
+	/** */
+	private static final long serialVersionUID = -8445784770687571492L;
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "format")
+	private String format;
+
+	@Column(name = "layout")
+	private String layout;
+
+	@Column(name = "interpretation")
+	private String interpretation;
+
+	@Column(name = "datasource_name")
+	private String datasourceName;
+
+	@Column(name = "datasource_id")
+	private String datasourceId;
+
+	@Column(name = "api_id")
+	private String apiId;
+
+	@Column(name = "current_version")
+	private String currentVersion;
+
+	@Column(name = "creation_date")
+	@Temporal(TemporalType.TIMESTAMP)
+	private Date creationDate;
+
+	@Column(name = "lastupdate")
+	@Temporal(TemporalType.TIMESTAMP)
+	private Date lastUpdate;
+
+	@Column(name = "size")
+	private long size = 0;
+
+	@Column(name = "n_versions")
+	private long numberOfVersions = 0;
+
+	@Column(name = "hdfs_path")
+	private String hdfsPath;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+	public String getDatasourceName() {
+		return datasourceName;
+	}
+
+	public void setDatasourceName(final String datasourceName) {
+		this.datasourceName = datasourceName;
+	}
+
+	public String getDatasourceId() {
+		return datasourceId;
+	}
+
+	public void setDatasourceId(final String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	public String getApiId() {
+		return apiId;
+	}
+
+	public void setApiId(final String apiId) {
+		this.apiId = apiId;
+	}
+
+	public String getCurrentVersion() {
+		return currentVersion;
+	}
+
+	public void setCurrentVersion(final String currentVersion) {
+		this.currentVersion = currentVersion;
+	}
+
+	public Date getCreationDate() {
+		return creationDate;
+	}
+
+	public void setCreationDate(final Date creationDate) {
+		this.creationDate = creationDate;
+	}
+
+	public Date getLastUpdate() {
+		return lastUpdate;
+	}
+
+	public void setLastUpdate(final Date lastUpdate) {
+		this.lastUpdate = lastUpdate;
+	}
+
+	public long getSize() {
+		return size;
+	}
+
+	public void setSize(final long size) {
+		this.size = size;
+	}
+
+	public long getNumberOfVersions() {
+		return numberOfVersions;
+	}
+
+	public void setNumberOfVersions(final long numberOfVersions) {
+		this.numberOfVersions = numberOfVersions;
+	}
+
+	public String getHdfsPath() {
+		return hdfsPath;
+	}
+
+	public void setHdfsPath(final String hdfsPath) {
+		this.hdfsPath = hdfsPath;
+	}
+
+	@Override
+	public String toString() {
+		return String
+			.format("MDStoreWithInfo [id=%s, format=%s, layout=%s, interpretation=%s, datasourceName=%s, datasourceId=%s, apiId=%s, currentVersion=%s, creationDate=%s, lastUpdate=%s, size=%s, numberOfVersions=%s, hdfsPath=%s]", id, format, layout, interpretation, datasourceName, datasourceId, apiId, currentVersion, creationDate, lastUpdate, size, numberOfVersions, hdfsPath);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(id);
+	}
+
+	@Override
+	public boolean equals(final Object obj) {
+		if (this == obj) { return true; }
+		if (!(obj instanceof MDStoreWithInfo)) { return false; }
+		final MDStoreWithInfo other = (MDStoreWithInfo) obj;
+		return Objects.equals(id, other.id);
+	}
+
+}

libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MetadataRecord.java

@@ -0,0 +1,126 @@
+package eu.dnetlib.data.mdstore.model;
+
+import java.io.Serializable;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+/**
+ * This class models a record in a Metadata store collection
+ */
+public class MetadataRecord implements Serializable {
+
+	private static final long serialVersionUID = -226995288313973437L;
+
+	/** The D-Net Identifier associated to the record */
+	private String id;
+
+	/** The original Identifier of the record */
+	private String originalId;
+
+	/** The encoding of the record, should be JSON or XML */
+	private String encoding;
+
+	/**
+	 * The information about the provenance of the record see @{@link Provenance} for the model of this information
+	 */
+	private Provenance provenance;
+
+	/** The content of the metadata */
+	private String body;
+
+	/** the date when the record has been stored */
+	private Long dateOfCollection;
+
+	/** the date when the record has been stored */
+	private Long dateOfTransformation;
+
+	public MetadataRecord() {
+
+	}
+
+	public MetadataRecord(
+		final String originalId,
+		final String encoding,
+		final Provenance provenance,
+		final String body,
+		final Long dateOfCollection) {
+
+		this.originalId = originalId;
+		this.encoding = encoding;
+		this.provenance = provenance;
+		this.body = body;
+		this.dateOfCollection = dateOfCollection;
+		this.id = generateIdentifier(originalId, this.provenance.getNsPrefix());
+	}
+
+	public static String generateIdentifier(final String originalId, final String nsPrefix) {
+		return String.format("%s::%s", nsPrefix, DigestUtils.md5Hex(originalId));
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getOriginalId() {
+		return originalId;
+	}
+
+	public void setOriginalId(final String originalId) {
+		this.originalId = originalId;
+	}
+
+	public String getEncoding() {
+		return encoding;
+	}
+
+	public void setEncoding(final String encoding) {
+		this.encoding = encoding;
+	}
+
+	public Provenance getProvenance() {
+		return provenance;
+	}
+
+	public void setProvenance(final Provenance provenance) {
+		this.provenance = provenance;
+	}
+
+	public String getBody() {
+		return body;
+	}
+
+	public void setBody(final String body) {
+		this.body = body;
+	}
+
+	public Long getDateOfCollection() {
+		return dateOfCollection;
+	}
+
+	public void setDateOfCollection(final Long dateOfCollection) {
+		this.dateOfCollection = dateOfCollection;
+	}
+
+	public Long getDateOfTransformation() {
+		return dateOfTransformation;
+	}
+
+	public void setDateOfTransformation(final Long dateOfTransformation) {
+		this.dateOfTransformation = dateOfTransformation;
+	}
+
+	@Override
+	public boolean equals(final Object o) {
+		if (!(o instanceof MetadataRecord)) { return false; }
+		return ((MetadataRecord) o).getId().equalsIgnoreCase(id);
+	}
+
+	@Override
+	public int hashCode() {
+		return id.hashCode();
+	}
+}

libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/Provenance.java

@@ -0,0 +1,50 @@
+package eu.dnetlib.data.mdstore.model;
+
+import java.io.Serializable;
+
+/**
+ * @author Sandro La Bruzzo
+ *         <p>
+ *         Provenace class models the provenance of the record in the metadataStore It contains the identifier and the name of the
+ *         datasource that gives the record
+ */
+public class Provenance implements Serializable {
+
+	private String datasourceId;
+
+	private String datasourceName;
+
+	private String nsPrefix;
+
+	public Provenance() {}
+
+	public Provenance(final String datasourceId, final String datasourceName, final String nsPrefix) {
+		this.datasourceId = datasourceId;
+		this.datasourceName = datasourceName;
+		this.nsPrefix = nsPrefix;
+	}
+
+	public String getDatasourceId() {
+		return datasourceId;
+	}
+
+	public void setDatasourceId(final String datasourceId) {
+		this.datasourceId = datasourceId;
+	}
+
+	public String getDatasourceName() {
+		return datasourceName;
+	}
+
+	public void setDatasourceName(final String datasourceName) {
+		this.datasourceName = datasourceName;
+	}
+
+	public String getNsPrefix() {
+		return nsPrefix;
+	}
+
+	public void setNsPrefix(final String nsPrefix) {
+		this.nsPrefix = nsPrefix;
+	}
+}

libs/dnet-is-common/src/main/java/eu/dnetlib/utils/XmlIndenter.java

@@ -10,11 +10,9 @@ import org.dom4j.Node;
 import org.dom4j.io.OutputFormat;
 import org.dom4j.io.XMLWriter;
 
-import eu.dnetlib.is.resource.SimpleResourceService;
-
 public class XmlIndenter {
 
-	private static final Log log = LogFactory.getLog(SimpleResourceService.class);
+	private static final Log log = LogFactory.getLog(XmlIndenter.class);
 
 	public static String indent(final String xml) {
 		try {

libs/pom.xml

@@ -17,6 +17,7 @@
 		<module>dnet-openaire-broker-common</module>
 		<module>dnet-broker-apps-common</module>
 		<module>dnet-is-common</module>
+		<module>dnet-data-services</module>
 	</modules>
 
 </project>