mdstore ui - partial implementation

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java

@@ -0,0 +1,129 @@
+package eu.dnetlib.data.mdstore;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import eu.dnetlib.common.controller.AbstractDnetController;
+import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
+import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
+import eu.dnetlib.errors.MDStoreManagerException;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+
+public class AbstractMDStoreController extends AbstractDnetController {
+
+	@Autowired
+	protected MDStoreService service;
+
+	@Operation(summary = "Return all the mdstores")
+	@GetMapping("/")
+	public Iterable<MDStoreWithInfo> find() {
+		return service.listMdStores();
+	}
+
+	@Operation(summary = "Return a mdstores by id")
+	@GetMapping("/mdstore/{mdId}")
+	public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
+		return service.findMdStore(mdId);
+	}
+
+	@Operation(summary = "Increase the read count of the current mdstore")
+	@GetMapping("/mdstore/{mdId}/startReading")
+	public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
+		return service.startReading(mdId);
+	}
+
+	@Operation(summary = "Create a new mdstore")
+	@PostMapping("/new/{format}/{layout}/{interpretation}")
+	public MDStoreWithInfo createMDStore(
+		@Parameter(name = "mdstore format") @PathVariable final String format,
+		@Parameter(name = "mdstore layout") @PathVariable final String layout,
+		@Parameter(name = "mdstore interpretation") @PathVariable final String interpretation,
+		@Parameter(name = "datasource name") @RequestParam(required = true) final String dsName,
+		@Parameter(name = "datasource id") @RequestParam(required = true) final String dsId,
+		@Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
+		final String id = service.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
+		return service.findMdStore(id);
+	}
+
+	@Operation(summary = "Delete a mdstore by id")
+	@DeleteMapping("/mdstore/{mdId}")
+	public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId)
+		throws MDStoreManagerException {
+		service.deleteMdStore(mdId);
+
+		return StatusResponse.DELETED;
+	}
+
+	@Operation(summary = "Return all the versions of a mdstore")
+	@GetMapping("/mdstore/{mdId}/versions")
+	public Iterable<MDStoreVersion> listVersions(@PathVariable final String mdId) throws MDStoreManagerException {
+		return service.listVersions(mdId);
+	}
+
+	@Operation(summary = "Create a new preliminary version of a mdstore")
+	@GetMapping("/mdstore/{mdId}/newVersion")
+	public MDStoreVersion prepareNewVersion(
+		@Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) {
+		return service.prepareMdStoreVersion(mdId);
+	}
+
+	@Operation(summary = "Promote a preliminary version to current")
+	@GetMapping("/version/{versionId}/commit/{size}")
+	public MDStoreVersion commitVersion(
+		@Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId,
+		@Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException {
+		try {
+			return service.commitMdStoreVersion(versionId, size);
+		} finally {
+			service.deleteExpiredVersions();
+		}
+	}
+
+	@Operation(summary = "Abort a preliminary version")
+	@GetMapping("/version/{versionId}/abort")
+	public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId)
+		throws MDStoreManagerException {
+
+		service.deleteMdStoreVersion(versionId, true);
+
+		return StatusResponse.ABORTED;
+	}
+
+	@Operation(summary = "Return an existing mdstore version")
+	@GetMapping("/version/{versionId}")
+	public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId)
+		throws MDStoreManagerException {
+		return service.findVersion(versionId);
+	}
+
+	@Operation(summary = "Delete a mdstore version")
+	@DeleteMapping("/version/{versionId}")
+	public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId,
+		@Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force)
+		throws MDStoreManagerException {
+
+		service.deleteMdStoreVersion(versionId, force);
+
+		return StatusResponse.DELETED;
+	}
+
+	@Operation(summary = "Decrease the read count of a mdstore version")
+	@GetMapping("/version/{versionId}/endReading")
+	public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId)
+		throws MDStoreManagerException {
+		return service.endReading(versionId);
+	}
+
+	@Operation(summary = "Reset the read count of a mdstore version")
+	@GetMapping("/version/{versionId}/resetReading")
+	public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId)
+		throws MDStoreManagerException {
+		return service.resetReading(versionId);
+	}
+
+}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.data.mdstore;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.errors.MDStoreManagerException;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@RequestMapping("/ajax/mdstores")
+@Tag(name = "Metadata Stores")
+public class MDStoreAjaxController extends AbstractMDStoreController {
+
+	@Value("${dhp.mdstore-manager.hadoop.cluster}")
+	private String hadoopCluster;
+
+	@Value("${dhp.mdstore-manager.hadoop.user}")
+	private String hadoopUser;
+
+	@Value("${dhp.mdstore-manager.hdfs.base-path}")
+	private String hdfsBasePath;
+
+	@Operation(summary = "Show informations")
+	@GetMapping("/info")
+	public Map<String, Object> info() {
+		final Map<String, Object> info = new LinkedHashMap<>();
+		info.put("number_of_mdstores", service.countMdStores());
+		info.put("hadoop_user", hadoopUser);
+		info.put("hadoop_cluster", hadoopCluster);
+		info.put("hdfs_base_path", hdfsBasePath);
+		info.put("expired_versions", service.listExpiredVersions());
+		return info;
+	}
+
+	@Operation(summary = "read the parquet file of a mdstore version")
+	@GetMapping("/version/{versionId}/parquet/content/{limit}")
+	public List<Map<String, String>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
+		return service.listVersionParquet(versionId, limit);
+	}
+
+	@Operation(summary = "read the parquet file of a mdstore (current version)")
+	@GetMapping("/mdstore/{mdId}/parquet/content/{limit}")
+	public List<Map<String, String>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
+		return service.listMdstoreParquet(mdId, limit);
+	}
+}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java

@@ -0,0 +1,50 @@
+package eu.dnetlib.data.mdstore;
+
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.errors.MDStoreManagerException;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@RequestMapping("/api/mdstores")
+@Tag(name = "Metadata Stores")
+public class MDStoreApiController extends AbstractMDStoreController {
+
+	@Operation(summary = "Return all the mdstore identifiers")
+	@GetMapping("/ids")
+	public List<String> findIdentifiers() {
+		return service.listMdStoreIDs();
+	}
+
+	@Operation(summary = "Fix the inconsistencies on HDFS")
+	@GetMapping("/hdfs/inconsistencies")
+	public Set<String> fixHdfsInconsistencies(
+		@Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete)
+		throws MDStoreManagerException {
+
+		return service.fixHdfsInconsistencies(delete);
+	}
+
+	@Operation(summary = "Delete expired versions")
+	@DeleteMapping("/versions/expired")
+	public StatusResponse deleteExpiredVersions() {
+		return service.deleteExpiredVersions();
+	}
+
+	@Operation(summary = "list the file inside the path of a mdstore version")
+	@GetMapping("/version/{versionId}/parquet/files")
+	public Set<String> listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException {
+		return service.listVersionFiles(versionId);
+	}
+
+}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreController.java

@@ -1,231 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.DeleteMapping;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.google.common.collect.Sets;
-
-import eu.dnetlib.common.controller.AbstractDnetController;
-import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
-import eu.dnetlib.errors.MDStoreManagerException;
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.Parameter;
-import io.swagger.v3.oas.annotations.tags.Tag;
-
-@RestController
-@RequestMapping("/mdstores")
-@Tag(name = "Metadata Stores")
-public class MDStoreController extends AbstractDnetController {
-
-	@Autowired
-	private MDStoreService databaseUtils;
-
-	@Autowired
-	private HdfsClient hdfsClient;
-
-	private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
-
-	@Operation(summary = "Return all the mdstores")
-	@GetMapping("/")
-	public Iterable<MDStoreWithInfo> find() {
-		return databaseUtils.listMdStores();
-	}
-
-	@Operation(summary = "Return all the mdstore identifiers")
-	@GetMapping("/ids")
-	public List<String> findIdentifiers() {
-		return databaseUtils.listMdStoreIDs();
-	}
-
-	@Operation(summary = "Return a mdstores by id")
-	@GetMapping("/mdstore/{mdId}")
-	public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
-		return databaseUtils.findMdStore(mdId);
-	}
-
-	@Operation(summary = "Increase the read count of the current mdstore")
-	@GetMapping("/mdstore/{mdId}/startReading")
-	public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
-		return databaseUtils.startReading(mdId);
-	}
-
-	@Operation(summary = "Create a new mdstore")
-	@GetMapping("/new/{format}/{layout}/{interpretation}")
-	public MDStoreWithInfo createMDStore(
-		@Parameter(name = "mdstore format") @PathVariable final String format,
-		@Parameter(name = "mdstore layout") @PathVariable final String layout,
-		@Parameter(name = "mdstore interpretation") @PathVariable final String interpretation,
-		@Parameter(name = "datasource name") @RequestParam(required = true) final String dsName,
-		@Parameter(name = "datasource id") @RequestParam(required = true) final String dsId,
-		@Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
-		final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
-		return databaseUtils.findMdStore(id);
-	}
-
-	@Operation(summary = "Delete a mdstore by id")
-	@DeleteMapping("/mdstore/{mdId}")
-	public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId) throws MDStoreManagerException {
-		final String hdfsPath = databaseUtils.deleteMdStore(mdId);
-		hdfsClient.deletePath(hdfsPath);
-		return StatusResponse.DELETED;
-	}
-
-	@Operation(summary = "Return all the versions of a mdstore")
-	@GetMapping("/mdstore/{mdId}/versions")
-	public Iterable<MDStoreVersion> listVersions(@PathVariable final String mdId) throws MDStoreManagerException {
-		return databaseUtils.listVersions(mdId);
-	}
-
-	@Operation(summary = "Create a new preliminary version of a mdstore")
-	@GetMapping("/mdstore/{mdId}/newVersion")
-	public MDStoreVersion prepareNewVersion(@Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) {
-		return databaseUtils.prepareMdStoreVersion(mdId);
-	}
-
-	@Operation(summary = "Promote a preliminary version to current")
-	@GetMapping("/version/{versionId}/commit/{size}")
-	public MDStoreVersion commitVersion(@Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId,
-		@Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException {
-		try {
-			return databaseUtils.commitMdStoreVersion(versionId, size);
-		} finally {
-			deleteExpiredVersions();
-		}
-	}
-
-	@Operation(summary = "Abort a preliminary version")
-	@GetMapping("/version/{versionId}/abort")
-	public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId) throws MDStoreManagerException {
-		final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true);
-		hdfsClient.deletePath(hdfsPath);
-		return StatusResponse.ABORTED;
-	}
-
-	@Operation(summary = "Return an existing mdstore version")
-	@GetMapping("/version/{versionId}")
-	public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId)
-		throws MDStoreManagerException {
-		return databaseUtils.findVersion(versionId);
-	}
-
-	@Operation(summary = "Delete a mdstore version")
-	@DeleteMapping("/version/{versionId}")
-	public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId,
-		@Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force)
-		throws MDStoreManagerException {
-		final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, force);
-		hdfsClient.deletePath(hdfsPath);
-		return StatusResponse.DELETED;
-	}
-
-	@Operation(summary = "Decrease the read count of a mdstore version")
-	@GetMapping("/version/{versionId}/endReading")
-	public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId)
-		throws MDStoreManagerException {
-		return databaseUtils.endReading(versionId);
-	}
-
-	@Operation(summary = "Reset the read count of a mdstore version")
-	@GetMapping("/version/{versionId}/resetReading")
-	public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId)
-		throws MDStoreManagerException {
-		return databaseUtils.resetReading(versionId);
-	}
-
-	@Operation(summary = "Delete expired versions")
-	@DeleteMapping("/versions/expired")
-	public StatusResponse deleteExpiredVersions() {
-		new Thread(this::performDeleteOfExpiredVersions).start();
-		return StatusResponse.DELETING;
-	}
-
-	private synchronized void performDeleteOfExpiredVersions() {
-		log.info("Deleting expired version...");
-		for (final String versionId : databaseUtils.listExpiredVersions()) {
-			try {
-				final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true);
-				hdfsClient.deletePath(hdfsPath);
-			} catch (final MDStoreManagerException e) {
-				log.warn("Error deleteting version " + versionId, e);
-			}
-		}
-		log.info("Done.");
-	}
-
-	@Operation(summary = "Fix the inconsistencies on HDFS")
-	@GetMapping("/hdfs/inconsistencies")
-	public Set<String> fixHdfsInconsistencies(
-		@Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete)
-		throws MDStoreManagerException {
-
-		final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
-		final Set<String> validDirs = databaseUtils.listValidHdfsPaths();
-
-		final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
-		log.info("Found " + toDelete.size() + " hdfs paths to remove");
-
-		if (delete) {
-			for (final String p : toDelete) {
-				hdfsClient.deletePath(p);
-			}
-		}
-		return toDelete;
-	}
-
-	@Operation(summary = "Show informations")
-	@GetMapping("/info")
-	public Map<String, Object> info() {
-		final Map<String, Object> info = new LinkedHashMap<>();
-		info.put("number_of_mdstores", databaseUtils.countMdStores());
-		info.put("hadoop_user", hdfsClient.getHadoopUser());
-		info.put("hadoop_cluster", hdfsClient.getHadoopCluster());
-		info.put("hdfs_base_path", databaseUtils.getHdfsBasePath());
-		info.put("expired_versions", databaseUtils.listExpiredVersions());
-		return info;
-	}
-
-	@Operation(summary = "list the file inside the path of a mdstore version")
-	@GetMapping("/version/{versionId}/parquet/files")
-	public Set<String> listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException {
-		final String path = databaseUtils.findVersion(versionId).getHdfsPath();
-		return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
-	}
-
-	@Operation(summary = "read the parquet file of a mdstore version")
-	@GetMapping("/version/{versionId}/parquet/content/{limit}")
-	public List<Map<String, String>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
-		final String path = databaseUtils.findVersion(versionId).getHdfsPath();
-		return hdfsClient.readParquetFiles(path + "/store", limit);
-	}
-
-	@Operation(summary = "read the parquet file of a mdstore (current version)")
-	@GetMapping("/mdstore/{mdId}/parquet/content/{limit}")
-	public List<Map<String, String>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
-		final String versionId = databaseUtils.findMdStore(mdId).getCurrentVersion();
-		final String path = databaseUtils.findVersion(versionId).getHdfsPath();
-		return hdfsClient.readParquetFiles(path + "/store", limit);
-	}
-
-	protected void setDatabaseUtils(final MDStoreService databaseUtils) {
-		this.databaseUtils = databaseUtils;
-	}
-
-	protected void setHdfsClient(final HdfsClient hdfsClient) {
-		this.hdfsClient = hdfsClient;
-	}
-
-}

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java

@@ -3,6 +3,7 @@ package eu.dnetlib.data.mdstore;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -16,6 +17,9 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
 
+import com.google.common.collect.Sets;
+
+import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
 import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository;
 import eu.dnetlib.data.mdstore.repository.MDStoreRepository;
 import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository;
@@ -39,6 +43,8 @@ public class MDStoreService {
 	private MDStoreWithInfoRepository mdstoreWithInfoRepository;
 	@Autowired
 	private JdbcTemplate jdbcTemplate;
+	@Autowired
+	protected HdfsClient hdfsClient;
 
 	@Value("${dhp.mdstore-manager.hdfs.base-path}")
 	private String hdfsBasePath;
@@ -94,7 +100,7 @@ public class MDStoreService {
 	}
 
 	@Transactional
-	public String deleteMdStore(final String mdId) throws MDStoreManagerException {
+	public void deleteMdStore(final String mdId) throws MDStoreManagerException {
 
 		final Optional<MDStore> md = mdstoreRepository.findById(mdId);
 
@@ -117,7 +123,7 @@ public class MDStoreService {
 		mdstoreVersionRepository.deleteByMdstore(mdId);
 		mdstoreRepository.deleteById(mdId);
 
-		return md.get().getHdfsPath();
+		hdfsClient.deletePath(md.get().getHdfsPath());
 	}
 
 	@Transactional
@@ -164,8 +170,25 @@ public class MDStoreService {
 		return v;
 	}
 
+	public StatusResponse deleteExpiredVersions() {
+		new Thread(this::performDeleteOfExpiredVersions).start();
+		return StatusResponse.DELETING;
+	}
+
+	private synchronized void performDeleteOfExpiredVersions() {
+		log.info("Deleting expired version...");
+		for (final String versionId : listExpiredVersions()) {
+			try {
+				deleteMdStoreVersion(versionId, true);
+			} catch (final MDStoreManagerException e) {
+				log.warn("Error deleteting version " + versionId, e);
+			}
+		}
+		log.info("Done.");
+	}
+
 	@Transactional
-	public String deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException {
+	public void deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException {
 
 		final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
 
@@ -181,20 +204,37 @@ public class MDStoreService {
 
 		mdstoreVersionRepository.delete(v);
 
-		return v.getHdfsPath();
+		hdfsClient.deletePath(v.getHdfsPath());
 	}
 
-	public Set<String> listValidHdfsPaths() {
+	public Set<String> fixHdfsInconsistencies(final boolean delete) throws MDStoreManagerException {
-		return new HashSet<>(jdbcTemplate
+		final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
-			.queryForList(" select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class));
+		final Set<String> validDirs = new HashSet<>(jdbcTemplate
+			.queryForList("select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class));
+
+		final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
+		log.info("Found " + toDelete.size() + " hdfs paths to remove");
+
+		if (delete) {
+			for (final String p : toDelete) {
+				hdfsClient.deletePath(p);
+			}
+		}
+		return toDelete;
 	}
 
-	public String getHdfsBasePath() {
+	public List<Map<String, String>> listMdstoreParquet(final String mdId, final long limit) throws MDStoreManagerException {
-		return hdfsBasePath;
+		return listVersionParquet(findMdStore(mdId).getCurrentVersion(), limit);
 	}
 
-	public void setHdfsBasePath(final String hdfsBasePath) {
+	public List<Map<String, String>> listVersionParquet(final String versionId, final long limit) throws MDStoreManagerException {
-		this.hdfsBasePath = hdfsBasePath;
+		final String path = findVersion(versionId).getHdfsPath();
+		return hdfsClient.readParquetFiles(path + "/store", limit);
+	}
+
+	public Set<String> listVersionFiles(final String versionId) throws MDStoreManagerException {
+		final String path = findVersion(versionId).getHdfsPath();
+		return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
 	}
 
 }

frontends/dnet-is-application/src/app/app.module.ts

@@ -34,7 +34,7 @@ import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent, DsmAddApiDial
 import { MatPaginatorModule } from '@angular/material/paginator';
 import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
 import { SpinnerHttpInterceptor } from './common/spinner.service';
-import { MdstoresComponent, MdstoreInspectorComponent } from './mdstores/mdstores.component';
+import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component';
 
 @NgModule({
   declarations: [
@@ -62,7 +62,9 @@ import { MdstoresComponent, MdstoreInspectorComponent } from './mdstores/mdstore
     DsmAddApiDialog,
     DsmBrowseDialog,
     MdstoresComponent,
-    MdstoreInspectorComponent
+    MdstoreInspectorComponent,
+    MDStoreVersionsDialog,
+    AddMDStoreDialog
   ],
   imports: [
     BrowserModule,

frontends/dnet-is-application/src/app/common/is.model.ts

@@ -162,3 +162,29 @@ export interface DsmConf {
     contentDescTypes: string[],
     protocols: Protocol[]
 }
+
+export interface MDStore {
+    id: string,
+    format: string,
+    layout: string,
+    interpretation: string,
+    datasourceName: string,
+    datasourceId: string,
+    apiId: string,
+    currentVersion: string,
+    creationDate: string,
+    lastUpdate: string,
+    size: number,
+    numberOfVersions: number,
+    hdfsPath: string
+}
+
+export interface MDStoreVersion {
+    id: string,
+    mdstore: string,
+    writing: boolean,
+    readCount:number,
+    lastUpdate:string,
+    siz: number,
+    hdfsPath: string;
+}

frontends/dnet-is-application/src/app/common/is.service.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http';
-import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource } from './is.model';
+import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion } from './is.model';
 import { FormGroup } from '@angular/forms';
 import { MatSnackBar } from '@angular/material/snack-bar';
 
@@ -111,21 +111,21 @@ export class ISService {
     });
   }
 
-  loadContext(ctxId:string, onSuccess: Function): void {
+  loadContext(ctxId: string, onSuccess: Function): void {
     this.client.get<Context>('./ajax/contexts/' + encodeURIComponent(ctxId)).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
-  loadContextCategories(ctxId:string, onSuccess: Function): void {
+  loadContextCategories(ctxId: string, onSuccess: Function): void {
     this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(ctxId) + '/categories').subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
-  loadContextConcepts(level:number, nodeId:string, onSuccess: Function): void {
+  loadContextConcepts(level: number, nodeId: string, onSuccess: Function): void {
     this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(level) + '/' + encodeURIComponent(nodeId) + '/concepts').subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
@@ -139,42 +139,42 @@ export class ISService {
     });
   }
 
-  loadVocabulary(vocId:string, onSuccess: Function): void {
+  loadVocabulary(vocId: string, onSuccess: Function): void {
     this.client.get<Vocabulary>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
-  loadVocabularyTerms(vocId:string, onSuccess: Function): void {
+  loadVocabularyTerms(vocId: string, onSuccess: Function): void {
     this.client.get<VocabularyTerm[]>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms').subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
-  saveVocabulary(voc:Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void {
+  saveVocabulary(voc: Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void {
     this.client.post<void>('./ajax/vocs/', voc).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error, relatedForm)
     });
   }
 
-  saveVocabularyTerm(vocId:string, term:VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void {
+  saveVocabularyTerm(vocId: string, term: VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void {
     this.client.post<void>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms', term).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error, relatedForm)
     });
   }
 
-  deleteVocabulary(vocId:string, onSuccess: Function): void {
+  deleteVocabulary(vocId: string, onSuccess: Function): void {
     this.client.delete<void>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
-  deleteVocabularyTerm(vocId:string, termCode:string, onSuccess: Function): void {
+  deleteVocabularyTerm(vocId: string, termCode: string, onSuccess: Function): void {
     this.client.delete<void>('./ajax/vocs/'
       + encodeURIComponent(vocId)
       + '/terms/'
@@ -199,27 +199,68 @@ export class ISService {
     });
   }
 
-  dsmBrowse(field:string, onSuccess: Function) {
+  dsmBrowse(field: string, onSuccess: Function) {
     this.client.get<BrowseTerm[]>('./ajax/dsm/browse/' + encodeURIComponent(field)).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
-  dsmSearchByField(field:string, value:string, page:number, pageSize:number, onSuccess: Function) {
+  dsmSearchByField(field: string, value: string, page: number, pageSize: number, onSuccess: Function) {
     this.client.get<Page<Datasource>>('./ajax/dsm/searchByField/' + encodeURIComponent(field) + '/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
-  dsmSearch(value:string, page:number, pageSize:number, onSuccess: Function) {
+  dsmSearch(value: string, page: number, pageSize: number, onSuccess: Function) {
     this.client.get<Page<Datasource>>('./ajax/dsm/search/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
       next: data => onSuccess(data),
       error: error => this.showError(error)
     });
   }
 
+  loadMDStores(onSuccess: Function): void {
+    this.client.get<MDStore[]>("/ajax/mdstores/").subscribe({
+      next: data => onSuccess(data),
+      error: error => this.showError(error)
+    });
+  }
+
+  addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) {
+    const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
+
+    let body = new HttpParams()
+      .set('dsName', dsName)
+      .set('dsId', dsId)
+      .set('apiId', apiId);
+
+    this.client.post<void>('/ajax/mdstores/new/'
+      + encodeURIComponent(format)
+      + '/'
+      + encodeURIComponent(layout)
+      + '/'
+      + encodeURIComponent(interpretation), 
+        body, { headers: headers }).subscribe({
+        next: data => onSuccess(data),
+        error: error => this.showError(error, relatedForm)
+      });
+  }
+
+  deleteMDStore(mdId: string, onSuccess: Function): void {
+    this.client.delete<void>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
+      next: data => onSuccess(data),
+      error: error => this.showError(error)
+    });
+  }
+
+  prepareNewMDStoreVersion(mdId: string, onSuccess: Function): void {
+    this.client.get<MDStoreVersion>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/newVersion').subscribe({
+      next: data => onSuccess(data),
+      error: error => this.showError(error)
+    });
+  }
+
   private showError(error: any, form?: FormGroup) {
     const msg = this.errorMessage(error);
     if (form) {

frontends/dnet-is-application/src/app/mdstores/add-mdstore-dialog.html

@@ -0,0 +1,50 @@
+<form [formGroup]="newMdstoreForm" (ngSubmit)="onSubmit()">
+    <h1 mat-dialog-title>New MDStore</h1>
+  
+    <div mat-dialog-content>
+  
+      <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
+        <mat-label>Format</mat-label>
+        <input matInput formControlName="format" />
+        <mat-error *ngIf="newMdstoreForm.get('format')?.invalid">This field is <strong>required</strong></mat-error>
+      </mat-form-field>
+
+      <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
+        <mat-label>Layout</mat-label>
+        <input matInput formControlName="layout" />
+        <mat-error *ngIf="newMdstoreForm.get('layout')?.invalid">This field is <strong>required</strong></mat-error>
+      </mat-form-field>
+
+      <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
+        <mat-label>Interpretation</mat-label>
+        <input matInput formControlName="interpretation" />
+        <mat-error *ngIf="newMdstoreForm.get('interpretation')?.invalid">This field is <strong>required</strong></mat-error>
+      </mat-form-field>
+  
+      <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
+        <mat-label>Datasource Name</mat-label>
+        <input matInput formControlName="dsName" />
+      </mat-form-field>
+
+      <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
+        <mat-label>Datasource ID</mat-label>
+        <input matInput formControlName="dsId" />
+      </mat-form-field>
+
+      <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
+        <mat-label>API ID</mat-label>
+        <input matInput formControlName="apiId" />
+      </mat-form-field>
+
+    </div>
+  
+    <div mat-dialog-actions>
+      <button mat-stroked-button color="primary" type="submit" [disabled]="!newMdstoreForm.valid">Submit</button>
+      <button mat-stroked-button color="primary" mat-dialog-close>Close</button>
+      <mat-error *ngIf="newMdstoreForm.errors?.['serverError']">
+        {{ newMdstoreForm.errors?.['serverError'] }}
+      </mat-error>
+    </div>
+  
+  </form>
+  

frontends/dnet-is-application/src/app/mdstores/mdstores-versions-dialog.html

@@ -0,0 +1 @@
+VERSIONS

frontends/dnet-is-application/src/app/mdstores/mdstores.component.css

@@ -0,0 +1,25 @@
+.mdstore-info-table {
+    margin-top: 1em;
+    margin-bottom: 1em;
+    border-collapse: collapse; 
+}
+
+.mdstore-info-table tr:not(:last-child) {
+    border-bottom: 1pt solid lightgrey;
+}
+
+.mdstore-info-table th {
+    width: 30%;
+}
+
+.mdstore-info-table th, .mdstore-info-table td{
+    text-align: left;
+    font-size: 0.9em;
+    vertical-align: top;
+}
+
+.mdstore-info-table td button { 
+    font-size: 0.8em !important;
+    padding: 0 !important;
+    height: 2.5em !important;
+}

frontends/dnet-is-application/src/app/mdstores/mdstores.component.html

@@ -1 +1,60 @@
 <h2>Metadata Stores</h2>
+
+<button mat-stroked-button color="primary" (click)="openAddMdstoreDialog()">create a new mdstore</button>
+
+<mat-form-field appearance="fill" floatLabel="always" style="width: 100%; margin-top: 10px;">
+    <mat-label><b>Filter</b> (Total: {{(mdstores | searchFilter: searchText).length}})</mat-label>
+    <input matInput [(ngModel)]="searchText" placeholder="Filter..." autofocus />
+</mat-form-field>
+
+<mat-card *ngFor="let md of mdstores | searchFilter: searchText" style="margin-top: 10px;">
+    <mat-card-header>
+        <mat-card-title>{{md.id}}</mat-card-title>
+        <mat-card-subtitle>{{md.datasourceName}}</mat-card-subtitle>
+    </mat-card-header>
+    <mat-card-content>
+        <table class="mdstore-info-table">
+            <tr>
+                <th>Format / Layout / Interpretation</th>
+                <td>{{md.format}} / {{md.layout}} / {{md.interpretation}}</td>
+            </tr>
+            <tr>
+                <th>Datasource</th>
+                <td>
+                    <b>Name:</b> {{md.datasourceName}}<br />
+                    <b>ID:</b> {{md.datasourceId}}<br />
+                    <b>API:</b> {{md.apiId}}
+                </td>
+            </tr>
+            <tr>
+                <th>Creation Date</th>
+                <td>{{md.creationDate}}</td>
+            </tr>
+            <tr>
+                <th>Last Update</th>
+                <td>{{md.lastUpdate}}</td>
+            </tr>
+            <tr>
+                <th>Size</th>
+                <td>{{md.size}}</td>
+            </tr>
+            <tr>
+                <th>HDFS Path</th>
+                <td>{{md.hdfsPath}}</td>
+            </tr>
+            <tr>
+                <th>Versions</th>
+                <td>
+                    <a (click)="openVersionsDialog(md)">{{md.numberOfVersions}} version(s)</a>
+                    /
+                    <a (click)="createNewVersion(md)">prepare new version</a>
+                </td>
+            </tr>
+        </table>
+    </mat-card-content>
+    <mat-card-actions>
+        <a [routerLink]="['/mdstore/' + md.id + '/100']" mat-stroked-button color="primary">inspect</a>
+        <button mat-stroked-button color="warn" (click)="deleteMdstore(md)">delete</button>
+        <button mat-stroked-button color="info">zeppelin</button>
+    </mat-card-actions>
+</mat-card>

frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts

@@ -1,12 +1,59 @@
-import { Component } from '@angular/core';
+import { Component, Inject, OnInit } from '@angular/core';
+import { ISService } from '../common/is.service';
+import { ActivatedRoute } from '@angular/router';
+import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
+import { MDStore, MDStoreVersion } from '../common/is.model';
+import { FormControl, FormGroup, Validators } from '@angular/forms';
 
 @Component({
   selector: 'app-mdstores',
   templateUrl: './mdstores.component.html',
   styleUrls: ['./mdstores.component.css']
 })
-export class MdstoresComponent {
+export class MdstoresComponent implements OnInit {
 
+  mdstores:MDStore[] = [];
+  searchText:string = '';
+
+  constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) {}
+
+  ngOnInit() { this.reload() }
+  reload() { this.service.loadMDStores((data: MDStore[]) => this.mdstores = data); }
+
+  openVersionsDialog(md:MDStore): void {
+    const dialogRef = this.dialog.open(MDStoreVersionsDialog, {
+      data: md,
+      width: '80%'
+    });
+
+    dialogRef.afterClosed().subscribe(result => {
+      if (result) this.reload();
+    });
+  }
+
+  openAddMdstoreDialog(): void {
+    const dialogRef = this.dialog.open(AddMDStoreDialog, {
+      data: {},
+      width: '80%'
+    });
+
+    dialogRef.afterClosed().subscribe(result => {
+      if (result) this.reload();
+    });
+  }
+
+  createNewVersion(md:MDStore): void {
+    this.service.prepareNewMDStoreVersion(md.id, (data:MDStoreVersion) => {
+      md.numberOfVersions = md.numberOfVersions + 1;
+      this.openVersionsDialog(md);
+    });
+  }
+
+  deleteMdstore(md:MDStore) {
+    if (confirm('Are you sure?')) {
+      this.service.deleteMDStore(md.id, (data: void) => this.reload());
+    }
+  }
 }
 
 @Component({
@@ -17,3 +64,55 @@ export class MdstoresComponent {
 export class MdstoreInspectorComponent {
 
 }
+
+@Component({
+  selector: 'mdstores-versions-dialog',
+  templateUrl: './mdstores-versions-dialog.html',
+  styleUrls: ['./mdstores.component.css']
+})
+export class MDStoreVersionsDialog {
+ 
+  constructor(public dialogRef: MatDialogRef<MDStoreVersionsDialog>,  @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) {
+   
+  }
+  
+  onNoClick(): void {
+    this.dialogRef.close();
+  }
+}
+
+@Component({
+  selector: 'add-mdstore-dialog',
+  templateUrl: './add-mdstore-dialog.html',
+  styleUrls: ['./mdstores.component.css']
+})
+export class AddMDStoreDialog {
+
+  newMdstoreForm = new FormGroup({
+    format:  new FormControl('', [Validators.required]),
+    layout : new FormControl('', [Validators.required]),
+    interpretation : new FormControl('', [Validators.required]),
+    dsName : new FormControl(''),
+    dsId : new FormControl(''),
+    apiId : new FormControl(''),
+  });
+ 
+  constructor(public dialogRef: MatDialogRef<MDStoreVersionsDialog>,  @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) {
+   
+  }
+
+  onSubmit():void {
+    let format:string = this.newMdstoreForm.get('format')?.value!;
+    let layout:string = this.newMdstoreForm.get('layout')?.value!;
+    let interpretation:string = this.newMdstoreForm.get('interpretation')?.value!;
+    let dsName:string = this.newMdstoreForm.get('dsName')?.value!;
+    let dsId:string = this.newMdstoreForm.get('dsId')?.value!;
+    let apiId:string = this.newMdstoreForm.get('apiId')?.value!;
+    
+    this.service.addMDStore(format, layout, interpretation, dsName, dsId, apiId, (data: void) => this.dialogRef.close(1), this.newMdstoreForm);
+  }
+  
+  onNoClick(): void {
+    this.dialogRef.close();
+  }
+}

frontends/dnet-is-application/src/styles.css

@@ -8,6 +8,23 @@ body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; }
 
 table { 
     width: 100%; 
+    table-layout: fixed !important;
+}
+
+tr {
+    height: auto !important;
+}
+
+th.mat-sort-header-sorted { color: black !important; }
+
+th, td {
+    white-space: nowrap !important; 
+    overflow: hidden !important; 
+    text-overflow: ellipsis !important;
+    font-size: 0.9em !important;
+    padding-top: 0.5em !important;
+    padding-bottom: 0.5em !important;
+    text-align: left;
 }
 
 .table-buttons { text-align: right !important; }
@@ -39,25 +56,6 @@ a:hover, a:not([href]):hover {
     text-decoration: underline;
 }
 
-table {
-    table-layout: fixed !important;
-}
-
-tr {
-    height: auto !important;
-}
-
-th.mat-sort-header-sorted { color: black !important; }
-
-th, td {
-    white-space: nowrap !important; 
-    overflow: hidden !important; 
-    text-overflow: ellipsis !important;
-    font-size: 0.9em !important;
-    padding-top: 0.5em !important;
-    padding-bottom: 0.5em !important;
-}
-
 .badge-label {
     padding-top: 0.3em;
     padding-bottom: 0.3em;