dnet-applications/apps/dhp-mdstore-manager/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java

package eu.dnetlib.data.mdstore.manager.controller;

import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import com.google.common.collect.Sets;

import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.utils.DatabaseUtils;
import eu.dnetlib.data.mdstore.manager.utils.HdfsClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;

@RestController
@RequestMapping("/mdstores")
@Tag(name = "Metadata Stores")
public class MDStoreController extends AbstractDnetController {

	@Autowired
	private DatabaseUtils databaseUtils;

	@Autowired
	private HdfsClient hdfsClient;

	private static final Logger log = LoggerFactory.getLogger(DatabaseUtils.class);

	@Operation(summary = "Return all the mdstores")
	@GetMapping("/")
	public Iterable<MDStoreWithInfo> find() {
		return databaseUtils.listMdStores();
	}

	@Operation(summary = "Return all the mdstore identifiers")
	@GetMapping("/ids")
	public List<String> findIdentifiers() {
		return databaseUtils.listMdStoreIDs();
	}

	@Operation(summary = "Return a mdstores by id")
	@GetMapping("/mdstore/{mdId}")
	public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
		return databaseUtils.findMdStore(mdId);
	}

	@Operation(summary = "Increase the read count of the current mdstore")
	@GetMapping("/mdstore/{mdId}/startReading")
	public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
		return databaseUtils.startReading(mdId);
	}

	@Operation(summary = "Create a new mdstore")
	@GetMapping("/new/{format}/{layout}/{interpretation}")
	public MDStoreWithInfo createMDStore(
		@Parameter(name = "mdstore format") @PathVariable final String format,
		@Parameter(name = "mdstore layout") @PathVariable final String layout,
		@Parameter(name = "mdstore interpretation") @PathVariable final String interpretation,
		@Parameter(name = "datasource name") @RequestParam(required = true) final String dsName,
		@Parameter(name = "datasource id") @RequestParam(required = true) final String dsId,
		@Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
		final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
		return databaseUtils.findMdStore(id);
	}

	@Operation(summary = "Delete a mdstore by id")
	@DeleteMapping("/mdstore/{mdId}")
	public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId) throws MDStoreManagerException {
		final String hdfsPath = databaseUtils.deleteMdStore(mdId);
		hdfsClient.deletePath(hdfsPath);
		return StatusResponse.DELETED;
	}

	@Operation(summary = "Return all the versions of a mdstore")
	@GetMapping("/mdstore/{mdId}/versions")
	public Iterable<MDStoreVersion> listVersions(@PathVariable final String mdId) throws MDStoreManagerException {
		return databaseUtils.listVersions(mdId);
	}

	@Operation(summary = "Create a new preliminary version of a mdstore")
	@GetMapping("/mdstore/{mdId}/newVersion")
	public MDStoreVersion prepareNewVersion(@Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) {
		return databaseUtils.prepareMdStoreVersion(mdId);
	}

	@Operation(summary = "Promote a preliminary version to current")
	@GetMapping("/version/{versionId}/commit/{size}")
	public MDStoreVersion commitVersion(@Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId,
		@Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException {
		try {
			return databaseUtils.commitMdStoreVersion(versionId, size);
		} finally {
			deleteExpiredVersions();
		}
	}

	@Operation(summary = "Abort a preliminary version")
	@GetMapping("/version/{versionId}/abort")
	public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId) throws MDStoreManagerException {
		final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true);
		hdfsClient.deletePath(hdfsPath);
		return StatusResponse.ABORTED;
	}

	@Operation(summary = "Return an existing mdstore version")
	@GetMapping("/version/{versionId}")
	public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId)
		throws MDStoreManagerException {
		return databaseUtils.findVersion(versionId);
	}

	@Operation(summary = "Delete a mdstore version")
	@DeleteMapping("/version/{versionId}")
	public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId,
		@Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force)
		throws MDStoreManagerException {
		final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, force);
		hdfsClient.deletePath(hdfsPath);
		return StatusResponse.DELETED;
	}

	@Operation(summary = "Decrease the read count of a mdstore version")
	@GetMapping("/version/{versionId}/endReading")
	public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId)
		throws MDStoreManagerException {
		return databaseUtils.endReading(versionId);
	}

	@Operation(summary = "Reset the read count of a mdstore version")
	@GetMapping("/version/{versionId}/resetReading")
	public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId)
		throws MDStoreManagerException {
		return databaseUtils.resetReading(versionId);
	}

	@Operation(summary = "Delete expired versions")
	@DeleteMapping("/versions/expired")
	public StatusResponse deleteExpiredVersions() {
		new Thread(this::performDeleteOfExpiredVersions).start();
		return StatusResponse.DELETING;
	}

	private synchronized void performDeleteOfExpiredVersions() {
		log.info("Deleting expired version...");
		for (final String versionId : databaseUtils.listExpiredVersions()) {
			try {
				final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true);
				hdfsClient.deletePath(hdfsPath);
			} catch (final MDStoreManagerException e) {
				log.warn("Error deleteting version " + versionId, e);
			}
		}
		log.info("Done.");
	}

	@Operation(summary = "Fix the inconsistencies on HDFS")
	@GetMapping("/hdfs/inconsistencies")
	public Set<String> fixHdfsInconsistencies(
		@Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete)
		throws MDStoreManagerException {

		final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
		final Set<String> validDirs = databaseUtils.listValidHdfsPaths();

		final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
		log.info("Found " + toDelete.size() + " hdfs paths to remove");

		if (delete) {
			for (final String p : toDelete) {
				hdfsClient.deletePath(p);
			}
		}
		return toDelete;
	}

	@Operation(summary = "Show informations")
	@GetMapping("/info")
	public Map<String, Object> info() {
		final Map<String, Object> info = new LinkedHashMap<>();
		info.put("number_of_mdstores", databaseUtils.countMdStores());
		info.put("hadoop_user", hdfsClient.getHadoopUser());
		info.put("hadoop_cluster", hdfsClient.getHadoopCluster());
		info.put("hdfs_base_path", databaseUtils.getHdfsBasePath());
		info.put("expired_versions", databaseUtils.listExpiredVersions());
		return info;
	}

	@Operation(summary = "list the file inside the path of a mdstore version")
	@GetMapping("/version/{versionId}/parquet/files")
	public Set<String> listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException {
		final String path = databaseUtils.findVersion(versionId).getHdfsPath();
		return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
	}

	@Operation(summary = "read the parquet file of a mdstore version")
	@GetMapping("/version/{versionId}/parquet/content/{limit}")
	public List<Map<String, String>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
		final String path = databaseUtils.findVersion(versionId).getHdfsPath();
		return hdfsClient.readParquetFiles(path + "/store", limit);
	}

	@Operation(summary = "read the parquet file of a mdstore (current version)")
	@GetMapping("/mdstore/{mdId}/parquet/content/{limit}")
	public List<Map<String, String>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
		final String versionId = databaseUtils.findMdStore(mdId).getCurrentVersion();
		final String path = databaseUtils.findVersion(versionId).getHdfsPath();
		return hdfsClient.readParquetFiles(path + "/store", limit);
	}

	protected void setDatabaseUtils(final DatabaseUtils databaseUtils) {
		this.databaseUtils = databaseUtils;
	}

	protected void setHdfsClient(final HdfsClient hdfsClient) {
		this.hdfsClient = hdfsClient;
	}

}