D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mdstore ui - partial implementation

This commit is contained in:
Michele Artini 2023-02-07 11:22:20 +01:00
parent 87df817768
commit 32c9766db7
14 changed files with 627 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java Normal file
View File

@ -0,0 +1,129 @@
package eu.dnetlib.data.mdstore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
public class AbstractMDStoreController extends AbstractDnetController {
@Autowired
protected MDStoreService service;
@Operation(summary = "Return all the mdstores")
@GetMapping("/")
public Iterable<MDStoreWithInfo> find() {
return service.listMdStores();
}
@Operation(summary = "Return a mdstores by id")
@GetMapping("/mdstore/{mdId}")
public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
return service.findMdStore(mdId);
}
@Operation(summary = "Increase the read count of the current mdstore")
@GetMapping("/mdstore/{mdId}/startReading")
public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
return service.startReading(mdId);
}
@Operation(summary = "Create a new mdstore")
@PostMapping("/new/{format}/{layout}/{interpretation}")
public MDStoreWithInfo createMDStore(
@Parameter(name = "mdstore format") @PathVariable final String format,
@Parameter(name = "mdstore layout") @PathVariable final String layout,
@Parameter(name = "mdstore interpretation") @PathVariable final String interpretation,
@Parameter(name = "datasource name") @RequestParam(required = true) final String dsName,
@Parameter(name = "datasource id") @RequestParam(required = true) final String dsId,
@Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
final String id = service.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
return service.findMdStore(id);
}
@Operation(summary = "Delete a mdstore by id")
@DeleteMapping("/mdstore/{mdId}")
public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId)
throws MDStoreManagerException {
service.deleteMdStore(mdId);
return StatusResponse.DELETED;
}
@Operation(summary = "Return all the versions of a mdstore")
@GetMapping("/mdstore/{mdId}/versions")
public Iterable<MDStoreVersion> listVersions(@PathVariable final String mdId) throws MDStoreManagerException {
return service.listVersions(mdId);
}
@Operation(summary = "Create a new preliminary version of a mdstore")
@GetMapping("/mdstore/{mdId}/newVersion")
public MDStoreVersion prepareNewVersion(
@Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) {
return service.prepareMdStoreVersion(mdId);
}
@Operation(summary = "Promote a preliminary version to current")
@GetMapping("/version/{versionId}/commit/{size}")
public MDStoreVersion commitVersion(
@Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId,
@Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException {
try {
return service.commitMdStoreVersion(versionId, size);
} finally {
service.deleteExpiredVersions();
}
}
@Operation(summary = "Abort a preliminary version")
@GetMapping("/version/{versionId}/abort")
public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId)
throws MDStoreManagerException {
service.deleteMdStoreVersion(versionId, true);
return StatusResponse.ABORTED;
}
@Operation(summary = "Return an existing mdstore version")
@GetMapping("/version/{versionId}")
public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId)
throws MDStoreManagerException {
return service.findVersion(versionId);
}
@Operation(summary = "Delete a mdstore version")
@DeleteMapping("/version/{versionId}")
public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId,
@Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force)
throws MDStoreManagerException {
service.deleteMdStoreVersion(versionId, force);
return StatusResponse.DELETED;
}
@Operation(summary = "Decrease the read count of a mdstore version")
@GetMapping("/version/{versionId}/endReading")
public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId)
throws MDStoreManagerException {
return service.endReading(versionId);
}
@Operation(summary = "Reset the read count of a mdstore version")
@GetMapping("/version/{versionId}/resetReading")
public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId)
throws MDStoreManagerException {
return service.resetReading(versionId);
}
}

54
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreAjaxController.java Normal file
View File

@ -0,0 +1,54 @@
package eu.dnetlib.data.mdstore;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.errors.MDStoreManagerException;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/ajax/mdstores")
@Tag(name = "Metadata Stores")
public class MDStoreAjaxController extends AbstractMDStoreController {
@Value("${dhp.mdstore-manager.hadoop.cluster}")
private String hadoopCluster;
@Value("${dhp.mdstore-manager.hadoop.user}")
private String hadoopUser;
@Value("${dhp.mdstore-manager.hdfs.base-path}")
private String hdfsBasePath;
@Operation(summary = "Show informations")
@GetMapping("/info")
public Map<String, Object> info() {
final Map<String, Object> info = new LinkedHashMap<>();
info.put("number_of_mdstores", service.countMdStores());
info.put("hadoop_user", hadoopUser);
info.put("hadoop_cluster", hadoopCluster);
info.put("hdfs_base_path", hdfsBasePath);
info.put("expired_versions", service.listExpiredVersions());
return info;
}
@Operation(summary = "read the parquet file of a mdstore version")
@GetMapping("/version/{versionId}/parquet/content/{limit}")
public List<Map<String, String>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
return service.listVersionParquet(versionId, limit);
}
@Operation(summary = "read the parquet file of a mdstore (current version)")
@GetMapping("/mdstore/{mdId}/parquet/content/{limit}")
public List<Map<String, String>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
return service.listMdstoreParquet(mdId, limit);
}
}

50
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java Normal file
View File

@ -0,0 +1,50 @@
package eu.dnetlib.data.mdstore;
import java.util.List;
import java.util.Set;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.errors.MDStoreManagerException;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/api/mdstores")
@Tag(name = "Metadata Stores")
public class MDStoreApiController extends AbstractMDStoreController {
@Operation(summary = "Return all the mdstore identifiers")
@GetMapping("/ids")
public List<String> findIdentifiers() {
return service.listMdStoreIDs();
}
@Operation(summary = "Fix the inconsistencies on HDFS")
@GetMapping("/hdfs/inconsistencies")
public Set<String> fixHdfsInconsistencies(
@Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete)
throws MDStoreManagerException {
return service.fixHdfsInconsistencies(delete);
}
@Operation(summary = "Delete expired versions")
@DeleteMapping("/versions/expired")
public StatusResponse deleteExpiredVersions() {
return service.deleteExpiredVersions();
}
@Operation(summary = "list the file inside the path of a mdstore version")
@GetMapping("/version/{versionId}/parquet/files")
public Set<String> listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException {
return service.listVersionFiles(versionId);
}
}

231
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreController.java
View File

@ -1,231 +0,0 @@
package eu.dnetlib.data.mdstore;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Sets;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/mdstores")
@Tag(name = "Metadata Stores")
public class MDStoreController extends AbstractDnetController {
@Autowired
private MDStoreService databaseUtils;
@Autowired
private HdfsClient hdfsClient;
private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
@Operation(summary = "Return all the mdstores")
@GetMapping("/")
public Iterable<MDStoreWithInfo> find() {
return databaseUtils.listMdStores();
}
@Operation(summary = "Return all the mdstore identifiers")
@GetMapping("/ids")
public List<String> findIdentifiers() {
return databaseUtils.listMdStoreIDs();
}
@Operation(summary = "Return a mdstores by id")
@GetMapping("/mdstore/{mdId}")
public MDStoreWithInfo getMdStore(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
return databaseUtils.findMdStore(mdId);
}
@Operation(summary = "Increase the read count of the current mdstore")
@GetMapping("/mdstore/{mdId}/startReading")
public MDStoreVersion startReading(@Parameter(name = "the mdstore identifier") @PathVariable final String mdId) throws MDStoreManagerException {
return databaseUtils.startReading(mdId);
}
@Operation(summary = "Create a new mdstore")
@GetMapping("/new/{format}/{layout}/{interpretation}")
public MDStoreWithInfo createMDStore(
@Parameter(name = "mdstore format") @PathVariable final String format,
@Parameter(name = "mdstore layout") @PathVariable final String layout,
@Parameter(name = "mdstore interpretation") @PathVariable final String interpretation,
@Parameter(name = "datasource name") @RequestParam(required = true) final String dsName,
@Parameter(name = "datasource id") @RequestParam(required = true) final String dsId,
@Parameter(name = "api id") @RequestParam(required = true) final String apiId) throws MDStoreManagerException {
final String id = databaseUtils.createMDStore(format, layout, interpretation, dsName, dsId, apiId);
return databaseUtils.findMdStore(id);
}
@Operation(summary = "Delete a mdstore by id")
@DeleteMapping("/mdstore/{mdId}")
public StatusResponse delete(@Parameter(name = "the id of the mdstore that will be deleted") @PathVariable final String mdId) throws MDStoreManagerException {
final String hdfsPath = databaseUtils.deleteMdStore(mdId);
hdfsClient.deletePath(hdfsPath);
return StatusResponse.DELETED;
}
@Operation(summary = "Return all the versions of a mdstore")
@GetMapping("/mdstore/{mdId}/versions")
public Iterable<MDStoreVersion> listVersions(@PathVariable final String mdId) throws MDStoreManagerException {
return databaseUtils.listVersions(mdId);
}
@Operation(summary = "Create a new preliminary version of a mdstore")
@GetMapping("/mdstore/{mdId}/newVersion")
public MDStoreVersion prepareNewVersion(@Parameter(name = "the id of the mdstore for which will be created a new version") @PathVariable final String mdId) {
return databaseUtils.prepareMdStoreVersion(mdId);
}
@Operation(summary = "Promote a preliminary version to current")
@GetMapping("/version/{versionId}/commit/{size}")
public MDStoreVersion commitVersion(@Parameter(name = "the id of the version that will be promoted to the current version") @PathVariable final String versionId,
@Parameter(name = "the size of the new current mdstore") @PathVariable final long size) throws MDStoreManagerException {
try {
return databaseUtils.commitMdStoreVersion(versionId, size);
} finally {
deleteExpiredVersions();
}
}
@Operation(summary = "Abort a preliminary version")
@GetMapping("/version/{versionId}/abort")
public StatusResponse commitVersion(@Parameter(name = "the id of the version to abort") @PathVariable final String versionId) throws MDStoreManagerException {
final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true);
hdfsClient.deletePath(hdfsPath);
return StatusResponse.ABORTED;
}
@Operation(summary = "Return an existing mdstore version")
@GetMapping("/version/{versionId}")
public MDStoreVersion getVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId)
throws MDStoreManagerException {
return databaseUtils.findVersion(versionId);
}
@Operation(summary = "Delete a mdstore version")
@DeleteMapping("/version/{versionId}")
public StatusResponse deleteVersion(@Parameter(name = "the id of the version that has to be deleted") @PathVariable final String versionId,
@Parameter(name = "if true, the controls on writing and readcount values will be skipped") @RequestParam(required = false, defaultValue = "false") final boolean force)
throws MDStoreManagerException {
final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, force);
hdfsClient.deletePath(hdfsPath);
return StatusResponse.DELETED;
}
@Operation(summary = "Decrease the read count of a mdstore version")
@GetMapping("/version/{versionId}/endReading")
public MDStoreVersion endReading(@Parameter(name = "the id of the version that has been completely read") @PathVariable final String versionId)
throws MDStoreManagerException {
return databaseUtils.endReading(versionId);
}
@Operation(summary = "Reset the read count of a mdstore version")
@GetMapping("/version/{versionId}/resetReading")
public MDStoreVersion resetReading(@Parameter(name = "the id of the version") @PathVariable final String versionId)
throws MDStoreManagerException {
return databaseUtils.resetReading(versionId);
}
@Operation(summary = "Delete expired versions")
@DeleteMapping("/versions/expired")
public StatusResponse deleteExpiredVersions() {
new Thread(this::performDeleteOfExpiredVersions).start();
return StatusResponse.DELETING;
}
private synchronized void performDeleteOfExpiredVersions() {
log.info("Deleting expired version...");
for (final String versionId : databaseUtils.listExpiredVersions()) {
try {
final String hdfsPath = databaseUtils.deleteMdStoreVersion(versionId, true);
hdfsClient.deletePath(hdfsPath);
} catch (final MDStoreManagerException e) {
log.warn("Error deleteting version " + versionId, e);
}
}
log.info("Done.");
}
@Operation(summary = "Fix the inconsistencies on HDFS")
@GetMapping("/hdfs/inconsistencies")
public Set<String> fixHdfsInconsistencies(
@Parameter(name = "force the deletion of hdfs paths") @RequestParam(required = false, defaultValue = "false") final boolean delete)
throws MDStoreManagerException {
final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
final Set<String> validDirs = databaseUtils.listValidHdfsPaths();
final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
log.info("Found " + toDelete.size() + " hdfs paths to remove");
if (delete) {
for (final String p : toDelete) {
hdfsClient.deletePath(p);
}
}
return toDelete;
}
@Operation(summary = "Show informations")
@GetMapping("/info")
public Map<String, Object> info() {
final Map<String, Object> info = new LinkedHashMap<>();
info.put("number_of_mdstores", databaseUtils.countMdStores());
info.put("hadoop_user", hdfsClient.getHadoopUser());
info.put("hadoop_cluster", hdfsClient.getHadoopCluster());
info.put("hdfs_base_path", databaseUtils.getHdfsBasePath());
info.put("expired_versions", databaseUtils.listExpiredVersions());
return info;
}
@Operation(summary = "list the file inside the path of a mdstore version")
@GetMapping("/version/{versionId}/parquet/files")
public Set<String> listVersionFiles(@PathVariable final String versionId) throws MDStoreManagerException {
final String path = databaseUtils.findVersion(versionId).getHdfsPath();
return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
}
@Operation(summary = "read the parquet file of a mdstore version")
@GetMapping("/version/{versionId}/parquet/content/{limit}")
public List<Map<String, String>> listVersionParquet(@PathVariable final String versionId, @PathVariable final long limit) throws MDStoreManagerException {
final String path = databaseUtils.findVersion(versionId).getHdfsPath();
return hdfsClient.readParquetFiles(path + "/store", limit);
}
@Operation(summary = "read the parquet file of a mdstore (current version)")
@GetMapping("/mdstore/{mdId}/parquet/content/{limit}")
public List<Map<String, String>> listMdstoreParquet(@PathVariable final String mdId, @PathVariable final long limit) throws MDStoreManagerException {
final String versionId = databaseUtils.findMdStore(mdId).getCurrentVersion();
final String path = databaseUtils.findVersion(versionId).getHdfsPath();
return hdfsClient.readParquetFiles(path + "/store", limit);
}
protected void setDatabaseUtils(final MDStoreService databaseUtils) {
this.databaseUtils = databaseUtils;
}
protected void setHdfsClient(final HdfsClient hdfsClient) {
this.hdfsClient = hdfsClient;
}
}

60
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.data.mdstore;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -16,6 +17,9 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository;
import eu.dnetlib.data.mdstore.repository.MDStoreRepository;
import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository;
@ -39,6 +43,8 @@ public class MDStoreService {
private MDStoreWithInfoRepository mdstoreWithInfoRepository;
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
protected HdfsClient hdfsClient;
@Value("${dhp.mdstore-manager.hdfs.base-path}")
private String hdfsBasePath;
@ -94,7 +100,7 @@ public class MDStoreService {
}
@Transactional
public String deleteMdStore(final String mdId) throws MDStoreManagerException {
public void deleteMdStore(final String mdId) throws MDStoreManagerException {
final Optional<MDStore> md = mdstoreRepository.findById(mdId);
@ -117,7 +123,7 @@ public class MDStoreService {
mdstoreVersionRepository.deleteByMdstore(mdId);
mdstoreRepository.deleteById(mdId);
return md.get().getHdfsPath();
hdfsClient.deletePath(md.get().getHdfsPath());
}
@Transactional
@ -164,8 +170,25 @@ public class MDStoreService {
return v;
}
public StatusResponse deleteExpiredVersions() {
new Thread(this::performDeleteOfExpiredVersions).start();
return StatusResponse.DELETING;
}
private synchronized void performDeleteOfExpiredVersions() {
log.info("Deleting expired version...");
for (final String versionId : listExpiredVersions()) {
try {
deleteMdStoreVersion(versionId, true);
} catch (final MDStoreManagerException e) {
log.warn("Error deleteting version " + versionId, e);
}
}
log.info("Done.");
}
@Transactional
public String deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException {
public void deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
@ -181,20 +204,37 @@ public class MDStoreService {
mdstoreVersionRepository.delete(v);
return v.getHdfsPath();
hdfsClient.deletePath(v.getHdfsPath());
}
public Set<String> listValidHdfsPaths() {
return new HashSet<>(jdbcTemplate
public Set<String> fixHdfsInconsistencies(final boolean delete) throws MDStoreManagerException {
final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
final Set<String> validDirs = new HashSet<>(jdbcTemplate
.queryForList("select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class));
final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
log.info("Found " + toDelete.size() + " hdfs paths to remove");
if (delete) {
for (final String p : toDelete) {
hdfsClient.deletePath(p);
}
}
return toDelete;
}
public String getHdfsBasePath() {
return hdfsBasePath;
public List<Map<String, String>> listMdstoreParquet(final String mdId, final long limit) throws MDStoreManagerException {
return listVersionParquet(findMdStore(mdId).getCurrentVersion(), limit);
}
public void setHdfsBasePath(final String hdfsBasePath) {
this.hdfsBasePath = hdfsBasePath;
public List<Map<String, String>> listVersionParquet(final String versionId, final long limit) throws MDStoreManagerException {
final String path = findVersion(versionId).getHdfsPath();
return hdfsClient.readParquetFiles(path + "/store", limit);
}
public Set<String> listVersionFiles(final String versionId) throws MDStoreManagerException {
final String path = findVersion(versionId).getHdfsPath();
return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
}
}

6
frontends/dnet-is-application/src/app/app.module.ts
View File

@ -34,7 +34,7 @@ import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent, DsmAddApiDial
import { MatPaginatorModule } from '@angular/material/paginator';
import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
import { SpinnerHttpInterceptor } from './common/spinner.service';
import { MdstoresComponent, MdstoreInspectorComponent } from './mdstores/mdstores.component';
import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component';
@NgModule({
declarations: [
@ -62,7 +62,9 @@ import { MdstoresComponent, MdstoreInspectorComponent } from './mdstores/mdstore
DsmAddApiDialog,
DsmBrowseDialog,
MdstoresComponent,
MdstoreInspectorComponent
MdstoreInspectorComponent,
MDStoreVersionsDialog,
AddMDStoreDialog
],
imports: [
BrowserModule,

26
frontends/dnet-is-application/src/app/common/is.model.ts
View File

@ -162,3 +162,29 @@ export interface DsmConf {
contentDescTypes: string[],
protocols: Protocol[]
}
export interface MDStore {
id: string,
format: string,
layout: string,
interpretation: string,
datasourceName: string,
datasourceId: string,
apiId: string,
currentVersion: string,
creationDate: string,
lastUpdate: string,
size: number,
numberOfVersions: number,
hdfsPath: string
}
export interface MDStoreVersion {
id: string,
mdstore: string,
writing: boolean,
readCount:number,
lastUpdate:string,
siz: number,
hdfsPath: string;
}

43
frontends/dnet-is-application/src/app/common/is.service.ts
View File

@ -1,6 +1,6 @@
import { Injectable } from '@angular/core';
import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http';
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource } from './is.model';
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion } from './is.model';
import { FormGroup } from '@angular/forms';
import { MatSnackBar } from '@angular/material/snack-bar';
@ -220,6 +220,47 @@ export class ISService {
});
}
loadMDStores(onSuccess: Function): void {
this.client.get<MDStore[]>("/ajax/mdstores/").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams()
.set('dsName', dsName)
.set('dsId', dsId)
.set('apiId', apiId);
this.client.post<void>('/ajax/mdstores/new/'
+ encodeURIComponent(format)
+ '/'
+ encodeURIComponent(layout)
+ '/'
+ encodeURIComponent(interpretation),
body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
deleteMDStore(mdId: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
prepareNewMDStoreVersion(mdId: string, onSuccess: Function): void {
this.client.get<MDStoreVersion>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/newVersion').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
private showError(error: any, form?: FormGroup) {
const msg = this.errorMessage(error);
if (form) {

50
frontends/dnet-is-application/src/app/mdstores/add-mdstore-dialog.html Normal file
View File

@ -0,0 +1,50 @@
<form [formGroup]="newMdstoreForm" (ngSubmit)="onSubmit()">
<h1 mat-dialog-title>New MDStore</h1>
<div mat-dialog-content>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Format</mat-label>
<input matInput formControlName="format" />
<mat-error *ngIf="newMdstoreForm.get('format')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Layout</mat-label>
<input matInput formControlName="layout" />
<mat-error *ngIf="newMdstoreForm.get('layout')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Interpretation</mat-label>
<input matInput formControlName="interpretation" />
<mat-error *ngIf="newMdstoreForm.get('interpretation')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Datasource Name</mat-label>
<input matInput formControlName="dsName" />
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Datasource ID</mat-label>
<input matInput formControlName="dsId" />
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>API ID</mat-label>
<input matInput formControlName="apiId" />
</mat-form-field>
</div>
<div mat-dialog-actions>
<button mat-stroked-button color="primary" type="submit" [disabled]="!newMdstoreForm.valid">Submit</button>
<button mat-stroked-button color="primary" mat-dialog-close>Close</button>
<mat-error *ngIf="newMdstoreForm.errors?.['serverError']">
{{ newMdstoreForm.errors?.['serverError'] }}
</mat-error>
</div>
</form>

1
frontends/dnet-is-application/src/app/mdstores/mdstores-versions-dialog.html Normal file
View File

@ -0,0 +1 @@
VERSIONS

25
frontends/dnet-is-application/src/app/mdstores/mdstores.component.css
View File

@ -0,0 +1,25 @@
.mdstore-info-table {
margin-top: 1em;
margin-bottom: 1em;
border-collapse: collapse;
}
.mdstore-info-table tr:not(:last-child) {
border-bottom: 1pt solid lightgrey;
}
.mdstore-info-table th {
width: 30%;
}
.mdstore-info-table th, .mdstore-info-table td{
text-align: left;
font-size: 0.9em;
vertical-align: top;
}
.mdstore-info-table td button {
font-size: 0.8em !important;
padding: 0 !important;
height: 2.5em !important;
}

59
frontends/dnet-is-application/src/app/mdstores/mdstores.component.html
View File

@ -1 +1,60 @@
<h2>Metadata Stores</h2>
<button mat-stroked-button color="primary" (click)="openAddMdstoreDialog()">create a new mdstore</button>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%; margin-top: 10px;">
<mat-label><b>Filter</b> (Total: {{(mdstores | searchFilter: searchText).length}})</mat-label>
<input matInput [(ngModel)]="searchText" placeholder="Filter..." autofocus />
</mat-form-field>
<mat-card *ngFor="let md of mdstores | searchFilter: searchText" style="margin-top: 10px;">
<mat-card-header>
<mat-card-title>{{md.id}}</mat-card-title>
<mat-card-subtitle>{{md.datasourceName}}</mat-card-subtitle>
</mat-card-header>
<mat-card-content>
<table class="mdstore-info-table">
<tr>
<th>Format / Layout / Interpretation</th>
<td>{{md.format}} / {{md.layout}} / {{md.interpretation}}</td>
</tr>
<tr>
<th>Datasource</th>
<td>
<b>Name:</b> {{md.datasourceName}}<br />
<b>ID:</b> {{md.datasourceId}}<br />
<b>API:</b> {{md.apiId}}
</td>
</tr>
<tr>
<th>Creation Date</th>
<td>{{md.creationDate}}</td>
</tr>
<tr>
<th>Last Update</th>
<td>{{md.lastUpdate}}</td>
</tr>
<tr>
<th>Size</th>
<td>{{md.size}}</td>
</tr>
<tr>
<th>HDFS Path</th>
<td>{{md.hdfsPath}}</td>
</tr>
<tr>
<th>Versions</th>
<td>
<a (click)="openVersionsDialog(md)">{{md.numberOfVersions}} version(s)</a>
/
<a (click)="createNewVersion(md)">prepare new version</a>
</td>
</tr>
</table>
</mat-card-content>
<mat-card-actions>
<a [routerLink]="['/mdstore/' + md.id + '/100']" mat-stroked-button color="primary">inspect</a>
<button mat-stroked-button color="warn" (click)="deleteMdstore(md)">delete</button>
<button mat-stroked-button color="info">zeppelin</button>
</mat-card-actions>
</mat-card>

103
frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts
View File

@ -1,12 +1,59 @@
import { Component } from '@angular/core';
import { Component, Inject, OnInit } from '@angular/core';
import { ISService } from '../common/is.service';
import { ActivatedRoute } from '@angular/router';
import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
import { MDStore, MDStoreVersion } from '../common/is.model';
import { FormControl, FormGroup, Validators } from '@angular/forms';
@Component({
selector: 'app-mdstores',
templateUrl: './mdstores.component.html',
styleUrls: ['./mdstores.component.css']
})
export class MdstoresComponent {
export class MdstoresComponent implements OnInit {
mdstores:MDStore[] = [];
searchText:string = '';
constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) {}
ngOnInit() { this.reload() }
reload() { this.service.loadMDStores((data: MDStore[]) => this.mdstores = data); }
openVersionsDialog(md:MDStore): void {
const dialogRef = this.dialog.open(MDStoreVersionsDialog, {
data: md,
width: '80%'
});
dialogRef.afterClosed().subscribe(result => {
if (result) this.reload();
});
}
openAddMdstoreDialog(): void {
const dialogRef = this.dialog.open(AddMDStoreDialog, {
data: {},
width: '80%'
});
dialogRef.afterClosed().subscribe(result => {
if (result) this.reload();
});
}
createNewVersion(md:MDStore): void {
this.service.prepareNewMDStoreVersion(md.id, (data:MDStoreVersion) => {
md.numberOfVersions = md.numberOfVersions + 1;
this.openVersionsDialog(md);
});
}
deleteMdstore(md:MDStore) {
if (confirm('Are you sure?')) {
this.service.deleteMDStore(md.id, (data: void) => this.reload());
}
}
}
@Component({
@ -17,3 +64,55 @@ export class MdstoresComponent {
export class MdstoreInspectorComponent {
}
@Component({
selector: 'mdstores-versions-dialog',
templateUrl: './mdstores-versions-dialog.html',
styleUrls: ['./mdstores.component.css']
})
export class MDStoreVersionsDialog {
constructor(public dialogRef: MatDialogRef<MDStoreVersionsDialog>, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) {
}
onNoClick(): void {
this.dialogRef.close();
}
}
@Component({
selector: 'add-mdstore-dialog',
templateUrl: './add-mdstore-dialog.html',
styleUrls: ['./mdstores.component.css']
})
export class AddMDStoreDialog {
newMdstoreForm = new FormGroup({
format: new FormControl('', [Validators.required]),
layout : new FormControl('', [Validators.required]),
interpretation : new FormControl('', [Validators.required]),
dsName : new FormControl(''),
dsId : new FormControl(''),
apiId : new FormControl(''),
});
constructor(public dialogRef: MatDialogRef<MDStoreVersionsDialog>, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) {
}
onSubmit():void {
let format:string = this.newMdstoreForm.get('format')?.value!;
let layout:string = this.newMdstoreForm.get('layout')?.value!;
let interpretation:string = this.newMdstoreForm.get('interpretation')?.value!;
let dsName:string = this.newMdstoreForm.get('dsName')?.value!;
let dsId:string = this.newMdstoreForm.get('dsId')?.value!;
let apiId:string = this.newMdstoreForm.get('apiId')?.value!;
this.service.addMDStore(format, layout, interpretation, dsName, dsId, apiId, (data: void) => this.dialogRef.close(1), this.newMdstoreForm);
}
onNoClick(): void {
this.dialogRef.close();
}
}

36
frontends/dnet-is-application/src/styles.css
View File

@ -8,6 +8,23 @@ body { margin: 0; font-family: Roboto, "Helvetica Neue", sans-serif; }
table {
width: 100%;
table-layout: fixed !important;
}
tr {
height: auto !important;
}
th.mat-sort-header-sorted { color: black !important; }
th, td {
white-space: nowrap !important;
overflow: hidden !important;
text-overflow: ellipsis !important;
font-size: 0.9em !important;
padding-top: 0.5em !important;
padding-bottom: 0.5em !important;
text-align: left;
}
.table-buttons { text-align: right !important; }
@ -39,25 +56,6 @@ a:hover, a:not([href]):hover {
text-decoration: underline;
}
table {
table-layout: fixed !important;
}
tr {
height: auto !important;
}
th.mat-sort-header-sorted { color: black !important; }
th, td {
white-space: nowrap !important;
overflow: hidden !important;
text-overflow: ellipsis !important;
font-size: 0.9em !important;
padding-top: 0.5em !important;
padding-bottom: 0.5em !important;
}
.badge-label {
padding-top: 0.3em;
padding-bottom: 0.3em;