mdstore inspector

This commit is contained in:
Michele Artini 2023-02-08 15:12:39 +01:00
parent c9c0ac00b8
commit 5d861c406a
7 changed files with 548 additions and 484 deletions

View File

@ -1,74 +0,0 @@
package eu.dnetlib.data.mdstore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
@Controller
public class MDInspectorController {
@Autowired
private MDStoreService service;
private static final Logger log = LoggerFactory.getLogger(MDInspectorController.class);
@Value("${dhp.mdstore-manager.inspector.records.max}")
private Long MAX_MD_RECORDS;
@RequestMapping("/mdrecords/{id}/{limit}")
public String mdstoreInspector(final ModelMap map, @PathVariable final String id, @PathVariable final long limit) throws MDStoreManagerException {
final MDStoreWithInfo md;
final MDStoreVersion ver;
if (isMdstoreId(id)) {
log.debug("MDSTORE: " + id);
md = service.findMdStore(id);
ver = service.findVersion(md.getCurrentVersion());
} else {
log.debug("VERSION: " + id);
ver = service.findVersion(id);
md = service.findMdStore(ver.getMdstore());
}
map.addAttribute("mdId", md.getId());
map.addAttribute("versionId", ver.getId());
map.addAttribute("dsId", md.getDatasourceId());
map.addAttribute("dsName", md.getDatasourceName());
map.addAttribute("apiId", md.getApiId());
map.addAttribute("format", md.getFormat());
map.addAttribute("layout", md.getLayout());
map.addAttribute("interpretation", md.getInterpretation());
map.addAttribute("path", ver.getHdfsPath());
map.addAttribute("lastUpdate", ver.getLastUpdate());
map.addAttribute("size", ver.getSize());
map.addAttribute("limit", Math.min(limit, MAX_MD_RECORDS));
if (md.getCurrentVersion().equals(ver.getId())) {
map.addAttribute("status", "current");
} else if (ver.isWriting()) {
map.addAttribute("status", "writing");
} else {
map.addAttribute("status", "expired");
}
return "inspector";
}
private boolean isMdstoreId(final String id) {
return id.length() < 40;
}
}

View File

@ -55,9 +55,7 @@ dhp.mdstore-manager.hadoop.user = dnet.dev
dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org
dhp.mdstore-manager.hadoop.zeppelin.login =
dhp.mdstore-manager.hadoop.zeppelin.password =
dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
dhp.mdstore-manager.inspector.records.max = 1000

View File

@ -1,190 +1,200 @@
export interface ResourceType {
id: string
name: string
contentType: string
count: number
simple: boolean
id: string
name: string
contentType: string
count: number
simple: boolean
}
export interface KeyValue {
k: string;
v: string;
k: string;
v: string;
}
export interface BrowseTerm {
term: string,
name: string,
total: number
term: string,
name: string,
total: number
}
export interface Module {
group: string;
name: string;
versions: string[];
files: string[];
group: string;
name: string;
versions: string[];
files: string[];
}
export interface ProtocolParam {
name: string
label: string
type: string
optional: boolean
hasSelFunction: boolean
name: string
label: string
type: string
optional: boolean
hasSelFunction: boolean
}
export interface Protocol {
id: string
params: ProtocolParam[]
id: string
params: ProtocolParam[]
}
export interface WfHistoryEntry {
processId: string,
name: string,
family: string,
status: string,
startDate: string,
endDate: string,
dsId?: string,
dsName?: string,
dsApi?: string,
details: Map<string, string>
processId: string,
name: string,
family: string,
status: string,
startDate: string,
endDate: string,
dsId?: string,
dsName?: string,
dsApi?: string,
details: Map<string, string>
}
export interface SimpleResource {
id: string,
name: string,
type: string,
description?: string,
creationDate?: string,
modificationDate?: string
id: string,
name: string,
type: string,
description?: string,
creationDate?: string,
modificationDate?: string
}
export interface ContextParam {
name: string,
value: string
name: string,
value: string
}
export interface Context {
id: string,
label: string,
parameters: ContextParam[],
nChilds: number,
type: string
id: string,
label: string,
parameters: ContextParam[],
nChilds: number,
type: string
}
export interface ContextNode {
id: string,
label: string,
parameters: ContextParam[],
nChilds: number,
claim: boolean,
parent: string,
populated?: boolean,
childs?: ContextNode[]
id: string,
label: string,
parameters: ContextParam[],
nChilds: number,
claim: boolean,
parent: string,
populated?: boolean,
childs?: ContextNode[]
}
export interface Vocabulary {
id: string,
name: string,
description?: string
id: string,
name: string,
description?: string
}
export interface VocabularyTermSynonym {
term: string,
encoding: string
term: string,
encoding: string
}
export interface VocabularyTerm {
code: string,
vocabulary: string,
name: string,
encoding: string,
synonyms: VocabularyTermSynonym[]
code: string,
vocabulary: string,
name: string,
encoding: string,
synonyms: VocabularyTermSynonym[]
}
export interface Api {
id: string,
protocol: string,
compliance: string,
active: boolean,
aggrDate: string,
aggrTotal: number
id: string,
protocol: string,
compliance: string,
active: boolean,
aggrDate: string,
aggrTotal: number
}
export interface ApiParam {
param:string,
value:string
param: string,
value: string
}
export interface ApiInsert {
id: string,
protocol: string,
datasource: string,
contentdescription: string,
removable: boolean,
compatibility: string,
metadataIdentifierPath: string,
baseurl: string,
apiParams: ApiParam[]
id: string,
protocol: string,
datasource: string,
contentdescription: string,
removable: boolean,
compatibility: string,
metadataIdentifierPath: string,
baseurl: string,
apiParams: ApiParam[]
};
export interface Organization {
name:string,
country:string
name: string,
country: string
}
export interface Datasource {
id: string,
name: string,
otherName?: string,
nsprefix: string,
websiteUrl?: string,
type: string,
consenttermsofuse?: boolean,
fulltextdownload?: boolean,
collectedFrom: string,
organizations: Organization[],
apis: Api[]
id: string,
name: string,
otherName?: string,
nsprefix: string,
websiteUrl?: string,
type: string,
consenttermsofuse?: boolean,
fulltextdownload?: boolean,
collectedFrom: string,
organizations: Organization[],
apis: Api[]
}
export interface Page<T> {
content: T[],
totalPages: number,
totalElements: number,
size: number,
number: number
content: T[],
totalPages: number,
totalElements: number,
size: number,
number: number
}
export interface DsmConf {
compatibilityLevels: string[],
contentDescTypes: string[],
protocols: Protocol[]
compatibilityLevels: string[],
contentDescTypes: string[],
protocols: Protocol[]
}
export interface MDStore {
id: string,
format: string,
layout: string,
interpretation: string,
datasourceName: string,
datasourceId: string,
apiId: string,
currentVersion: string,
creationDate: string,
lastUpdate: string,
size: number,
numberOfVersions: number,
hdfsPath: string
id: string,
format: string,
layout: string,
interpretation: string,
datasourceName: string,
datasourceId: string,
apiId: string,
currentVersion: string,
creationDate: string,
lastUpdate: string,
size: number,
numberOfVersions: number,
hdfsPath: string
}
export interface MDStoreVersion {
id: string,
mdstore: string,
writing: boolean,
readCount:number,
lastUpdate:string,
size: number,
hdfsPath: string;
id: string,
mdstore: string,
writing: boolean,
readCount: number,
lastUpdate: string,
size: number,
hdfsPath: string;
}
export interface MDStoreRecord {
id: string,
originalId: string,
encoding: string,
body: string,
dateOfCollection: string,
dateOfTransformation: string,
provenance: any
}

View File

@ -1,322 +1,343 @@
import { Injectable } from '@angular/core';
import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http';
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion } from './is.model';
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord } from './is.model';
import { FormGroup } from '@angular/forms';
import { MatSnackBar } from '@angular/material/snack-bar';
@Injectable({
providedIn: 'root'
providedIn: 'root'
})
export class ISService {
constructor(public client: HttpClient, public snackBar: MatSnackBar) { }
constructor(public client: HttpClient, public snackBar: MatSnackBar) { }
loadResourceTypes(onSuccess: Function): void {
this.client.get<ResourceType[]>("/ajax/resourceTypes").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadResourceTypes(onSuccess: Function): void {
this.client.get<ResourceType[]>("/ajax/resourceTypes").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadResourceType(id: string, onSuccess: Function): void {
this.client.get<ResourceType>("/ajax/resourceTypes/" + encodeURIComponent(id)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadResourceType(id: string, onSuccess: Function): void {
this.client.get<ResourceType>("/ajax/resourceTypes/" + encodeURIComponent(id)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadInfo(onSuccess: Function): void {
this.client.get<any[]>("/ajax/info/").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadInfo(onSuccess: Function): void {
this.client.get<any[]>("/ajax/info/").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadProtocols(onSuccess: Function): void {
this.client.get<Protocol[]>("/ajax/protocols/").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadProtocols(onSuccess: Function): void {
this.client.get<Protocol[]>("/ajax/protocols/").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadSimpleResources(type: string, onSuccess: Function): void {
this.client.get<SimpleResource[]>("/ajax/resources/" + encodeURIComponent(type)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadSimpleResources(type: string, onSuccess: Function): void {
this.client.get<SimpleResource[]>("/ajax/resources/" + encodeURIComponent(type)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadSimpleResourceContent(id: any, onSuccess: Function): void {
const headers = new HttpHeaders().set('Content-Type', 'text/plain; charset=utf-8');
this.client.get<string>("/ajax/resources/" + encodeURIComponent(id) + '/content', {
headers, responseType: 'text' as 'json'
}).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadSimpleResourceContent(id: any, onSuccess: Function): void {
const headers = new HttpHeaders().set('Content-Type', 'text/plain; charset=utf-8');
this.client.get<string>("/ajax/resources/" + encodeURIComponent(id) + '/content', {
headers, responseType: 'text' as 'json'
}).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
saveSimpleResourceMedatata(res: SimpleResource, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('/ajax/resources/' + encodeURIComponent(res.id) + '/metadata', res).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
saveSimpleResourceMedatata(res: SimpleResource, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('/ajax/resources/' + encodeURIComponent(res.id) + '/metadata', res).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
saveSimpleResourceContent(id: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams().set('content', content);
this.client.post<void>('/ajax/resources/' + encodeURIComponent(id) + '/content', body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
saveSimpleResourceContent(id: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams().set('content', content);
this.client.post<void>('/ajax/resources/' + encodeURIComponent(id) + '/content', body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
addSimpleResource(name: string, type: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams()
.set('name', name)
.set('type', type)
.set('description', description)
.set('content', content);
this.client.post<void>('/ajax/resources/', body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
addSimpleResource(name: string, type: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams()
.set('name', name)
.set('type', type)
.set('description', description)
.set('content', content);
this.client.post<void>('/ajax/resources/', body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
deleteSimpleResource(resourceId: string, onSuccess: Function): void {
this.client.delete<void>('/ajax/resources/' + encodeURIComponent(resourceId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
deleteSimpleResource(resourceId: string, onSuccess: Function): void {
this.client.delete<void>('/ajax/resources/' + encodeURIComponent(resourceId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadWfHistory(total: number, from: number, to: number, onSuccess: Function): void {
let params = new HttpParams();
if (total && total > 0) { params = params.append('total', total); }
if (from && from > 0) { params = params.append('from', from); }
if (to && to > 0) { params = params.append('to', to); }
loadWfHistory(total: number, from: number, to: number, onSuccess: Function): void {
let params = new HttpParams();
if (total && total > 0) { params = params.append('total', total); }
if (from && from > 0) { params = params.append('from', from); }
if (to && to > 0) { params = params.append('to', to); }
this.client.get<WfHistoryEntry[]>('/ajax/wfs/', { params: params }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
this.client.get<WfHistoryEntry[]>('/ajax/wfs/', { params: params }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContexts(onSuccess: Function): void {
this.client.get<Context[]>('./ajax/contexts/').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContexts(onSuccess: Function): void {
this.client.get<Context[]>('./ajax/contexts/').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContext(ctxId: string, onSuccess: Function): void {
this.client.get<Context>('./ajax/contexts/' + encodeURIComponent(ctxId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContext(ctxId: string, onSuccess: Function): void {
this.client.get<Context>('./ajax/contexts/' + encodeURIComponent(ctxId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContextCategories(ctxId: string, onSuccess: Function): void {
this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(ctxId) + '/categories').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContextCategories(ctxId: string, onSuccess: Function): void {
this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(ctxId) + '/categories').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContextConcepts(level: number, nodeId: string, onSuccess: Function): void {
this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(level) + '/' + encodeURIComponent(nodeId) + '/concepts').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadContextConcepts(level: number, nodeId: string, onSuccess: Function): void {
this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(level) + '/' + encodeURIComponent(nodeId) + '/concepts').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadVocabularies(onSuccess: Function): void {
this.client.get<Vocabulary[]>('./ajax/vocs/').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadVocabularies(onSuccess: Function): void {
this.client.get<Vocabulary[]>('./ajax/vocs/').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadVocabulary(vocId: string, onSuccess: Function): void {
this.client.get<Vocabulary>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadVocabulary(vocId: string, onSuccess: Function): void {
this.client.get<Vocabulary>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadVocabularyTerms(vocId: string, onSuccess: Function): void {
this.client.get<VocabularyTerm[]>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadVocabularyTerms(vocId: string, onSuccess: Function): void {
this.client.get<VocabularyTerm[]>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
saveVocabulary(voc: Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('./ajax/vocs/', voc).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
saveVocabulary(voc: Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('./ajax/vocs/', voc).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
saveVocabularyTerm(vocId: string, term: VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms', term).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
saveVocabularyTerm(vocId: string, term: VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms', term).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
deleteVocabulary(vocId: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
deleteVocabulary(vocId: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
deleteVocabularyTerm(vocId: string, termCode: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/vocs/'
+ encodeURIComponent(vocId)
+ '/terms/'
+ encodeURIComponent(termCode)
).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
deleteVocabularyTerm(vocId: string, termCode: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/vocs/'
+ encodeURIComponent(vocId)
+ '/terms/'
+ encodeURIComponent(termCode)
).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmConf(onSuccess: Function) {
this.client.get<DsmConf>('./ajax/dsm/conf').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmConf(onSuccess: Function) {
this.client.get<DsmConf>('./ajax/dsm/conf').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmBrowsableFields(onSuccess: Function) {
this.client.get<KeyValue[]>('./ajax/dsm/browsableFields').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmBrowsableFields(onSuccess: Function) {
this.client.get<KeyValue[]>('./ajax/dsm/browsableFields').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmBrowse(field: string, onSuccess: Function) {
this.client.get<BrowseTerm[]>('./ajax/dsm/browse/' + encodeURIComponent(field)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmBrowse(field: string, onSuccess: Function) {
this.client.get<BrowseTerm[]>('./ajax/dsm/browse/' + encodeURIComponent(field)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmSearchByField(field: string, value: string, page: number, pageSize: number, onSuccess: Function) {
this.client.get<Page<Datasource>>('./ajax/dsm/searchByField/' + encodeURIComponent(field) + '/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmSearchByField(field: string, value: string, page: number, pageSize: number, onSuccess: Function) {
this.client.get<Page<Datasource>>('./ajax/dsm/searchByField/' + encodeURIComponent(field) + '/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmSearch(value: string, page: number, pageSize: number, onSuccess: Function) {
this.client.get<Page<Datasource>>('./ajax/dsm/search/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
dsmSearch(value: string, page: number, pageSize: number, onSuccess: Function) {
this.client.get<Page<Datasource>>('./ajax/dsm/search/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadMDStores(onSuccess: Function): void {
this.client.get<MDStore[]>("/ajax/mdstores/").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadMDStores(onSuccess: Function): void {
this.client.get<MDStore[]>("/ajax/mdstores/").subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
loadMDStore(mdId: string, onSuccess: Function): void {
this.client.get<MDStore>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
let body = new HttpParams()
.set('dsName', dsName)
.set('dsId', dsId)
.set('apiId', apiId);
addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
this.client.post<void>('/ajax/mdstores/new/'
+ encodeURIComponent(format)
+ '/'
+ encodeURIComponent(layout)
+ '/'
+ encodeURIComponent(interpretation),
body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
let body = new HttpParams()
.set('dsName', dsName)
.set('dsId', dsId)
.set('apiId', apiId);
deleteMDStore(mdId: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
this.client.post<void>('/ajax/mdstores/new/'
+ encodeURIComponent(format)
+ '/'
+ encodeURIComponent(layout)
+ '/'
+ encodeURIComponent(interpretation),
body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
prepareNewMDStoreVersion(mdId: string, onSuccess: Function): void {
this.client.get<MDStoreVersion>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/newVersion').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
deleteMDStore(mdId: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
commitMDStoreVersion(versionId:string, size:number, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/commit/' + size).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
prepareNewMDStoreVersion(mdId: string, onSuccess: Function): void {
this.client.get<MDStoreVersion>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/newVersion').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
abortMDStoreVersion(versionId:string, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/abort').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
commitMDStoreVersion(versionId: string, size: number, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/commit/' + size).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
deleteMDStoreVersion(versionId:string, onSuccess: Function) {
this.client.delete<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
abortMDStoreVersion(versionId: string, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/abort').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
resetReadingMDStoreVersion(versionId:string, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/resetReading').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
deleteMDStoreVersion(versionId: string, onSuccess: Function) {
this.client.delete<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadMDStoreVersions(mdId:string, onSuccess: Function): void {
this.client.get<MDStoreVersion[]>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/versions').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
resetReadingMDStoreVersion(versionId: string, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/resetReading').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
private showError(error: any, form?: FormGroup) {
const msg = this.errorMessage(error);
if (form) {
form.setErrors({ serverError: msg })
} else if (this.snackBar) {
this.snackBar.open(msg, 'ERROR', {
duration: 5000,
});
} else {
alert(msg);
}
}
loadMDStoreVersions(mdId: string, onSuccess: Function): void {
this.client.get<MDStoreVersion[]>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/versions').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
private errorMessage(error: any) {
if (error.error && error.error.message) {
return error.error.message;
} else if (error.message) {
return error.message;
} else {
return 'Generic server side error';
}
}
loadMDStoreVersion(versionId: string, onSuccess: Function): void {
this.client.get<MDStoreVersion>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void {
this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/parquet/content/' + limit).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
private showError(error: any, form?: FormGroup) {
const msg = this.errorMessage(error);
if (form) {
form.setErrors({ serverError: msg })
} else if (this.snackBar) {
this.snackBar.open(msg, 'ERROR', {
duration: 5000,
});
} else {
alert(msg);
}
}
private errorMessage(error: any) {
if (error.error && error.error.message) {
return error.error.message;
} else if (error.message) {
return error.message;
} else {
return 'Generic server side error';
}
}
}

View File

@ -1 +1,85 @@
<h2>Metadata Store Inspector</h2>
<h2>Metadata Store Inspector</h2>
<table class="mdstore-table">
<tr>
<th rowspan="3" class="col-xs-1" style="width: 10em">MdStore</th>
<th style="width: 20em">ID</th>
<td>{{mdstore?.id}}</td>
</tr>
<tr>
<th>Format / Layout / Interpretation</th>
<td>{{mdstore?.format}} / {{mdstore?.layout}} / {{mdstore?.interpretation}}</td>
</tr>
<tr>
<th>Related Datasource</th>
<td>{{mdstore?.datasourceName}}</td>
</tr>
<tr>
<th rowspan="4">Version</th>
<th>ID</th>
<td>
<span *ngIf="version?.id == mdstore?.currentVersion" class="badge-label badge-success">current</span>
<span *ngIf="version?.writing && (version?.id != mdstore?.currentVersion)"
class="badge-label badge-warning">writing</span>
<span *ngIf="!version?.writing && (version?.id != mdstore?.currentVersion)"
class=" badge-label badge-failure">expired</span>
{{version?.id}}
</td>
</tr>
<tr>
<th>Hdfs Path</th>
<td>{{version?.hdfsPath}}</td>
</tr>
<tr>
<th>Last Update</th>
<td>{{version?.lastUpdate}}</td>
</tr>
<tr>
<th>Size</th>
<td>{{version?.size}}</td>
</tr>
</table>
<br />
<h3 *ngIf="records.length > 0">The display is limited to the first {{limit}} records</h3>
<br />
<mat-card *ngFor="let rec of records" style="margin-top: 10px;">
<mat-card-header>
<mat-card-title *ngIf="!rec.id">the record is unreadable</mat-card-title>
<mat-card-title *ngIf="rec.id">{{rec.id}}</mat-card-title>
</mat-card-header>
<mat-card-content>
<p *ngIf="!rec.id">Invalid record format</p>
<table class="mdstore-table" *ngIf="rec.id">
<tr>
<th style="width: 15em;">Original Id</th>
<td>{{rec.originalId}}</td>
</tr>
<tr>
<th>Collected on</th>
<td>{{rec.dateOfCollection | date:'medium'}}</td>
</tr>
<tr>
<th>Transformed on</th>
<td>{{rec.dateOfTransformation | date:'medium'}}</td>
</tr>
<tr>
<th>Provenance</th>
<td class="small">
<span *ngIf="rec.provenance.datasourceName"><b>Datasource Name</b>:
{{rec.provenance.datasourceName}}<br /></span>
<span *ngIf="rec.provenance.datasourceId"><b>Datasource ID</b>: {{rec.provenance.datasourceId}}<br /></span>
<span *ngIf="rec.provenance.nsPrefix"><b>Prefix</b>: {{rec.provenance.nsPrefix}}<br /></span>
</td>
</tr>
<tr>
<th>Format</th>
<td>{{rec.encoding}}</td>
</tr>
</table>
<pre class="small">{{rec.body}}</pre>
</mat-card-content>
</mat-card>

View File

@ -1,33 +1,34 @@
.mdstore-table {
border-collapse: collapse;
border-collapse: collapse;
border-bottom: 1pt solid lightgrey;
}
.mdstore-table tr:not(:last-child) {
border-bottom: 1pt solid lightgrey;
.mdstore-table tr {
border-top: 1pt solid lightgrey;
}
.mdstore-table th,
.mdstore-table td {
text-align: left;
font-size: 0.9em;
vertical-align: top;
padding-left: 1em;
padding-right: 1em;
text-align: left;
font-size: 0.9em;
vertical-align: top;
padding-left: 1em;
padding-right: 1em;
}
.mdstore-table td button,
.mdstore-table td a.mdc-button {
font-size: 0.8em !important;
padding: 0 !important;
height: 2.5em !important;
font-size: 0.8em !important;
padding: 0 !important;
height: 2.5em !important;
}
.mdstore-table tr.active-row {
background-color: #daffda;
background-color: #daffda;
}
.mdstore-table mat-icon {
width: 1em;
height: 1em;
font-size: 1em;
width: 1em;
height: 1em;
font-size: 1em;
}

View File

@ -2,7 +2,7 @@ import { Component, Inject, OnInit } from '@angular/core';
import { ISService } from '../common/is.service';
import { ActivatedRoute, Router } from '@angular/router';
import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
import { MDStore, MDStoreVersion } from '../common/is.model';
import { MDStore, MDStoreRecord, MDStoreVersion } from '../common/is.model';
import { FormControl, FormGroup, Validators } from '@angular/forms';
@Component({
@ -61,8 +61,32 @@ export class MdstoresComponent implements OnInit {
templateUrl: './mdstore-inspector.component.html',
styleUrls: ['./mdstores.component.css']
})
export class MdstoreInspectorComponent {
export class MdstoreInspectorComponent implements OnInit {
mdstore?: MDStore = undefined;
version?: MDStoreVersion = undefined;
records: MDStoreRecord[] = [];
limit: number = 0;
constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) {
}
ngOnInit() {
this.route.params.subscribe(params => {
const versionId = params['versionId'];
this.limit = params['limit'];
this.service.loadMDStoreVersion(versionId, (data: MDStoreVersion) => {
this.version = data;
this.service.loadMDStore(this.version.mdstore, (data: MDStore) => {
this.mdstore = data;
this.service.loadMDStoreVersionRecords(versionId, this.limit, (data: MDStoreRecord[]) => {
this.records = data;
});
});
});
});
}
}
@Component({