D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mdstore inspector

This commit is contained in:
Michele Artini 2023-02-08 15:12:39 +01:00
parent c9c0ac00b8
commit 5d861c406a
7 changed files with 548 additions and 484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDInspectorController.java
View File

@ -1,74 +0,0 @@
package eu.dnetlib.data.mdstore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
@Controller
public class MDInspectorController {
@Autowired
private MDStoreService service;
private static final Logger log = LoggerFactory.getLogger(MDInspectorController.class);
@Value("${dhp.mdstore-manager.inspector.records.max}")
private Long MAX_MD_RECORDS;
@RequestMapping("/mdrecords/{id}/{limit}")
public String mdstoreInspector(final ModelMap map, @PathVariable final String id, @PathVariable final long limit) throws MDStoreManagerException {
final MDStoreWithInfo md;
final MDStoreVersion ver;
if (isMdstoreId(id)) {
log.debug("MDSTORE: " + id);
md = service.findMdStore(id);
ver = service.findVersion(md.getCurrentVersion());
} else {
log.debug("VERSION: " + id);
ver = service.findVersion(id);
md = service.findMdStore(ver.getMdstore());
}
map.addAttribute("mdId", md.getId());
map.addAttribute("versionId", ver.getId());
map.addAttribute("dsId", md.getDatasourceId());
map.addAttribute("dsName", md.getDatasourceName());
map.addAttribute("apiId", md.getApiId());
map.addAttribute("format", md.getFormat());
map.addAttribute("layout", md.getLayout());
map.addAttribute("interpretation", md.getInterpretation());
map.addAttribute("path", ver.getHdfsPath());
map.addAttribute("lastUpdate", ver.getLastUpdate());
map.addAttribute("size", ver.getSize());
map.addAttribute("limit", Math.min(limit, MAX_MD_RECORDS));
if (md.getCurrentVersion().equals(ver.getId())) {
map.addAttribute("status", "current");
} else if (ver.isWriting()) {
map.addAttribute("status", "writing");
} else {
map.addAttribute("status", "expired");
}
return "inspector";
}
private boolean isMdstoreId(final String id) {
return id.length() < 40;
}
}

2
apps/dnet-is-application/src/main/resources/application.properties
View File

@ -55,9 +55,7 @@ dhp.mdstore-manager.hadoop.user = dnet.dev
dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org
dhp.mdstore-manager.hadoop.zeppelin.login = dhp.mdstore-manager.hadoop.zeppelin.login =
dhp.mdstore-manager.hadoop.zeppelin.password = dhp.mdstore-manager.hadoop.zeppelin.password =
dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
dhp.mdstore-manager.inspector.records.max = 1000

250
frontends/dnet-is-application/src/app/common/is.model.ts
View File

@ -1,190 +1,200 @@
export interface ResourceType { export interface ResourceType {
id: string id: string
name: string name: string
contentType: string contentType: string
count: number count: number
simple: boolean simple: boolean
} }
export interface KeyValue { export interface KeyValue {
k: string; k: string;
v: string; v: string;
} }
export interface BrowseTerm { export interface BrowseTerm {
term: string, term: string,
name: string, name: string,
total: number total: number
} }
export interface Module { export interface Module {
group: string; group: string;
name: string; name: string;
versions: string[]; versions: string[];
files: string[]; files: string[];
} }
export interface ProtocolParam { export interface ProtocolParam {
name: string name: string
label: string label: string
type: string type: string
optional: boolean optional: boolean
hasSelFunction: boolean hasSelFunction: boolean
} }
export interface Protocol { export interface Protocol {
id: string id: string
params: ProtocolParam[] params: ProtocolParam[]
} }
export interface WfHistoryEntry { export interface WfHistoryEntry {
processId: string, processId: string,
name: string, name: string,
family: string, family: string,
status: string, status: string,
startDate: string, startDate: string,
endDate: string, endDate: string,
dsId?: string, dsId?: string,
dsName?: string, dsName?: string,
dsApi?: string, dsApi?: string,
details: Map<string, string> details: Map<string, string>
} }
export interface SimpleResource { export interface SimpleResource {
id: string, id: string,
name: string, name: string,
type: string, type: string,
description?: string, description?: string,
creationDate?: string, creationDate?: string,
modificationDate?: string modificationDate?: string
} }
export interface ContextParam { export interface ContextParam {
name: string, name: string,
value: string value: string
} }
export interface Context { export interface Context {
id: string, id: string,
label: string, label: string,
parameters: ContextParam[], parameters: ContextParam[],
nChilds: number, nChilds: number,
type: string type: string
} }
export interface ContextNode { export interface ContextNode {
id: string, id: string,
label: string, label: string,
parameters: ContextParam[], parameters: ContextParam[],
nChilds: number, nChilds: number,
claim: boolean, claim: boolean,
parent: string, parent: string,
populated?: boolean, populated?: boolean,
childs?: ContextNode[] childs?: ContextNode[]
} }
export interface Vocabulary { export interface Vocabulary {
id: string, id: string,
name: string, name: string,
description?: string description?: string
} }
export interface VocabularyTermSynonym { export interface VocabularyTermSynonym {
term: string, term: string,
encoding: string encoding: string
} }
export interface VocabularyTerm { export interface VocabularyTerm {
code: string, code: string,
vocabulary: string, vocabulary: string,
name: string, name: string,
encoding: string, encoding: string,
synonyms: VocabularyTermSynonym[] synonyms: VocabularyTermSynonym[]
} }
export interface Api { export interface Api {
id: string, id: string,
protocol: string, protocol: string,
compliance: string, compliance: string,
active: boolean, active: boolean,
aggrDate: string, aggrDate: string,
aggrTotal: number aggrTotal: number
} }
export interface ApiParam { export interface ApiParam {
param:string, param: string,
value:string value: string
} }
export interface ApiInsert { export interface ApiInsert {
id: string, id: string,
protocol: string, protocol: string,
datasource: string, datasource: string,
contentdescription: string, contentdescription: string,
removable: boolean, removable: boolean,
compatibility: string, compatibility: string,
metadataIdentifierPath: string, metadataIdentifierPath: string,
baseurl: string, baseurl: string,
apiParams: ApiParam[] apiParams: ApiParam[]
}; };
export interface Organization { export interface Organization {
name:string, name: string,
country:string country: string
} }
export interface Datasource { export interface Datasource {
id: string, id: string,
name: string, name: string,
otherName?: string, otherName?: string,
nsprefix: string, nsprefix: string,
websiteUrl?: string, websiteUrl?: string,
type: string, type: string,
consenttermsofuse?: boolean, consenttermsofuse?: boolean,
fulltextdownload?: boolean, fulltextdownload?: boolean,
collectedFrom: string, collectedFrom: string,
organizations: Organization[], organizations: Organization[],
apis: Api[] apis: Api[]
} }
export interface Page<T> { export interface Page<T> {
content: T[], content: T[],
totalPages: number, totalPages: number,
totalElements: number, totalElements: number,
size: number, size: number,
number: number number: number
} }
export interface DsmConf { export interface DsmConf {
compatibilityLevels: string[], compatibilityLevels: string[],
contentDescTypes: string[], contentDescTypes: string[],
protocols: Protocol[] protocols: Protocol[]
} }
export interface MDStore { export interface MDStore {
id: string, id: string,
format: string, format: string,
layout: string, layout: string,
interpretation: string, interpretation: string,
datasourceName: string, datasourceName: string,
datasourceId: string, datasourceId: string,
apiId: string, apiId: string,
currentVersion: string, currentVersion: string,
creationDate: string, creationDate: string,
lastUpdate: string, lastUpdate: string,
size: number, size: number,
numberOfVersions: number, numberOfVersions: number,
hdfsPath: string hdfsPath: string
} }
export interface MDStoreVersion { export interface MDStoreVersion {
id: string, id: string,
mdstore: string, mdstore: string,
writing: boolean, writing: boolean,
readCount:number, readCount: number,
lastUpdate:string, lastUpdate: string,
size: number, size: number,
hdfsPath: string; hdfsPath: string;
}
export interface MDStoreRecord {
id: string,
originalId: string,
encoding: string,
body: string,
dateOfCollection: string,
dateOfTransformation: string,
provenance: any
} }

561
frontends/dnet-is-application/src/app/common/is.service.ts
View File

@ -1,322 +1,343 @@
import { Injectable } from '@angular/core'; import { Injectable } from '@angular/core';
import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http'; import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http';
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion } from './is.model'; import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord } from './is.model';
import { FormGroup } from '@angular/forms'; import { FormGroup } from '@angular/forms';
import { MatSnackBar } from '@angular/material/snack-bar'; import { MatSnackBar } from '@angular/material/snack-bar';
@Injectable({ @Injectable({
providedIn: 'root' providedIn: 'root'
}) })
export class ISService { export class ISService {
constructor(public client: HttpClient, public snackBar: MatSnackBar) { } constructor(public client: HttpClient, public snackBar: MatSnackBar) { }
loadResourceTypes(onSuccess: Function): void { loadResourceTypes(onSuccess: Function): void {
this.client.get<ResourceType[]>("/ajax/resourceTypes").subscribe({ this.client.get<ResourceType[]>("/ajax/resourceTypes").subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadResourceType(id: string, onSuccess: Function): void { loadResourceType(id: string, onSuccess: Function): void {
this.client.get<ResourceType>("/ajax/resourceTypes/" + encodeURIComponent(id)).subscribe({ this.client.get<ResourceType>("/ajax/resourceTypes/" + encodeURIComponent(id)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadInfo(onSuccess: Function): void { loadInfo(onSuccess: Function): void {
this.client.get<any[]>("/ajax/info/").subscribe({ this.client.get<any[]>("/ajax/info/").subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadProtocols(onSuccess: Function): void { loadProtocols(onSuccess: Function): void {
this.client.get<Protocol[]>("/ajax/protocols/").subscribe({ this.client.get<Protocol[]>("/ajax/protocols/").subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadSimpleResources(type: string, onSuccess: Function): void { loadSimpleResources(type: string, onSuccess: Function): void {
this.client.get<SimpleResource[]>("/ajax/resources/" + encodeURIComponent(type)).subscribe({ this.client.get<SimpleResource[]>("/ajax/resources/" + encodeURIComponent(type)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadSimpleResourceContent(id: any, onSuccess: Function): void { loadSimpleResourceContent(id: any, onSuccess: Function): void {
const headers = new HttpHeaders().set('Content-Type', 'text/plain; charset=utf-8'); const headers = new HttpHeaders().set('Content-Type', 'text/plain; charset=utf-8');
this.client.get<string>("/ajax/resources/" + encodeURIComponent(id) + '/content', { this.client.get<string>("/ajax/resources/" + encodeURIComponent(id) + '/content', {
headers, responseType: 'text' as 'json' headers, responseType: 'text' as 'json'
}).subscribe({ }).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
saveSimpleResourceMedatata(res: SimpleResource, onSuccess: Function, relatedForm?: FormGroup): void { saveSimpleResourceMedatata(res: SimpleResource, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('/ajax/resources/' + encodeURIComponent(res.id) + '/metadata', res).subscribe({ this.client.post<void>('/ajax/resources/' + encodeURIComponent(res.id) + '/metadata', res).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error, relatedForm) error: error => this.showError(error, relatedForm)
}); });
} }
saveSimpleResourceContent(id: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void { saveSimpleResourceContent(id: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded') const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams().set('content', content); let body = new HttpParams().set('content', content);
this.client.post<void>('/ajax/resources/' + encodeURIComponent(id) + '/content', body, { headers: headers }).subscribe({ this.client.post<void>('/ajax/resources/' + encodeURIComponent(id) + '/content', body, { headers: headers }).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error, relatedForm) error: error => this.showError(error, relatedForm)
}); });
} }
addSimpleResource(name: string, type: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void { addSimpleResource(name: string, type: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded') const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams() let body = new HttpParams()
.set('name', name) .set('name', name)
.set('type', type) .set('type', type)
.set('description', description) .set('description', description)
.set('content', content); .set('content', content);
this.client.post<void>('/ajax/resources/', body, { headers: headers }).subscribe({ this.client.post<void>('/ajax/resources/', body, { headers: headers }).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error, relatedForm) error: error => this.showError(error, relatedForm)
}); });
} }
deleteSimpleResource(resourceId: string, onSuccess: Function): void { deleteSimpleResource(resourceId: string, onSuccess: Function): void {
this.client.delete<void>('/ajax/resources/' + encodeURIComponent(resourceId)).subscribe({ this.client.delete<void>('/ajax/resources/' + encodeURIComponent(resourceId)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadWfHistory(total: number, from: number, to: number, onSuccess: Function): void { loadWfHistory(total: number, from: number, to: number, onSuccess: Function): void {
let params = new HttpParams(); let params = new HttpParams();
if (total && total > 0) { params = params.append('total', total); } if (total && total > 0) { params = params.append('total', total); }
if (from && from > 0) { params = params.append('from', from); } if (from && from > 0) { params = params.append('from', from); }
if (to && to > 0) { params = params.append('to', to); } if (to && to > 0) { params = params.append('to', to); }
this.client.get<WfHistoryEntry[]>('/ajax/wfs/', { params: params }).subscribe({ this.client.get<WfHistoryEntry[]>('/ajax/wfs/', { params: params }).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadContexts(onSuccess: Function): void { loadContexts(onSuccess: Function): void {
this.client.get<Context[]>('./ajax/contexts/').subscribe({ this.client.get<Context[]>('./ajax/contexts/').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadContext(ctxId: string, onSuccess: Function): void { loadContext(ctxId: string, onSuccess: Function): void {
this.client.get<Context>('./ajax/contexts/' + encodeURIComponent(ctxId)).subscribe({ this.client.get<Context>('./ajax/contexts/' + encodeURIComponent(ctxId)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadContextCategories(ctxId: string, onSuccess: Function): void { loadContextCategories(ctxId: string, onSuccess: Function): void {
this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(ctxId) + '/categories').subscribe({ this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(ctxId) + '/categories').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadContextConcepts(level: number, nodeId: string, onSuccess: Function): void { loadContextConcepts(level: number, nodeId: string, onSuccess: Function): void {
this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(level) + '/' + encodeURIComponent(nodeId) + '/concepts').subscribe({ this.client.get<ContextNode[]>('./ajax/contexts/' + encodeURIComponent(level) + '/' + encodeURIComponent(nodeId) + '/concepts').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadVocabularies(onSuccess: Function): void { loadVocabularies(onSuccess: Function): void {
this.client.get<Vocabulary[]>('./ajax/vocs/').subscribe({ this.client.get<Vocabulary[]>('./ajax/vocs/').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadVocabulary(vocId: string, onSuccess: Function): void { loadVocabulary(vocId: string, onSuccess: Function): void {
this.client.get<Vocabulary>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({ this.client.get<Vocabulary>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadVocabularyTerms(vocId: string, onSuccess: Function): void { loadVocabularyTerms(vocId: string, onSuccess: Function): void {
this.client.get<VocabularyTerm[]>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms').subscribe({ this.client.get<VocabularyTerm[]>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
saveVocabulary(voc: Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void { saveVocabulary(voc: Vocabulary, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('./ajax/vocs/', voc).subscribe({ this.client.post<void>('./ajax/vocs/', voc).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error, relatedForm) error: error => this.showError(error, relatedForm)
}); });
} }
saveVocabularyTerm(vocId: string, term: VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void { saveVocabularyTerm(vocId: string, term: VocabularyTerm, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms', term).subscribe({ this.client.post<void>('./ajax/vocs/' + encodeURIComponent(vocId) + '/terms', term).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error, relatedForm) error: error => this.showError(error, relatedForm)
}); });
} }
deleteVocabulary(vocId: string, onSuccess: Function): void { deleteVocabulary(vocId: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({ this.client.delete<void>('./ajax/vocs/' + encodeURIComponent(vocId)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
deleteVocabularyTerm(vocId: string, termCode: string, onSuccess: Function): void { deleteVocabularyTerm(vocId: string, termCode: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/vocs/' this.client.delete<void>('./ajax/vocs/'
+ encodeURIComponent(vocId) + encodeURIComponent(vocId)
+ '/terms/' + '/terms/'
+ encodeURIComponent(termCode) + encodeURIComponent(termCode)
).subscribe({ ).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
dsmConf(onSuccess: Function) { dsmConf(onSuccess: Function) {
this.client.get<DsmConf>('./ajax/dsm/conf').subscribe({ this.client.get<DsmConf>('./ajax/dsm/conf').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
dsmBrowsableFields(onSuccess: Function) { dsmBrowsableFields(onSuccess: Function) {
this.client.get<KeyValue[]>('./ajax/dsm/browsableFields').subscribe({ this.client.get<KeyValue[]>('./ajax/dsm/browsableFields').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
dsmBrowse(field: string, onSuccess: Function) { dsmBrowse(field: string, onSuccess: Function) {
this.client.get<BrowseTerm[]>('./ajax/dsm/browse/' + encodeURIComponent(field)).subscribe({ this.client.get<BrowseTerm[]>('./ajax/dsm/browse/' + encodeURIComponent(field)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
dsmSearchByField(field: string, value: string, page: number, pageSize: number, onSuccess: Function) { dsmSearchByField(field: string, value: string, page: number, pageSize: number, onSuccess: Function) {
this.client.get<Page<Datasource>>('./ajax/dsm/searchByField/' + encodeURIComponent(field) + '/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({ this.client.get<Page<Datasource>>('./ajax/dsm/searchByField/' + encodeURIComponent(field) + '/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
dsmSearch(value: string, page: number, pageSize: number, onSuccess: Function) { dsmSearch(value: string, page: number, pageSize: number, onSuccess: Function) {
this.client.get<Page<Datasource>>('./ajax/dsm/search/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({ this.client.get<Page<Datasource>>('./ajax/dsm/search/' + page + '/' + pageSize + '?value=' + encodeURIComponent(value)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadMDStores(onSuccess: Function): void { loadMDStores(onSuccess: Function): void {
this.client.get<MDStore[]>("/ajax/mdstores/").subscribe({ this.client.get<MDStore[]>("/ajax/mdstores/").subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) { loadMDStore(mdId: string, onSuccess: Function): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded') this.client.get<MDStore>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
let body = new HttpParams() addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) {
.set('dsName', dsName) const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
.set('dsId', dsId)
.set('apiId', apiId);
this.client.post<void>('/ajax/mdstores/new/' let body = new HttpParams()
+ encodeURIComponent(format) .set('dsName', dsName)
+ '/' .set('dsId', dsId)
+ encodeURIComponent(layout) .set('apiId', apiId);
+ '/'
+ encodeURIComponent(interpretation),
body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
deleteMDStore(mdId: string, onSuccess: Function): void { this.client.post<void>('/ajax/mdstores/new/'
this.client.delete<void>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({ + encodeURIComponent(format)
next: data => onSuccess(data), + '/'
error: error => this.showError(error) + encodeURIComponent(layout)
}); + '/'
} + encodeURIComponent(interpretation),
body, { headers: headers }).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
prepareNewMDStoreVersion(mdId: string, onSuccess: Function): void { deleteMDStore(mdId: string, onSuccess: Function): void {
this.client.get<MDStoreVersion>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/newVersion').subscribe({ this.client.delete<void>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
commitMDStoreVersion(versionId:string, size:number, onSuccess: Function) { prepareNewMDStoreVersion(mdId: string, onSuccess: Function): void {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/commit/' + size).subscribe({ this.client.get<MDStoreVersion>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/newVersion').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
abortMDStoreVersion(versionId:string, onSuccess: Function) { commitMDStoreVersion(versionId: string, size: number, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/abort').subscribe({ this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/commit/' + size).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
deleteMDStoreVersion(versionId:string, onSuccess: Function) { abortMDStoreVersion(versionId: string, onSuccess: Function) {
this.client.delete<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({ this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/abort').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
resetReadingMDStoreVersion(versionId:string, onSuccess: Function) { deleteMDStoreVersion(versionId: string, onSuccess: Function) {
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/resetReading').subscribe({ this.client.delete<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
loadMDStoreVersions(mdId:string, onSuccess: Function): void { resetReadingMDStoreVersion(versionId: string, onSuccess: Function) {
this.client.get<MDStoreVersion[]>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/versions').subscribe({ this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/resetReading').subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
} }
private showError(error: any, form?: FormGroup) { loadMDStoreVersions(mdId: string, onSuccess: Function): void {
const msg = this.errorMessage(error); this.client.get<MDStoreVersion[]>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/versions').subscribe({
if (form) { next: data => onSuccess(data),
form.setErrors({ serverError: msg }) error: error => this.showError(error)
} else if (this.snackBar) { });
this.snackBar.open(msg, 'ERROR', { }
duration: 5000,
});
} else {
alert(msg);
}
}
private errorMessage(error: any) { loadMDStoreVersion(versionId: string, onSuccess: Function): void {
if (error.error && error.error.message) { this.client.get<MDStoreVersion>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
return error.error.message; next: data => onSuccess(data),
} else if (error.message) { error: error => this.showError(error)
return error.message; });
} else { }
return 'Generic server side error';
} loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void {
} this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/parquet/content/' + limit).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
private showError(error: any, form?: FormGroup) {
const msg = this.errorMessage(error);
if (form) {
form.setErrors({ serverError: msg })
} else if (this.snackBar) {
this.snackBar.open(msg, 'ERROR', {
duration: 5000,
});
} else {
alert(msg);
}
}
private errorMessage(error: any) {
if (error.error && error.error.message) {
return error.error.message;
} else if (error.message) {
return error.message;
} else {
return 'Generic server side error';
}
}
} }

84
frontends/dnet-is-application/src/app/mdstores/mdstore-inspector.component.html
View File

@ -1 +1,85 @@
<h2>Metadata Store Inspector</h2> <h2>Metadata Store Inspector</h2>
<table class="mdstore-table">
<tr>
<th rowspan="3" class="col-xs-1" style="width: 10em">MdStore</th>
<th style="width: 20em">ID</th>
<td>{{mdstore?.id}}</td>
</tr>
<tr>
<th>Format / Layout / Interpretation</th>
<td>{{mdstore?.format}} / {{mdstore?.layout}} / {{mdstore?.interpretation}}</td>
</tr>
<tr>
<th>Related Datasource</th>
<td>{{mdstore?.datasourceName}}</td>
</tr>
<tr>
<th rowspan="4">Version</th>
<th>ID</th>
<td>
<span *ngIf="version?.id == mdstore?.currentVersion" class="badge-label badge-success">current</span>
<span *ngIf="version?.writing && (version?.id != mdstore?.currentVersion)"
class="badge-label badge-warning">writing</span>
<span *ngIf="!version?.writing && (version?.id != mdstore?.currentVersion)"
class=" badge-label badge-failure">expired</span>
{{version?.id}}
</td>
</tr>
<tr>
<th>Hdfs Path</th>
<td>{{version?.hdfsPath}}</td>
</tr>
<tr>
<th>Last Update</th>
<td>{{version?.lastUpdate}}</td>
</tr>
<tr>
<th>Size</th>
<td>{{version?.size}}</td>
</tr>
</table>
<br />
<h3 *ngIf="records.length > 0">The display is limited to the first {{limit}} records</h3>
<br />
<mat-card *ngFor="let rec of records" style="margin-top: 10px;">
<mat-card-header>
<mat-card-title *ngIf="!rec.id">the record is unreadable</mat-card-title>
<mat-card-title *ngIf="rec.id">{{rec.id}}</mat-card-title>
</mat-card-header>
<mat-card-content>
<p *ngIf="!rec.id">Invalid record format</p>
<table class="mdstore-table" *ngIf="rec.id">
<tr>
<th style="width: 15em;">Original Id</th>
<td>{{rec.originalId}}</td>
</tr>
<tr>
<th>Collected on</th>
<td>{{rec.dateOfCollection | date:'medium'}}</td>
</tr>
<tr>
<th>Transformed on</th>
<td>{{rec.dateOfTransformation | date:'medium'}}</td>
</tr>
<tr>
<th>Provenance</th>
<td class="small">
<span *ngIf="rec.provenance.datasourceName"><b>Datasource Name</b>:
{{rec.provenance.datasourceName}}<br /></span>
<span *ngIf="rec.provenance.datasourceId"><b>Datasource ID</b>: {{rec.provenance.datasourceId}}<br /></span>
<span *ngIf="rec.provenance.nsPrefix"><b>Prefix</b>: {{rec.provenance.nsPrefix}}<br /></span>
</td>
</tr>
<tr>
<th>Format</th>
<td>{{rec.encoding}}</td>
</tr>
</table>
<pre class="small">{{rec.body}}</pre>
</mat-card-content>
</mat-card>

31
frontends/dnet-is-application/src/app/mdstores/mdstores.component.css
View File

@ -1,33 +1,34 @@
.mdstore-table { .mdstore-table {
border-collapse: collapse; border-collapse: collapse;
border-bottom: 1pt solid lightgrey;
} }
.mdstore-table tr:not(:last-child) { .mdstore-table tr {
border-bottom: 1pt solid lightgrey; border-top: 1pt solid lightgrey;
} }
.mdstore-table th, .mdstore-table th,
.mdstore-table td { .mdstore-table td {
text-align: left; text-align: left;
font-size: 0.9em; font-size: 0.9em;
vertical-align: top; vertical-align: top;
padding-left: 1em; padding-left: 1em;
padding-right: 1em; padding-right: 1em;
} }
.mdstore-table td button, .mdstore-table td button,
.mdstore-table td a.mdc-button { .mdstore-table td a.mdc-button {
font-size: 0.8em !important; font-size: 0.8em !important;
padding: 0 !important; padding: 0 !important;
height: 2.5em !important; height: 2.5em !important;
} }
.mdstore-table tr.active-row { .mdstore-table tr.active-row {
background-color: #daffda; background-color: #daffda;
} }
.mdstore-table mat-icon { .mdstore-table mat-icon {
width: 1em; width: 1em;
height: 1em; height: 1em;
font-size: 1em; font-size: 1em;
} }

28
frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts
View File

@ -2,7 +2,7 @@ import { Component, Inject, OnInit } from '@angular/core';
import { ISService } from '../common/is.service'; import { ISService } from '../common/is.service';
import { ActivatedRoute, Router } from '@angular/router'; import { ActivatedRoute, Router } from '@angular/router';
import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
import { MDStore, MDStoreVersion } from '../common/is.model'; import { MDStore, MDStoreRecord, MDStoreVersion } from '../common/is.model';
import { FormControl, FormGroup, Validators } from '@angular/forms'; import { FormControl, FormGroup, Validators } from '@angular/forms';
@Component({ @Component({
@ -61,8 +61,32 @@ export class MdstoresComponent implements OnInit {
templateUrl: './mdstore-inspector.component.html', templateUrl: './mdstore-inspector.component.html',
styleUrls: ['./mdstores.component.css'] styleUrls: ['./mdstores.component.css']
}) })
export class MdstoreInspectorComponent { export class MdstoreInspectorComponent implements OnInit {
mdstore?: MDStore = undefined;
version?: MDStoreVersion = undefined;
records: MDStoreRecord[] = [];
limit: number = 0;
constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) {
}
ngOnInit() {
this.route.params.subscribe(params => {
const versionId = params['versionId'];
this.limit = params['limit'];
this.service.loadMDStoreVersion(versionId, (data: MDStoreVersion) => {
this.version = data;
this.service.loadMDStore(this.version.mdstore, (data: MDStore) => {
this.mdstore = data;
this.service.loadMDStoreVersionRecords(versionId, this.limit, (data: MDStoreRecord[]) => {
this.records = data;
});
});
});
});
}
} }
@Component({ @Component({