mdstore inspector
This commit is contained in:
parent
c9c0ac00b8
commit
5d861c406a
|
|
@ -1,74 +0,0 @@
|
|||
package eu.dnetlib.data.mdstore;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.ui.ModelMap;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
|
||||
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
|
||||
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
|
||||
@Controller
|
||||
public class MDInspectorController {
|
||||
|
||||
@Autowired
|
||||
private MDStoreService service;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MDInspectorController.class);
|
||||
|
||||
@Value("${dhp.mdstore-manager.inspector.records.max}")
|
||||
private Long MAX_MD_RECORDS;
|
||||
|
||||
@RequestMapping("/mdrecords/{id}/{limit}")
|
||||
public String mdstoreInspector(final ModelMap map, @PathVariable final String id, @PathVariable final long limit) throws MDStoreManagerException {
|
||||
|
||||
final MDStoreWithInfo md;
|
||||
final MDStoreVersion ver;
|
||||
|
||||
if (isMdstoreId(id)) {
|
||||
log.debug("MDSTORE: " + id);
|
||||
md = service.findMdStore(id);
|
||||
ver = service.findVersion(md.getCurrentVersion());
|
||||
} else {
|
||||
log.debug("VERSION: " + id);
|
||||
ver = service.findVersion(id);
|
||||
md = service.findMdStore(ver.getMdstore());
|
||||
}
|
||||
|
||||
map.addAttribute("mdId", md.getId());
|
||||
map.addAttribute("versionId", ver.getId());
|
||||
|
||||
map.addAttribute("dsId", md.getDatasourceId());
|
||||
map.addAttribute("dsName", md.getDatasourceName());
|
||||
map.addAttribute("apiId", md.getApiId());
|
||||
|
||||
map.addAttribute("format", md.getFormat());
|
||||
map.addAttribute("layout", md.getLayout());
|
||||
map.addAttribute("interpretation", md.getInterpretation());
|
||||
|
||||
map.addAttribute("path", ver.getHdfsPath());
|
||||
map.addAttribute("lastUpdate", ver.getLastUpdate());
|
||||
map.addAttribute("size", ver.getSize());
|
||||
|
||||
map.addAttribute("limit", Math.min(limit, MAX_MD_RECORDS));
|
||||
|
||||
if (md.getCurrentVersion().equals(ver.getId())) {
|
||||
map.addAttribute("status", "current");
|
||||
} else if (ver.isWriting()) {
|
||||
map.addAttribute("status", "writing");
|
||||
} else {
|
||||
map.addAttribute("status", "expired");
|
||||
}
|
||||
|
||||
return "inspector";
|
||||
}
|
||||
|
||||
private boolean isMdstoreId(final String id) {
|
||||
return id.length() < 40;
|
||||
}
|
||||
}
|
||||
|
|
@ -55,9 +55,7 @@ dhp.mdstore-manager.hadoop.user = dnet.dev
|
|||
dhp.mdstore-manager.hadoop.zeppelin.base-url = https://hadoop-zeppelin.garr-pa1.d4science.org
|
||||
dhp.mdstore-manager.hadoop.zeppelin.login =
|
||||
dhp.mdstore-manager.hadoop.zeppelin.password =
|
||||
|
||||
dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
|
||||
|
||||
dhp.mdstore-manager.inspector.records.max = 1000
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -113,8 +113,8 @@ export interface Api {
|
|||
}
|
||||
|
||||
export interface ApiParam {
|
||||
param:string,
|
||||
value:string
|
||||
param: string,
|
||||
value: string
|
||||
}
|
||||
|
||||
export interface ApiInsert {
|
||||
|
|
@ -130,8 +130,8 @@ export interface ApiInsert {
|
|||
};
|
||||
|
||||
export interface Organization {
|
||||
name:string,
|
||||
country:string
|
||||
name: string,
|
||||
country: string
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -183,8 +183,18 @@ export interface MDStoreVersion {
|
|||
id: string,
|
||||
mdstore: string,
|
||||
writing: boolean,
|
||||
readCount:number,
|
||||
lastUpdate:string,
|
||||
readCount: number,
|
||||
lastUpdate: string,
|
||||
size: number,
|
||||
hdfsPath: string;
|
||||
}
|
||||
|
||||
export interface MDStoreRecord {
|
||||
id: string,
|
||||
originalId: string,
|
||||
encoding: string,
|
||||
body: string,
|
||||
dateOfCollection: string,
|
||||
dateOfTransformation: string,
|
||||
provenance: any
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { Injectable } from '@angular/core';
|
||||
import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http';
|
||||
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion } from './is.model';
|
||||
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord } from './is.model';
|
||||
import { FormGroup } from '@angular/forms';
|
||||
import { MatSnackBar } from '@angular/material/snack-bar';
|
||||
|
||||
|
|
@ -227,6 +227,13 @@ export class ISService {
|
|||
});
|
||||
}
|
||||
|
||||
loadMDStore(mdId: string, onSuccess: Function): void {
|
||||
this.client.get<MDStore>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) {
|
||||
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
|
||||
|
||||
|
|
@ -261,41 +268,55 @@ export class ISService {
|
|||
});
|
||||
}
|
||||
|
||||
commitMDStoreVersion(versionId:string, size:number, onSuccess: Function) {
|
||||
commitMDStoreVersion(versionId: string, size: number, onSuccess: Function) {
|
||||
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/commit/' + size).subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
abortMDStoreVersion(versionId:string, onSuccess: Function) {
|
||||
abortMDStoreVersion(versionId: string, onSuccess: Function) {
|
||||
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/abort').subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
deleteMDStoreVersion(versionId:string, onSuccess: Function) {
|
||||
deleteMDStoreVersion(versionId: string, onSuccess: Function) {
|
||||
this.client.delete<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
resetReadingMDStoreVersion(versionId:string, onSuccess: Function) {
|
||||
resetReadingMDStoreVersion(versionId: string, onSuccess: Function) {
|
||||
this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/resetReading').subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
loadMDStoreVersions(mdId:string, onSuccess: Function): void {
|
||||
loadMDStoreVersions(mdId: string, onSuccess: Function): void {
|
||||
this.client.get<MDStoreVersion[]>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/versions').subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
loadMDStoreVersion(versionId: string, onSuccess: Function): void {
|
||||
this.client.get<MDStoreVersion>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void {
|
||||
this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/parquet/content/' + limit).subscribe({
|
||||
next: data => onSuccess(data),
|
||||
error: error => this.showError(error)
|
||||
});
|
||||
}
|
||||
|
||||
private showError(error: any, form?: FormGroup) {
|
||||
const msg = this.errorMessage(error);
|
||||
if (form) {
|
||||
|
|
|
|||
|
|
@ -1 +1,85 @@
|
|||
<h2>Metadata Store Inspector</h2>
|
||||
|
||||
<table class="mdstore-table">
|
||||
<tr>
|
||||
<th rowspan="3" class="col-xs-1" style="width: 10em">MdStore</th>
|
||||
<th style="width: 20em">ID</th>
|
||||
<td>{{mdstore?.id}}</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<th>Format / Layout / Interpretation</th>
|
||||
<td>{{mdstore?.format}} / {{mdstore?.layout}} / {{mdstore?.interpretation}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Related Datasource</th>
|
||||
<td>{{mdstore?.datasourceName}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th rowspan="4">Version</th>
|
||||
<th>ID</th>
|
||||
<td>
|
||||
<span *ngIf="version?.id == mdstore?.currentVersion" class="badge-label badge-success">current</span>
|
||||
<span *ngIf="version?.writing && (version?.id != mdstore?.currentVersion)"
|
||||
class="badge-label badge-warning">writing</span>
|
||||
<span *ngIf="!version?.writing && (version?.id != mdstore?.currentVersion)"
|
||||
class=" badge-label badge-failure">expired</span>
|
||||
|
||||
{{version?.id}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Hdfs Path</th>
|
||||
<td>{{version?.hdfsPath}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Last Update</th>
|
||||
<td>{{version?.lastUpdate}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Size</th>
|
||||
<td>{{version?.size}}</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<br />
|
||||
<h3 *ngIf="records.length > 0">The display is limited to the first {{limit}} records</h3>
|
||||
<br />
|
||||
|
||||
<mat-card *ngFor="let rec of records" style="margin-top: 10px;">
|
||||
<mat-card-header>
|
||||
<mat-card-title *ngIf="!rec.id">the record is unreadable</mat-card-title>
|
||||
<mat-card-title *ngIf="rec.id">{{rec.id}}</mat-card-title>
|
||||
</mat-card-header>
|
||||
<mat-card-content>
|
||||
<p *ngIf="!rec.id">Invalid record format</p>
|
||||
<table class="mdstore-table" *ngIf="rec.id">
|
||||
<tr>
|
||||
<th style="width: 15em;">Original Id</th>
|
||||
<td>{{rec.originalId}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Collected on</th>
|
||||
<td>{{rec.dateOfCollection | date:'medium'}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Transformed on</th>
|
||||
<td>{{rec.dateOfTransformation | date:'medium'}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Provenance</th>
|
||||
<td class="small">
|
||||
<span *ngIf="rec.provenance.datasourceName"><b>Datasource Name</b>:
|
||||
{{rec.provenance.datasourceName}}<br /></span>
|
||||
<span *ngIf="rec.provenance.datasourceId"><b>Datasource ID</b>: {{rec.provenance.datasourceId}}<br /></span>
|
||||
<span *ngIf="rec.provenance.nsPrefix"><b>Prefix</b>: {{rec.provenance.nsPrefix}}<br /></span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Format</th>
|
||||
<td>{{rec.encoding}}</td>
|
||||
</tr>
|
||||
</table>
|
||||
<pre class="small">{{rec.body}}</pre>
|
||||
</mat-card-content>
|
||||
</mat-card>
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
.mdstore-table {
|
||||
border-collapse: collapse;
|
||||
border-bottom: 1pt solid lightgrey;
|
||||
}
|
||||
|
||||
.mdstore-table tr:not(:last-child) {
|
||||
border-bottom: 1pt solid lightgrey;
|
||||
.mdstore-table tr {
|
||||
border-top: 1pt solid lightgrey;
|
||||
}
|
||||
|
||||
.mdstore-table th,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { Component, Inject, OnInit } from '@angular/core';
|
|||
import { ISService } from '../common/is.service';
|
||||
import { ActivatedRoute, Router } from '@angular/router';
|
||||
import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
|
||||
import { MDStore, MDStoreVersion } from '../common/is.model';
|
||||
import { MDStore, MDStoreRecord, MDStoreVersion } from '../common/is.model';
|
||||
import { FormControl, FormGroup, Validators } from '@angular/forms';
|
||||
|
||||
@Component({
|
||||
|
|
@ -61,8 +61,32 @@ export class MdstoresComponent implements OnInit {
|
|||
templateUrl: './mdstore-inspector.component.html',
|
||||
styleUrls: ['./mdstores.component.css']
|
||||
})
|
||||
export class MdstoreInspectorComponent {
|
||||
export class MdstoreInspectorComponent implements OnInit {
|
||||
|
||||
mdstore?: MDStore = undefined;
|
||||
version?: MDStoreVersion = undefined;
|
||||
records: MDStoreRecord[] = [];
|
||||
|
||||
limit: number = 0;
|
||||
|
||||
constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) {
|
||||
}
|
||||
ngOnInit() {
|
||||
this.route.params.subscribe(params => {
|
||||
const versionId = params['versionId'];
|
||||
this.limit = params['limit'];
|
||||
|
||||
this.service.loadMDStoreVersion(versionId, (data: MDStoreVersion) => {
|
||||
this.version = data;
|
||||
this.service.loadMDStore(this.version.mdstore, (data: MDStore) => {
|
||||
this.mdstore = data;
|
||||
this.service.loadMDStoreVersionRecords(versionId, this.limit, (data: MDStoreRecord[]) => {
|
||||
this.records = data;
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Component({
|
||||
|
|
|
|||
Loading…
Reference in New Issue