mdstore inspector

apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDInspectorController.java

@@ -1,74 +0,0 @@
-package eu.dnetlib.data.mdstore;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Controller;
-import org.springframework.ui.ModelMap;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
-
-import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
-import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
-import eu.dnetlib.errors.MDStoreManagerException;
-
-@Controller
-public class MDInspectorController {
-
-	@Autowired
-	private MDStoreService service;
-
-	private static final Logger log = LoggerFactory.getLogger(MDInspectorController.class);
-
-	@Value("${dhp.mdstore-manager.inspector.records.max}")
-	private Long MAX_MD_RECORDS;
-
-	@RequestMapping("/mdrecords/{id}/{limit}")
-	public String mdstoreInspector(final ModelMap map, @PathVariable final String id, @PathVariable final long limit) throws MDStoreManagerException {
-
-		final MDStoreWithInfo md;
-		final MDStoreVersion ver;
-
-		if (isMdstoreId(id)) {
-			log.debug("MDSTORE: " + id);
-			md = service.findMdStore(id);
-			ver = service.findVersion(md.getCurrentVersion());
-		} else {
-			log.debug("VERSION: " + id);
-			ver = service.findVersion(id);
-			md = service.findMdStore(ver.getMdstore());
-		}
-
-		map.addAttribute("mdId", md.getId());
-		map.addAttribute("versionId", ver.getId());
-
-		map.addAttribute("dsId", md.getDatasourceId());
-		map.addAttribute("dsName", md.getDatasourceName());
-		map.addAttribute("apiId", md.getApiId());
-
-		map.addAttribute("format", md.getFormat());
-		map.addAttribute("layout", md.getLayout());
-		map.addAttribute("interpretation", md.getInterpretation());
-
-		map.addAttribute("path", ver.getHdfsPath());
-		map.addAttribute("lastUpdate", ver.getLastUpdate());
-		map.addAttribute("size", ver.getSize());
-
-		map.addAttribute("limit", Math.min(limit, MAX_MD_RECORDS));
-
-		if (md.getCurrentVersion().equals(ver.getId())) {
-			map.addAttribute("status", "current");
-		} else if (ver.isWriting()) {
-			map.addAttribute("status", "writing");
-		} else {
-			map.addAttribute("status", "expired");
-		}
-
-		return "inspector";
-	}
-
-	private boolean isMdstoreId(final String id) {
-		return id.length() < 40;
-	}
-}

apps/dnet-is-application/src/main/resources/application.properties

@@ -55,9 +55,7 @@ dhp.mdstore-manager.hadoop.user    = dnet.dev
 dhp.mdstore-manager.hadoop.zeppelin.base-url =  https://hadoop-zeppelin.garr-pa1.d4science.org
 dhp.mdstore-manager.hadoop.zeppelin.login = 
 dhp.mdstore-manager.hadoop.zeppelin.password = 
-
 dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
 
-dhp.mdstore-manager.inspector.records.max = 1000
 
 

frontends/dnet-is-application/src/app/common/is.model.ts

@@ -113,8 +113,8 @@ export interface Api {
 }
 
 export interface ApiParam {
-    param:string,
+	param: string,
-    value:string
+	value: string
 }
 
 export interface ApiInsert {
@@ -130,8 +130,8 @@ export interface ApiInsert {
 };
 
 export interface Organization {
-    name:string,
+	name: string,
-    country:string
+	country: string
 }
 
 
@@ -183,8 +183,18 @@ export interface MDStoreVersion {
 	id: string,
 	mdstore: string,
 	writing: boolean,
-    readCount:number,
+	readCount: number,
-    lastUpdate:string,
+	lastUpdate: string,
 	size: number,
 	hdfsPath: string;
 }
+
+export interface MDStoreRecord {
+	id: string,
+	originalId: string,
+	encoding: string,
+	body: string,
+	dateOfCollection: string,
+	dateOfTransformation: string,
+	provenance: any
+}

frontends/dnet-is-application/src/app/common/is.service.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http';
-import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion } from './is.model';
+import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord } from './is.model';
 import { FormGroup } from '@angular/forms';
 import { MatSnackBar } from '@angular/material/snack-bar';
 
@@ -227,6 +227,13 @@ export class ISService {
 		});
 	}
 
+	loadMDStore(mdId: string, onSuccess: Function): void {
+		this.client.get<MDStore>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId)).subscribe({
+			next: data => onSuccess(data),
+			error: error => this.showError(error)
+		});
+	}
+
 	addMDStore(format: string, layout: string, interpretation: string, dsName: string, dsId: string, apiId: string, onSuccess: Function, relatedForm?: FormGroup) {
 		const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
 
@@ -261,41 +268,55 @@ export class ISService {
 		});
 	}
 
-  commitMDStoreVersion(versionId:string, size:number, onSuccess: Function) {
+	commitMDStoreVersion(versionId: string, size: number, onSuccess: Function) {
 		this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/commit/' + size).subscribe({
 			next: data => onSuccess(data),
 			error: error => this.showError(error)
 		});
 	}
 
-  abortMDStoreVersion(versionId:string, onSuccess: Function) {
+	abortMDStoreVersion(versionId: string, onSuccess: Function) {
 		this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/abort').subscribe({
 			next: data => onSuccess(data),
 			error: error => this.showError(error)
 		});
 	}
 
-  deleteMDStoreVersion(versionId:string, onSuccess: Function) {
+	deleteMDStoreVersion(versionId: string, onSuccess: Function) {
 		this.client.delete<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
 			next: data => onSuccess(data),
 			error: error => this.showError(error)
 		});
 	}
 
-  resetReadingMDStoreVersion(versionId:string, onSuccess: Function) {
+	resetReadingMDStoreVersion(versionId: string, onSuccess: Function) {
 		this.client.get<any>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/resetReading').subscribe({
 			next: data => onSuccess(data),
 			error: error => this.showError(error)
 		});
 	}
 
-  loadMDStoreVersions(mdId:string, onSuccess: Function): void {
+	loadMDStoreVersions(mdId: string, onSuccess: Function): void {
 		this.client.get<MDStoreVersion[]>('./ajax/mdstores/mdstore/' + encodeURIComponent(mdId) + '/versions').subscribe({
 			next: data => onSuccess(data),
 			error: error => this.showError(error)
 		});
 	}
 
+	loadMDStoreVersion(versionId: string, onSuccess: Function): void {
+		this.client.get<MDStoreVersion>('./ajax/mdstores/version/' + encodeURIComponent(versionId)).subscribe({
+			next: data => onSuccess(data),
+			error: error => this.showError(error)
+		});
+	}
+
+	loadMDStoreVersionRecords(versionId: string, limit: number, onSuccess: Function): void {
+		this.client.get<MDStoreRecord[]>('./ajax/mdstores/version/' + encodeURIComponent(versionId) + '/parquet/content/' + limit).subscribe({
+			next: data => onSuccess(data),
+			error: error => this.showError(error)
+		});
+	}
+
 	private showError(error: any, form?: FormGroup) {
 		const msg = this.errorMessage(error);
 		if (form) {

frontends/dnet-is-application/src/app/mdstores/mdstore-inspector.component.html

@@ -1 +1,85 @@
 <h2>Metadata Store Inspector</h2>
+
+<table class="mdstore-table">
+	<tr>
+		<th rowspan="3" class="col-xs-1" style="width: 10em">MdStore</th>
+		<th style="width: 20em">ID</th>
+		<td>{{mdstore?.id}}</td>
+	</tr>
+
+	<tr>
+		<th>Format / Layout / Interpretation</th>
+		<td>{{mdstore?.format}} / {{mdstore?.layout}} / {{mdstore?.interpretation}}</td>
+	</tr>
+	<tr>
+		<th>Related Datasource</th>
+		<td>{{mdstore?.datasourceName}}</td>
+	</tr>
+	<tr>
+		<th rowspan="4">Version</th>
+		<th>ID</th>
+		<td>
+			<span *ngIf="version?.id == mdstore?.currentVersion" class="badge-label badge-success">current</span>
+			<span *ngIf="version?.writing && (version?.id != mdstore?.currentVersion)"
+				class="badge-label badge-warning">writing</span>
+			<span *ngIf="!version?.writing && (version?.id != mdstore?.currentVersion)"
+				class=" badge-label badge-failure">expired</span>
+
+			{{version?.id}}
+		</td>
+	</tr>
+	<tr>
+		<th>Hdfs Path</th>
+		<td>{{version?.hdfsPath}}</td>
+	</tr>
+	<tr>
+		<th>Last Update</th>
+		<td>{{version?.lastUpdate}}</td>
+	</tr>
+	<tr>
+		<th>Size</th>
+		<td>{{version?.size}}</td>
+	</tr>
+</table>
+
+<br />
+<h3 *ngIf="records.length > 0">The display is limited to the first {{limit}} records</h3>
+<br />
+
+<mat-card *ngFor="let rec of records" style="margin-top: 10px;">
+	<mat-card-header>
+		<mat-card-title *ngIf="!rec.id">the record is unreadable</mat-card-title>
+		<mat-card-title *ngIf="rec.id">{{rec.id}}</mat-card-title>
+	</mat-card-header>
+	<mat-card-content>
+		<p *ngIf="!rec.id">Invalid record format</p>
+		<table class="mdstore-table" *ngIf="rec.id">
+			<tr>
+				<th style="width: 15em;">Original Id</th>
+				<td>{{rec.originalId}}</td>
+			</tr>
+			<tr>
+				<th>Collected on</th>
+				<td>{{rec.dateOfCollection | date:'medium'}}</td>
+			</tr>
+			<tr>
+				<th>Transformed on</th>
+				<td>{{rec.dateOfTransformation | date:'medium'}}</td>
+			</tr>
+			<tr>
+				<th>Provenance</th>
+				<td class="small">
+					<span *ngIf="rec.provenance.datasourceName"><b>Datasource Name</b>:
+						{{rec.provenance.datasourceName}}<br /></span>
+					<span *ngIf="rec.provenance.datasourceId"><b>Datasource ID</b>: {{rec.provenance.datasourceId}}<br /></span>
+					<span *ngIf="rec.provenance.nsPrefix"><b>Prefix</b>: {{rec.provenance.nsPrefix}}<br /></span>
+				</td>
+			</tr>
+			<tr>
+				<th>Format</th>
+				<td>{{rec.encoding}}</td>
+			</tr>
+		</table>
+		<pre class="small">{{rec.body}}</pre>
+	</mat-card-content>
+</mat-card>

frontends/dnet-is-application/src/app/mdstores/mdstores.component.css

@@ -1,9 +1,10 @@
 .mdstore-table {
 	border-collapse: collapse;
+	border-bottom: 1pt solid lightgrey;
 }
 
-.mdstore-table tr:not(:last-child) {
+.mdstore-table tr {
-  border-bottom: 1pt solid lightgrey;
+	border-top: 1pt solid lightgrey;
 }
 
 .mdstore-table th,

frontends/dnet-is-application/src/app/mdstores/mdstores.component.ts

@@ -2,7 +2,7 @@ import { Component, Inject, OnInit } from '@angular/core';
 import { ISService } from '../common/is.service';
 import { ActivatedRoute, Router } from '@angular/router';
 import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
-import { MDStore, MDStoreVersion } from '../common/is.model';
+import { MDStore, MDStoreRecord, MDStoreVersion } from '../common/is.model';
 import { FormControl, FormGroup, Validators } from '@angular/forms';
 
 @Component({
@@ -61,8 +61,32 @@ export class MdstoresComponent implements OnInit {
 	templateUrl: './mdstore-inspector.component.html',
 	styleUrls: ['./mdstores.component.css']
 })
-export class MdstoreInspectorComponent {
+export class MdstoreInspectorComponent implements OnInit {
 
+	mdstore?: MDStore = undefined;
+	version?: MDStoreVersion = undefined;
+	records: MDStoreRecord[] = [];
+
+	limit: number = 0;
+
+	constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) {
+	}
+	ngOnInit() {
+		this.route.params.subscribe(params => {
+			const versionId = params['versionId'];
+			this.limit = params['limit'];
+
+			this.service.loadMDStoreVersion(versionId, (data: MDStoreVersion) => {
+				this.version = data;
+				this.service.loadMDStore(this.version.mdstore, (data: MDStore) => {
+					this.mdstore = data;
+					this.service.loadMDStoreVersionRecords(versionId, this.limit, (data: MDStoreRecord[]) => {
+						this.records = data;
+					});
+				});
+			});
+		});
+	}
 }
 
 @Component({