MaDgIK
/
monitor-dashboard
13
0
Fork 0
Code Issues Pull Requests Projects Releases 17 Wiki Activity

Create cache indicators and add methods in server. Remove dashboard path from development. Add button for cache indicators in manage stakeholders

This commit is contained in:
Konstantinos Triantafyllou 2023-06-07 14:20:48 +03:00
parent 2e3465ae45
commit 163c668577
10 changed files with 364 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@ -5,13 +5,13 @@
"ng": "ng",
"start": "ng serve --port 4600 --disable-host-check --host 0.0.0.0",
"build": "ng build",
"build-dev": "ng build --configuration=development --source-map --base-href /dashboard/",
"build-dev": "ng build --configuration=development --source-map",
"build-beta": "ng build --configuration=beta --base-href /dashboard/ --source-map",
"build-prod": "ng build --configuration production --base-href /dashboard/ --source-map",
"webpack-bundle-analyzer": "ng build --stats-json && webpack-bundle-analyzer dist/monitor-dashboard/browser/stats.json --host 0.0.0.0",
"test": "ng test",
"e2e": "ng e2e",
"dev:ssr": "ng run monitor-dashboard:serve-ssr",
"dev:ssr": "ng run monitor-dashboard:serve-ssr --port 4600",
"serve:ssr": "node dist/monitor-dashboard/server/main.js",
"build:ssr-dev": "npm run build-dev && ng run monitor-dashboard:server:development",
"build:ssr-beta": "npm run build-beta && ng run monitor-dashboard:server:beta",

96
server.ts
View File

@ -10,10 +10,10 @@ import {APP_BASE_HREF} from '@angular/common';
import {existsSync} from 'fs';
import {REQUEST, RESPONSE} from "./src/app/openaireLibrary/utils/tokens";
import {properties} from "./src/environments/environment";
import {statsToolParser} from "./src/stats-tool-parser";
import axios from "axios";
import axios, {AxiosHeaders} from "axios";
import {Stakeholder} from "./src/app/openaireLibrary/monitor/entities/stakeholder";
import {IndicatorUtils} from "./src/app/utils/indicator-utils";
import {CacheIndicators} from "./src/cache-indicators";
import {Session, User} from "./src/app/openaireLibrary/login/utils/helper.class";
var bodyParser = require('body-parser');
var jsonParser = bodyParser.json();
@ -24,6 +24,7 @@ export function app() {
server.use(compression());
const distFolder = join(process.cwd(), 'dist/monitor-dashboard/browser');
const indexHtml = existsSync(join(distFolder, 'index.original.html')) ? 'index.original.html' : 'index';
let cacheIndicators: CacheIndicators = new CacheIndicators();
// Our Universal express-engine (found @ https://github.com/angular/universal/tree/master/modules/express-engine)
server.engine('html', ngExpressEngine({
@ -43,44 +44,63 @@ export function app() {
});
server.post('/cache/:alias', jsonParser, async (req, res) => {
let stakeholder:Stakeholder = (await axios.get<Stakeholder>(properties.monitorServiceAPIURL + '/stakeholder/' + encodeURIComponent(req.params.alias))).data;
let list = [];
let indicatorUtils = new IndicatorUtils();
if(stakeholder) {
stakeholder.topics.forEach(topic => {
topic.categories.forEach(category => {
category.subCategories.forEach(subCategory => {
subCategory.numbers.forEach(section => {
section.indicators.forEach(indicator => {
indicator.indicatorPaths.forEach(indicatorPath => {
let url = indicatorUtils.getFullUrl(stakeholder, indicatorPath);
list.push(url);
});
});
});
subCategory.charts.forEach(section => {
section.indicators.forEach(indicator => {
indicator.indicatorPaths.forEach(indicatorPath => {
let url = indicatorUtils.getFullUrl(stakeholder, indicatorPath);
list.push(url);
});
});
});
});
await checkPermissions(req, res, (stakeholder, user) => {
if (cacheIndicators.completed(stakeholder._id)) {
res.send({
id: stakeholder._id,
report: cacheIndicators.createReport(stakeholder._id, cacheIndicators.stakeholderToCacheItems(stakeholder), stakeholder.name, user.email)
});
});
} else {
res.status(404).send('Stakeholder has not been found');
}
/*let [url, json] = req.body.url.split('?json=');
json = decodeURIComponent(json);
json = statsToolParser(JSON.parse(json));
const response = axios.post(url, json).then((data) => {
console.log(data);
});*/
res.send(list);
} else {
res.status(409).send('There is another active caching process for this stakeholder');
}
});
});
server.get('/cache/:alias', async (req, res) => {
await checkPermissions(req, res, stakeholder => {
if (cacheIndicators.exists(stakeholder._id)) {
res.send({
id: stakeholder._id,
report: cacheIndicators.getReport(stakeholder._id)
});
} else {
res.status(404).send('There is not an active caching process for this stakeholder');
}
});
});
async function checkPermissions(req, res, access: (stakeholder, user) => void) {
let headers: AxiosHeaders = new AxiosHeaders();
headers.set('Cookie', req.headers.cookie);
let userinfoRes = (await axios.get<any>(properties.userInfoUrl, {
withCredentials: true,
headers: headers
}).catch(error => {
return error.response;
}));
if (userinfoRes.status === 200) {
let user = new User(userinfoRes.data);
let stakeholderRes = (await axios.get<Stakeholder>(properties.monitorServiceAPIURL + '/stakeholder/' + encodeURIComponent(req.params.alias), {
withCredentials: true,
headers: headers
}).catch(error => {
return error.response;
}));
if (stakeholderRes.status === 200) {
let stakeholder = stakeholderRes.data;
if (Session.isPortalAdministrator(user) || Session.isCurator(stakeholder.type, user)) {
access(stakeholder, user);
} else {
res.status(403).send('You are forbidden to access this resource');
}
} else {
res.status(stakeholderRes.status).send(stakeholderRes.statusText);
}
} else {
res.status(userinfoRes.status).send(userinfoRes.data.message);
}
}
// Example Express Rest API endpoints
// server.get('/api/**', (req, res) => { });
// Serve static files from /browser

3
src/app/manageStakeholders/manageStakeholders.component.html
View File

@ -69,6 +69,9 @@
<li>
<a (click)="editStakeholder(stakeholder, !stakeholder.defaultId); hide(element)">Edit</a>
</li>
<li *ngIf="isCurator">
<a (click)="createReport(stakeholder);hide(element)">Cache Indicators</a>
</li>
<li class="uk-nav-divider"></li>
<ng-template ngFor [ngForOf]="stakeholderUtils.visibility" let-v>
<li [class.uk-active]="stakeholder.visibility === v.value">

12
src/app/manageStakeholders/manageStakeholders.component.ts
View File

@ -13,6 +13,8 @@ import {Session} from "../openaireLibrary/login/utils/helper.class";
import {EditStakeholderComponent} from "../general/edit-stakeholder/edit-stakeholder.component";
import {properties} from "../../environments/environment";
import {ActivatedRoute} from "@angular/router";
import {CacheIndicatorsService} from "../utils/services/cache-indicators.service";
import {NotificationHandler} from "../openaireLibrary/utils/notification-handler";
type Tab = 'all' | 'templates'| 'profiles';
@ -62,6 +64,7 @@ export class ManageStakeholdersComponent implements OnInit, OnDestroy {
@ViewChild('editStakeholderComponent', { static: true }) editStakeholderComponent: EditStakeholderComponent;
constructor(private stakeholderService: StakeholderService,
private cacheIndicatorsService: CacheIndicatorsService,
private userManagementService: UserManagementService,
private route: ActivatedRoute,
private title: Title,
@ -202,6 +205,15 @@ export class ManageStakeholdersComponent implements OnInit, OnDestroy {
this.editStakeholderModal.open();
}
public createReport(stakeholder: Stakeholder) {
this.cacheIndicatorsService.createReport(stakeholder.alias).subscribe(report => {
NotificationHandler.rise('A caching process for ' + stakeholder.name + ' has been started.' )
}, error => {
console.log(error);
NotificationHandler.rise(error.message(), 'danger');
});
}
public deleteStakeholderOpen(stakeholder: Stakeholder) {
this.stakeholder = stakeholder;
this.deleteStakeholderModal.alertTitle = 'Delete ' + this.stakeholder.index_name;

2
src/app/utils/indicator-utils.ts
View File

@ -215,7 +215,7 @@ export class IndicatorUtils {
return this.chartSources.get(source)[0] + url;
}
getNumberUrl(source: string, url: string): string {
getNumberUrl(source: SourceType, url: string): string {
return this.numberSources.get(this.getSourceType(source))[0] + url;
}

24
src/app/utils/services/cache-indicators.service.ts Normal file
View File

@ -0,0 +1,24 @@
import {Injectable} from "@angular/core";
import {HttpClient} from "@angular/common/http";
import {properties} from "../../../environments/environment";
import {Observable} from "rxjs";
import {CustomOptions} from "../../openaireLibrary/services/servicesUtils/customOptions.class";
import {map} from "rxjs/operators";
import {Report} from "../../../cache-indicators";
@Injectable({
providedIn: 'root'
})
export class CacheIndicatorsService {
constructor(private http: HttpClient) {
}
createReport(alias: string) {
return this.http.post<Report>(properties.domain + properties.baseLink + '/cache/' + alias, {}, CustomOptions.registryOptions());
}
getReport(alias: string) {
return this.http.get<Report>(properties.domain + properties.baseLink + '/cache/' + alias, CustomOptions.registryOptions());
}
}

262
src/cache-indicators.ts Normal file
View File

@ -0,0 +1,262 @@
import {IndicatorType, Stakeholder} from "./app/openaireLibrary/monitor/entities/stakeholder";
import axios from "axios";
import {IndicatorUtils} from "./app/utils/indicator-utils";
import {Composer} from "./app/openaireLibrary/utils/email/composer";
import {properties} from "./environments/environment";
import {error} from "protractor";
export interface CacheItem {
reportId: string,
type: IndicatorType,
url: string
}
export class Report {
creator: string;
name: string;
success: number;
errors: {
url: string,
status: number
}[];
total: number;
completed: boolean;
percentage: number
constructor(total: number, name: string, creator: string) {
this.creator = creator;
this.name = name;
this.success = 0;
this.errors = [];
this.total = total;
this.completed = false;
}
setPercentage() {
this.percentage = Math.floor((this.success + this.errors.length) / this.total * 100);
}
}
export class CacheIndicators {
private static BATCH_SIZE = 10;
private reports: Map<string, Report> = new Map<string, Report>();
private queue: CacheItem[] = [];
private process: Promise<void>;
private isFinished: boolean = true;
stakeholderToCacheItems(stakeholder: Stakeholder) {
let cacheItems: CacheItem[] = [];
let indicatorUtils = new IndicatorUtils();
stakeholder.topics.forEach(topic => {
topic.categories.forEach(category => {
category.subCategories.forEach(subCategory => {
subCategory.numbers.forEach(section => {
section.indicators.forEach(indicator => {
indicator.indicatorPaths.forEach(indicatorPath => {
let url = indicatorUtils.getNumberUrl(indicatorPath.source, indicatorUtils.getFullUrl(stakeholder, indicatorPath));
cacheItems.push({
reportId: stakeholder._id,
type: 'number',
url: url
});
});
});
});
subCategory.charts.forEach(section => {
section.indicators.forEach(indicator => {
indicator.indicatorPaths.forEach(indicatorPath => {
let url = indicatorUtils.getChartUrl(indicatorPath.source, indicatorUtils.getFullUrl(stakeholder, indicatorPath));
cacheItems.push({
reportId: stakeholder._id,
type: 'chart',
url: url
});
});
});
});
});
});
});
return cacheItems;
}
public exists(id: string) {
return this.reports.has(id);
}
public completed(id: string) {
return !this.exists(id) || this.reports.get(id).completed;
}
public createReport(id: string, cacheItems: CacheItem[], name: string, creator: string) {
let report = new Report(cacheItems.length, name, creator);
this.reports.set(id, report);
this.addItemsToQueue(cacheItems);
return report;
}
public getReport(id: string) {
return this.reports.get(id);
}
private async processQueue() {
this.isFinished = false;
while (this.queue.length > 0) {
let batch = this.queue.splice(0, CacheIndicators.BATCH_SIZE);
await this.processBatch(batch);
}
}
private async processBatch(batch: CacheItem[]) {
let promises: Promise<any>[] = [];
let ids = new Set<string>();
batch.forEach(item => {
let promise;
ids.add(item.reportId);
if (item.type === 'chart') {
let [url, json] = item.url.split('?json=');
json = decodeURIComponent(json);
json = statsToolParser(JSON.parse(json));
promise = axios.post(url, json);
} else {
promise = axios.get(item.url);
}
promises.push(promise.then(response => {
let report = this.reports.get(item.reportId);
if (report) {
report.success++;
report.setPercentage();
}
return response;
}).catch(error => {
let report = this.reports.get(item.reportId);
if (report) {
report.errors.push({url: item.url, status: error.response.status});
report.setPercentage();
}
return error.response;
}));
});
await Promise.all(promises);
ids.forEach(id => {
let report = this.reports.get(id);
if (report?.percentage === 100) {
report.completed = true;
this.sendEmail(report);
}
});
}
private addItemsToQueue(cacheItems: CacheItem[]) {
cacheItems.forEach(item => {
this.queue.push(item);
});
if (this.isFinished) {
this.processQueue().then(() => {
this.isFinished = true;
});
}
}
sendEmail(report: Report) {
let email = Composer.composeEmailToReportCachingProcess(report);
axios.post(properties.adminToolsAPIURL + "sendMail/", email).catch(error => {
console.error(error);
});
}
}
export function statsToolParser(dataJSONobj: any): any {
let RequestInfoObj = Object.assign({});
switch (dataJSONobj.library) {
case "GoogleCharts":
//Pass the Chart library to ChartDataFormatter
RequestInfoObj.library = dataJSONobj.library;
RequestInfoObj.orderBy = dataJSONobj.orderBy;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
//along with the requested Chart type
RequestInfoObj.chartsInfo = dataJSONobj.chartDescription.queriesInfo;
break;
case "eCharts":
//Pass the Chart library to ChartDataFormatter
RequestInfoObj.library = dataJSONobj.library;
RequestInfoObj.orderBy = dataJSONobj.orderBy;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
//along with the requested Chart type
for (let index = 0; index < dataJSONobj.chartDescription.queries.length; index++) {
let element = dataJSONobj.chartDescription.queries[index];
var ChartInfoObj = Object.assign({});
if (element.type === undefined)
ChartInfoObj.type = dataJSONobj.chartDescription.series[index].type;
else
ChartInfoObj.type = element.type;
if (element.name === undefined)
ChartInfoObj.name = null;
else
ChartInfoObj.name = element.name;
ChartInfoObj.query = element.query;
RequestInfoObj.chartsInfo.push(ChartInfoObj);
}
break;
case "HighCharts":
RequestInfoObj.library = dataJSONobj.library;
RequestInfoObj.orderBy = dataJSONobj.orderBy;
//Pass the Chart type to ChartDataFormatter
var defaultType = dataJSONobj.chartDescription.chart.type;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
//along with the requested Chart type
dataJSONobj.chartDescription.queries.forEach(element => {
var ChartInfoObj = Object.assign({});
if (element.type === undefined)
ChartInfoObj.type = defaultType;
else
ChartInfoObj.type = element.type;
if (element.name === undefined)
ChartInfoObj.name = null;
else
ChartInfoObj.name = element.name;
ChartInfoObj.query = element.query;
RequestInfoObj.chartsInfo.push(ChartInfoObj);
});
break;
case "HighMaps":
RequestInfoObj.library = dataJSONobj.library;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
dataJSONobj.mapDescription.queries.forEach(element => {
var ChartInfoObj = Object.assign({});
if (element.name === undefined)
ChartInfoObj.name = null;
else
ChartInfoObj.name = element.name;
ChartInfoObj.query = element.query;
RequestInfoObj.chartsInfo.push(ChartInfoObj);
});
break;
default:
console.log("Unsupported Library: " + dataJSONobj.library);
}
return RequestInfoObj;
}

2
src/environments/environment.ts
View File

@ -75,7 +75,7 @@ export let properties: EnvProperties = {
csvLimit: 2000,
pagingLimit: 20,
resultsPerPage: 10,
baseLink: "/dashboard",
baseLink: "/",
domain: "http://mpagasas.di.uoa.gr:4600",
searchLinkToResult: "/search/result?id=",
searchLinkToPublication: "/search/publication?articleId=",

2
src/index.html
View File

@ -4,7 +4,7 @@
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<base href="/dashboard"/>
<base href="/"/>
<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
<meta name="description" content="OpenAIRE Monitor, funder, funding, research, "/>
<meta property="og:description" content="OpenAIRE Monitor, funder, funding"/>

93
src/stats-tool-parser.ts
View File

@ -1,93 +0,0 @@
export function statsToolParser(dataJSONobj: any): any {
let RequestInfoObj = Object.assign({});
switch(dataJSONobj.library) {
case "GoogleCharts":
//Pass the Chart library to ChartDataFormatter
RequestInfoObj.library = dataJSONobj.library;
RequestInfoObj.orderBy = dataJSONobj.orderBy;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
//along with the requested Chart type
RequestInfoObj.chartsInfo = dataJSONobj.chartDescription.queriesInfo;
break;
case "eCharts":
//Pass the Chart library to ChartDataFormatter
RequestInfoObj.library = dataJSONobj.library;
RequestInfoObj.orderBy = dataJSONobj.orderBy;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
//along with the requested Chart type
for (let index = 0; index < dataJSONobj.chartDescription.queries.length; index++) {
let element = dataJSONobj.chartDescription.queries[index];
var ChartInfoObj = Object.assign({});
if(element.type === undefined)
ChartInfoObj.type = dataJSONobj.chartDescription.series[index].type;
else
ChartInfoObj.type = element.type;
if(element.name === undefined)
ChartInfoObj.name = null;
else
ChartInfoObj.name = element.name;
ChartInfoObj.query = element.query;
RequestInfoObj.chartsInfo.push(ChartInfoObj);
}
break;
case "HighCharts":
RequestInfoObj.library = dataJSONobj.library;
RequestInfoObj.orderBy = dataJSONobj.orderBy;
//Pass the Chart type to ChartDataFormatter
var defaultType = dataJSONobj.chartDescription.chart.type;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
//along with the requested Chart type
dataJSONobj.chartDescription.queries.
forEach(element => {
var ChartInfoObj = Object.assign({});
if(element.type === undefined)
ChartInfoObj.type = defaultType;
else
ChartInfoObj.type = element.type;
if(element.name === undefined)
ChartInfoObj.name = null;
else
ChartInfoObj.name = element.name;
ChartInfoObj.query = element.query;
RequestInfoObj.chartsInfo.push(ChartInfoObj);
});
break;
case "HighMaps":
RequestInfoObj.library = dataJSONobj.library;
//Create ChartInfo Object Array
RequestInfoObj.chartsInfo = [];
//Create ChartInfo and pass the Chart data queries to ChartDataFormatter
dataJSONobj.mapDescription.queries.
forEach(element => {
var ChartInfoObj = Object.assign({});
if(element.name === undefined)
ChartInfoObj.name = null;
else
ChartInfoObj.name = element.name;
ChartInfoObj.query = element.query;
RequestInfoObj.chartsInfo.push(ChartInfoObj);
});
break;
default:
console.log("Unsupported Library: "+ dataJSONobj.library);
}
return RequestInfoObj;
}