From 163c6685777a31c62d8bc64311348c9406d7b2b1 Mon Sep 17 00:00:00 2001 From: "k.triantafyllou" Date: Wed, 7 Jun 2023 14:20:48 +0300 Subject: [PATCH] Create cache indicators and add methods in server. Remove dashboard path from development. Add button for cache indicators in manage stakeholders --- package.json | 4 +- server.ts | 96 ++++--- .../manageStakeholders.component.html | 3 + .../manageStakeholders.component.ts | 12 + src/app/utils/indicator-utils.ts | 2 +- .../services/cache-indicators.service.ts | 24 ++ src/cache-indicators.ts | 262 ++++++++++++++++++ src/environments/environment.ts | 2 +- src/index.html | 2 +- src/stats-tool-parser.ts | 93 ------- 10 files changed, 364 insertions(+), 136 deletions(-) create mode 100644 src/app/utils/services/cache-indicators.service.ts create mode 100644 src/cache-indicators.ts delete mode 100644 src/stats-tool-parser.ts diff --git a/package.json b/package.json index 8fec5bd..38e13c9 100644 --- a/package.json +++ b/package.json @@ -5,13 +5,13 @@ "ng": "ng", "start": "ng serve --port 4600 --disable-host-check --host 0.0.0.0", "build": "ng build", - "build-dev": "ng build --configuration=development --source-map --base-href /dashboard/", + "build-dev": "ng build --configuration=development --source-map", "build-beta": "ng build --configuration=beta --base-href /dashboard/ --source-map", "build-prod": "ng build --configuration production --base-href /dashboard/ --source-map", "webpack-bundle-analyzer": "ng build --stats-json && webpack-bundle-analyzer dist/monitor-dashboard/browser/stats.json --host 0.0.0.0", "test": "ng test", "e2e": "ng e2e", - "dev:ssr": "ng run monitor-dashboard:serve-ssr", + "dev:ssr": "ng run monitor-dashboard:serve-ssr --port 4600", "serve:ssr": "node dist/monitor-dashboard/server/main.js", "build:ssr-dev": "npm run build-dev && ng run monitor-dashboard:server:development", "build:ssr-beta": "npm run build-beta && ng run monitor-dashboard:server:beta", diff --git a/server.ts b/server.ts index 4ae9f20..f4ec9c1 100644 --- a/server.ts +++ b/server.ts @@ -10,10 +10,10 @@ import {APP_BASE_HREF} from '@angular/common'; import {existsSync} from 'fs'; import {REQUEST, RESPONSE} from "./src/app/openaireLibrary/utils/tokens"; import {properties} from "./src/environments/environment"; -import {statsToolParser} from "./src/stats-tool-parser"; -import axios from "axios"; +import axios, {AxiosHeaders} from "axios"; import {Stakeholder} from "./src/app/openaireLibrary/monitor/entities/stakeholder"; -import {IndicatorUtils} from "./src/app/utils/indicator-utils"; +import {CacheIndicators} from "./src/cache-indicators"; +import {Session, User} from "./src/app/openaireLibrary/login/utils/helper.class"; var bodyParser = require('body-parser'); var jsonParser = bodyParser.json(); @@ -24,6 +24,7 @@ export function app() { server.use(compression()); const distFolder = join(process.cwd(), 'dist/monitor-dashboard/browser'); const indexHtml = existsSync(join(distFolder, 'index.original.html')) ? 'index.original.html' : 'index'; + let cacheIndicators: CacheIndicators = new CacheIndicators(); // Our Universal express-engine (found @ https://github.com/angular/universal/tree/master/modules/express-engine) server.engine('html', ngExpressEngine({ @@ -43,44 +44,63 @@ export function app() { }); server.post('/cache/:alias', jsonParser, async (req, res) => { - let stakeholder:Stakeholder = (await axios.get(properties.monitorServiceAPIURL + '/stakeholder/' + encodeURIComponent(req.params.alias))).data; - let list = []; - let indicatorUtils = new IndicatorUtils(); - if(stakeholder) { - stakeholder.topics.forEach(topic => { - topic.categories.forEach(category => { - category.subCategories.forEach(subCategory => { - subCategory.numbers.forEach(section => { - section.indicators.forEach(indicator => { - indicator.indicatorPaths.forEach(indicatorPath => { - let url = indicatorUtils.getFullUrl(stakeholder, indicatorPath); - list.push(url); - }); - }); - }); - subCategory.charts.forEach(section => { - section.indicators.forEach(indicator => { - indicator.indicatorPaths.forEach(indicatorPath => { - let url = indicatorUtils.getFullUrl(stakeholder, indicatorPath); - list.push(url); - }); - }); - }); - }); + await checkPermissions(req, res, (stakeholder, user) => { + if (cacheIndicators.completed(stakeholder._id)) { + res.send({ + id: stakeholder._id, + report: cacheIndicators.createReport(stakeholder._id, cacheIndicators.stakeholderToCacheItems(stakeholder), stakeholder.name, user.email) }); - }); - } else { - res.status(404).send('Stakeholder has not been found'); - } - /*let [url, json] = req.body.url.split('?json='); - json = decodeURIComponent(json); - json = statsToolParser(JSON.parse(json)); - const response = axios.post(url, json).then((data) => { - console.log(data); - });*/ - res.send(list); + } else { + res.status(409).send('There is another active caching process for this stakeholder'); + } + }); }); + server.get('/cache/:alias', async (req, res) => { + await checkPermissions(req, res, stakeholder => { + if (cacheIndicators.exists(stakeholder._id)) { + res.send({ + id: stakeholder._id, + report: cacheIndicators.getReport(stakeholder._id) + }); + } else { + res.status(404).send('There is not an active caching process for this stakeholder'); + } + }); + }); + + async function checkPermissions(req, res, access: (stakeholder, user) => void) { + let headers: AxiosHeaders = new AxiosHeaders(); + headers.set('Cookie', req.headers.cookie); + let userinfoRes = (await axios.get(properties.userInfoUrl, { + withCredentials: true, + headers: headers + }).catch(error => { + return error.response; + })); + if (userinfoRes.status === 200) { + let user = new User(userinfoRes.data); + let stakeholderRes = (await axios.get(properties.monitorServiceAPIURL + '/stakeholder/' + encodeURIComponent(req.params.alias), { + withCredentials: true, + headers: headers + }).catch(error => { + return error.response; + })); + if (stakeholderRes.status === 200) { + let stakeholder = stakeholderRes.data; + if (Session.isPortalAdministrator(user) || Session.isCurator(stakeholder.type, user)) { + access(stakeholder, user); + } else { + res.status(403).send('You are forbidden to access this resource'); + } + } else { + res.status(stakeholderRes.status).send(stakeholderRes.statusText); + } + } else { + res.status(userinfoRes.status).send(userinfoRes.data.message); + } + } + // Example Express Rest API endpoints // server.get('/api/**', (req, res) => { }); // Serve static files from /browser diff --git a/src/app/manageStakeholders/manageStakeholders.component.html b/src/app/manageStakeholders/manageStakeholders.component.html index edc45e0..696e680 100644 --- a/src/app/manageStakeholders/manageStakeholders.component.html +++ b/src/app/manageStakeholders/manageStakeholders.component.html @@ -69,6 +69,9 @@
  • Edit
    • +
  • + Cache Indicators +
  • diff --git a/src/app/manageStakeholders/manageStakeholders.component.ts b/src/app/manageStakeholders/manageStakeholders.component.ts index 47a3848..90cccdc 100644 --- a/src/app/manageStakeholders/manageStakeholders.component.ts +++ b/src/app/manageStakeholders/manageStakeholders.component.ts @@ -13,6 +13,8 @@ import {Session} from "../openaireLibrary/login/utils/helper.class"; import {EditStakeholderComponent} from "../general/edit-stakeholder/edit-stakeholder.component"; import {properties} from "../../environments/environment"; import {ActivatedRoute} from "@angular/router"; +import {CacheIndicatorsService} from "../utils/services/cache-indicators.service"; +import {NotificationHandler} from "../openaireLibrary/utils/notification-handler"; type Tab = 'all' | 'templates'| 'profiles'; @@ -62,6 +64,7 @@ export class ManageStakeholdersComponent implements OnInit, OnDestroy { @ViewChild('editStakeholderComponent', { static: true }) editStakeholderComponent: EditStakeholderComponent; constructor(private stakeholderService: StakeholderService, + private cacheIndicatorsService: CacheIndicatorsService, private userManagementService: UserManagementService, private route: ActivatedRoute, private title: Title, @@ -202,6 +205,15 @@ export class ManageStakeholdersComponent implements OnInit, OnDestroy { this.editStakeholderModal.open(); } + public createReport(stakeholder: Stakeholder) { + this.cacheIndicatorsService.createReport(stakeholder.alias).subscribe(report => { + NotificationHandler.rise('A caching process for ' + stakeholder.name + ' has been started.' ) + }, error => { + console.log(error); + NotificationHandler.rise(error.message(), 'danger'); + }); + } + public deleteStakeholderOpen(stakeholder: Stakeholder) { this.stakeholder = stakeholder; this.deleteStakeholderModal.alertTitle = 'Delete ' + this.stakeholder.index_name; diff --git a/src/app/utils/indicator-utils.ts b/src/app/utils/indicator-utils.ts index bf8af6c..4665a13 100644 --- a/src/app/utils/indicator-utils.ts +++ b/src/app/utils/indicator-utils.ts @@ -215,7 +215,7 @@ export class IndicatorUtils { return this.chartSources.get(source)[0] + url; } - getNumberUrl(source: string, url: string): string { + getNumberUrl(source: SourceType, url: string): string { return this.numberSources.get(this.getSourceType(source))[0] + url; } diff --git a/src/app/utils/services/cache-indicators.service.ts b/src/app/utils/services/cache-indicators.service.ts new file mode 100644 index 0000000..cc36298 --- /dev/null +++ b/src/app/utils/services/cache-indicators.service.ts @@ -0,0 +1,24 @@ +import {Injectable} from "@angular/core"; +import {HttpClient} from "@angular/common/http"; +import {properties} from "../../../environments/environment"; +import {Observable} from "rxjs"; +import {CustomOptions} from "../../openaireLibrary/services/servicesUtils/customOptions.class"; +import {map} from "rxjs/operators"; +import {Report} from "../../../cache-indicators"; + +@Injectable({ + providedIn: 'root' +}) +export class CacheIndicatorsService { + + constructor(private http: HttpClient) { + } + + createReport(alias: string) { + return this.http.post(properties.domain + properties.baseLink + '/cache/' + alias, {}, CustomOptions.registryOptions()); + } + + getReport(alias: string) { + return this.http.get(properties.domain + properties.baseLink + '/cache/' + alias, CustomOptions.registryOptions()); + } +} diff --git a/src/cache-indicators.ts b/src/cache-indicators.ts new file mode 100644 index 0000000..8817488 --- /dev/null +++ b/src/cache-indicators.ts @@ -0,0 +1,262 @@ +import {IndicatorType, Stakeholder} from "./app/openaireLibrary/monitor/entities/stakeholder"; +import axios from "axios"; +import {IndicatorUtils} from "./app/utils/indicator-utils"; +import {Composer} from "./app/openaireLibrary/utils/email/composer"; +import {properties} from "./environments/environment"; +import {error} from "protractor"; + + +export interface CacheItem { + reportId: string, + type: IndicatorType, + url: string +} + +export class Report { + creator: string; + name: string; + success: number; + errors: { + url: string, + status: number + }[]; + total: number; + completed: boolean; + percentage: number + + constructor(total: number, name: string, creator: string) { + this.creator = creator; + this.name = name; + this.success = 0; + this.errors = []; + this.total = total; + this.completed = false; + } + + setPercentage() { + this.percentage = Math.floor((this.success + this.errors.length) / this.total * 100); + } +} + +export class CacheIndicators { + + private static BATCH_SIZE = 10; + + private reports: Map = new Map(); + private queue: CacheItem[] = []; + private process: Promise; + private isFinished: boolean = true; + + stakeholderToCacheItems(stakeholder: Stakeholder) { + let cacheItems: CacheItem[] = []; + let indicatorUtils = new IndicatorUtils(); + stakeholder.topics.forEach(topic => { + topic.categories.forEach(category => { + category.subCategories.forEach(subCategory => { + subCategory.numbers.forEach(section => { + section.indicators.forEach(indicator => { + indicator.indicatorPaths.forEach(indicatorPath => { + let url = indicatorUtils.getNumberUrl(indicatorPath.source, indicatorUtils.getFullUrl(stakeholder, indicatorPath)); + cacheItems.push({ + reportId: stakeholder._id, + type: 'number', + url: url + }); + }); + }); + }); + subCategory.charts.forEach(section => { + section.indicators.forEach(indicator => { + indicator.indicatorPaths.forEach(indicatorPath => { + let url = indicatorUtils.getChartUrl(indicatorPath.source, indicatorUtils.getFullUrl(stakeholder, indicatorPath)); + cacheItems.push({ + reportId: stakeholder._id, + type: 'chart', + url: url + }); + }); + }); + }); + }); + }); + }); + return cacheItems; + } + + public exists(id: string) { + return this.reports.has(id); + } + + public completed(id: string) { + return !this.exists(id) || this.reports.get(id).completed; + } + + public createReport(id: string, cacheItems: CacheItem[], name: string, creator: string) { + let report = new Report(cacheItems.length, name, creator); + this.reports.set(id, report); + this.addItemsToQueue(cacheItems); + return report; + } + + public getReport(id: string) { + return this.reports.get(id); + } + + private async processQueue() { + this.isFinished = false; + while (this.queue.length > 0) { + let batch = this.queue.splice(0, CacheIndicators.BATCH_SIZE); + await this.processBatch(batch); + } + } + + private async processBatch(batch: CacheItem[]) { + let promises: Promise[] = []; + let ids = new Set(); + batch.forEach(item => { + let promise; + ids.add(item.reportId); + if (item.type === 'chart') { + let [url, json] = item.url.split('?json='); + json = decodeURIComponent(json); + json = statsToolParser(JSON.parse(json)); + promise = axios.post(url, json); + } else { + promise = axios.get(item.url); + } + promises.push(promise.then(response => { + let report = this.reports.get(item.reportId); + if (report) { + report.success++; + report.setPercentage(); + } + return response; + }).catch(error => { + let report = this.reports.get(item.reportId); + if (report) { + report.errors.push({url: item.url, status: error.response.status}); + report.setPercentage(); + } + return error.response; + })); + }); + await Promise.all(promises); + ids.forEach(id => { + let report = this.reports.get(id); + if (report?.percentage === 100) { + report.completed = true; + this.sendEmail(report); + } + }); + } + + private addItemsToQueue(cacheItems: CacheItem[]) { + cacheItems.forEach(item => { + this.queue.push(item); + }); + if (this.isFinished) { + this.processQueue().then(() => { + this.isFinished = true; + }); + } + } + + sendEmail(report: Report) { + let email = Composer.composeEmailToReportCachingProcess(report); + axios.post(properties.adminToolsAPIURL + "sendMail/", email).catch(error => { + console.error(error); + }); + } +} + +export function statsToolParser(dataJSONobj: any): any { + let RequestInfoObj = Object.assign({}); + switch (dataJSONobj.library) { + case "GoogleCharts": + //Pass the Chart library to ChartDataFormatter + RequestInfoObj.library = dataJSONobj.library; + RequestInfoObj.orderBy = dataJSONobj.orderBy; + + //Create ChartInfo Object Array + RequestInfoObj.chartsInfo = []; + //Create ChartInfo and pass the Chart data queries to ChartDataFormatter + //along with the requested Chart type + RequestInfoObj.chartsInfo = dataJSONobj.chartDescription.queriesInfo; + break; + case "eCharts": + //Pass the Chart library to ChartDataFormatter + RequestInfoObj.library = dataJSONobj.library; + RequestInfoObj.orderBy = dataJSONobj.orderBy; + + //Create ChartInfo Object Array + RequestInfoObj.chartsInfo = []; + + //Create ChartInfo and pass the Chart data queries to ChartDataFormatter + //along with the requested Chart type + for (let index = 0; index < dataJSONobj.chartDescription.queries.length; index++) { + let element = dataJSONobj.chartDescription.queries[index]; + var ChartInfoObj = Object.assign({}); + + if (element.type === undefined) + ChartInfoObj.type = dataJSONobj.chartDescription.series[index].type; + else + ChartInfoObj.type = element.type; + + if (element.name === undefined) + ChartInfoObj.name = null; + else + ChartInfoObj.name = element.name; + + ChartInfoObj.query = element.query; + RequestInfoObj.chartsInfo.push(ChartInfoObj); + } + break; + case "HighCharts": + RequestInfoObj.library = dataJSONobj.library; + RequestInfoObj.orderBy = dataJSONobj.orderBy; + //Pass the Chart type to ChartDataFormatter + var defaultType = dataJSONobj.chartDescription.chart.type; + //Create ChartInfo Object Array + RequestInfoObj.chartsInfo = []; + //Create ChartInfo and pass the Chart data queries to ChartDataFormatter + //along with the requested Chart type + dataJSONobj.chartDescription.queries.forEach(element => { + var ChartInfoObj = Object.assign({}); + + if (element.type === undefined) + ChartInfoObj.type = defaultType; + else + ChartInfoObj.type = element.type; + + if (element.name === undefined) + ChartInfoObj.name = null; + else + ChartInfoObj.name = element.name; + + ChartInfoObj.query = element.query; + RequestInfoObj.chartsInfo.push(ChartInfoObj); + }); + break; + case "HighMaps": + RequestInfoObj.library = dataJSONobj.library; + //Create ChartInfo Object Array + RequestInfoObj.chartsInfo = []; + + //Create ChartInfo and pass the Chart data queries to ChartDataFormatter + dataJSONobj.mapDescription.queries.forEach(element => { + var ChartInfoObj = Object.assign({}); + + if (element.name === undefined) + ChartInfoObj.name = null; + else + ChartInfoObj.name = element.name; + + ChartInfoObj.query = element.query; + RequestInfoObj.chartsInfo.push(ChartInfoObj); + }); + break; + default: + console.log("Unsupported Library: " + dataJSONobj.library); + } + return RequestInfoObj; +} diff --git a/src/environments/environment.ts b/src/environments/environment.ts index 5b23d2a..12c3305 100644 --- a/src/environments/environment.ts +++ b/src/environments/environment.ts @@ -75,7 +75,7 @@ export let properties: EnvProperties = { csvLimit: 2000, pagingLimit: 20, resultsPerPage: 10, - baseLink: "/dashboard", + baseLink: "/", domain: "http://mpagasas.di.uoa.gr:4600", searchLinkToResult: "/search/result?id=", searchLinkToPublication: "/search/publication?articleId=", diff --git a/src/index.html b/src/index.html index 456b6f1..797d990 100644 --- a/src/index.html +++ b/src/index.html @@ -4,7 +4,7 @@ - + diff --git a/src/stats-tool-parser.ts b/src/stats-tool-parser.ts deleted file mode 100644 index 1aab686..0000000 --- a/src/stats-tool-parser.ts +++ /dev/null @@ -1,93 +0,0 @@ -export function statsToolParser(dataJSONobj: any): any { - let RequestInfoObj = Object.assign({}); - switch(dataJSONobj.library) { - case "GoogleCharts": - //Pass the Chart library to ChartDataFormatter - RequestInfoObj.library = dataJSONobj.library; - RequestInfoObj.orderBy = dataJSONobj.orderBy; - - //Create ChartInfo Object Array - RequestInfoObj.chartsInfo = []; - //Create ChartInfo and pass the Chart data queries to ChartDataFormatter - //along with the requested Chart type - RequestInfoObj.chartsInfo = dataJSONobj.chartDescription.queriesInfo; - break; - case "eCharts": - //Pass the Chart library to ChartDataFormatter - RequestInfoObj.library = dataJSONobj.library; - RequestInfoObj.orderBy = dataJSONobj.orderBy; - - //Create ChartInfo Object Array - RequestInfoObj.chartsInfo = []; - - //Create ChartInfo and pass the Chart data queries to ChartDataFormatter - //along with the requested Chart type - for (let index = 0; index < dataJSONobj.chartDescription.queries.length; index++) { - let element = dataJSONobj.chartDescription.queries[index]; - var ChartInfoObj = Object.assign({}); - - if(element.type === undefined) - ChartInfoObj.type = dataJSONobj.chartDescription.series[index].type; - else - ChartInfoObj.type = element.type; - - if(element.name === undefined) - ChartInfoObj.name = null; - else - ChartInfoObj.name = element.name; - - ChartInfoObj.query = element.query; - RequestInfoObj.chartsInfo.push(ChartInfoObj); - } - break; - case "HighCharts": - RequestInfoObj.library = dataJSONobj.library; - RequestInfoObj.orderBy = dataJSONobj.orderBy; - //Pass the Chart type to ChartDataFormatter - var defaultType = dataJSONobj.chartDescription.chart.type; - //Create ChartInfo Object Array - RequestInfoObj.chartsInfo = []; - //Create ChartInfo and pass the Chart data queries to ChartDataFormatter - //along with the requested Chart type - dataJSONobj.chartDescription.queries. - forEach(element => { - var ChartInfoObj = Object.assign({}); - - if(element.type === undefined) - ChartInfoObj.type = defaultType; - else - ChartInfoObj.type = element.type; - - if(element.name === undefined) - ChartInfoObj.name = null; - else - ChartInfoObj.name = element.name; - - ChartInfoObj.query = element.query; - RequestInfoObj.chartsInfo.push(ChartInfoObj); - }); - break; - case "HighMaps": - RequestInfoObj.library = dataJSONobj.library; - //Create ChartInfo Object Array - RequestInfoObj.chartsInfo = []; - - //Create ChartInfo and pass the Chart data queries to ChartDataFormatter - dataJSONobj.mapDescription.queries. - forEach(element => { - var ChartInfoObj = Object.assign({}); - - if(element.name === undefined) - ChartInfoObj.name = null; - else - ChartInfoObj.name = element.name; - - ChartInfoObj.query = element.query; - RequestInfoObj.chartsInfo.push(ChartInfoObj); - }); - break; - default: - console.log("Unsupported Library: "+ dataJSONobj.library); - } - return RequestInfoObj; -}