MaDgIK
/
explore-services
13
0
Fork 0
Code Issues Pull Requests Projects Releases 17 Wiki Activity

Merge remote-tracking branch 'origin/develop'

This commit is contained in:
Konstantina Galouni 2024-02-08 19:10:13 +02:00
parent ee38203ee4 1d7a7ae2c4
commit a5019e083c
10 changed files with 242 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
explore/src/app/app.module.ts
View File

@ -39,11 +39,11 @@ import {AlertModalModule} from './openaireLibrary/utils/modal/alertModal.module'
exports: [AppComponent], exports: [AppComponent],
providers: [ providers: [
{provide: APP_ID, useValue: 'explore'}, {provide: APP_ID, useValue: 'explore'},
{ // {
provide: HTTP_INTERCEPTORS, // provide: HTTP_INTERCEPTORS,
useClass: CacheInterceptorService, // useClass: CacheInterceptorService,
multi: true // multi: true
}, // },
{ {
provide: HTTP_INTERCEPTORS, provide: HTTP_INTERCEPTORS,
useClass: HttpInterceptorService, useClass: HttpInterceptorService,

2
explore/src/app/contact/contact.component.ts
View File

@ -67,7 +67,7 @@ export class ContactComponent implements OnInit {
this.updateTitle(this.pageTitle); this.updateTitle(this.pageTitle);
this.updateDescription(this.description); this.updateDescription(this.description);
this.reset(); this.reset();
this.getPageContents(); // this.getPageContents();
this.sending = false; this.sending = false;
} }

249
explore/src/app/funders/funders.component.ts
View File

@ -17,6 +17,7 @@ import {NumberUtils} from '../openaireLibrary/utils/number-utils.class';
import {LayoutService} from '../openaireLibrary/dashboard/sharedComponents/sidebar/layout.service'; import {LayoutService} from '../openaireLibrary/dashboard/sharedComponents/sidebar/layout.service';
import {FormBuilder, FormControl} from '@angular/forms'; import {FormBuilder, FormControl} from '@angular/forms';
import {debounceTime, distinctUntilChanged} from 'rxjs/operators'; import {debounceTime, distinctUntilChanged} from 'rxjs/operators';
import {GroupedRequestsService} from "../openaireLibrary/services/groupedRequests.service";
@Component({ @Component({
selector: 'funders', selector: 'funders',
@ -85,7 +86,8 @@ export class FundersComponent implements OnInit {
private stakeholderService: StakeholderService, private stakeholderService: StakeholderService,
private layoutService: LayoutService, private layoutService: LayoutService,
private cdr: ChangeDetectorRef, private cdr: ChangeDetectorRef,
private fb: FormBuilder,) { private fb: FormBuilder,
private groupedRequestsService: GroupedRequestsService) {
} }
ngOnInit() { ngOnInit() {
@ -142,100 +144,155 @@ export class FundersComponent implements OnInit {
} }
private getFunders() { private getFunders() {
let refineParams = '&fq=resultbestaccessright%20exact%20%22Open%20Access%22'; this.subscriptions.push(this.groupedRequestsService.funders().subscribe(data => {
this.subscriptions.push( let funders = data["funders"];
zip( funders.forEach(queriedFunder => {
this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties), let id = queriedFunder.id;
this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties, refineParams), let funder = {
this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('funder', 'project', this.properties), "id": id,
this.stakeholderService.getStakeholders(this.properties.monitorServiceAPIURL, 'funder') "name": this.removePartAfterCharacters(queriedFunder.name, "||"),
).subscribe((data: any[]) => { "alias": '',
// storing all needed data to a map "researchProducts": +queriedFunder.results,
// 1st call "openAccessResearchProducts": +queriedFunder.openResults,
let queriedFunders1 = data[0][1][0].values; "openAccessPercentage": 0,
queriedFunders1.forEach(queriedFunder => { "projects": +queriedFunder.projects,
this.fundersMap.set(queriedFunder.id, { "monitorDashboard": '',
"id": queriedFunder.id, "monitorDashboardStatus": '',
"name": queriedFunder.name, "logoUrl": ''
"alias": '', };
"researchProducts": +queriedFunder.number,
"openAccessResearchProducts": 0,
"openAccessPercentage": 0,
"projects": 0,
"monitorDashboard": '',
"monitorDashboardStatus": '',
"logoUrl": ''
});
});
// 2nd call if((!funder.researchProducts || funder.researchProducts == 0) && (!funder.projects || funder.projects == 0)) {
let queriedFunders2 = data[1][1][0].values; return;
queriedFunders2.forEach(queriedFunder => { }
if(this.fundersMap.has(queriedFunder.id)) {
this.fundersMap.get(queriedFunder.id).openAccessResearchProducts = +queriedFunder.number;
}
});
// 3rd call if(queriedFunder.stakeholder) {
let queriedFunders3 = data[2][1][0].values; let stakeholder = queriedFunder.stakeholder;
queriedFunders3.forEach(queriedFunder => { funder.alias = stakeholder.alias;
if(+queriedFunder.number > 1) { funder.monitorDashboard = stakeholder.alias;
if (this.fundersMap.has(queriedFunder.id)) { funder.monitorDashboardStatus = stakeholder.visibility;
this.fundersMap.get(queriedFunder.id).projects = +queriedFunder.number; funder.logoUrl = (stakeholder.isUpload ? properties.utilsService + "/download/" : "")+ (stakeholder.logoUrl);
} else { }
this.fundersMap.set(queriedFunder.id, {
"id": queriedFunder.id,
"name": queriedFunder.name,
"alias": '',
"researchProducts": 0,
"openAccessResearchProducts": 0,
"openAccessPercentage": 0,
"projects": +queriedFunder.number,
"monitorDashboard": '',
"monitorDashboardStatus": '',
"logoUrl": ''
});
}
}
});
// 4th call if(funder.openAccessResearchProducts && funder.researchProducts) {
let queriedFunders4 = data[3]; funder.openAccessPercentage = Math.round((funder.openAccessResearchProducts / funder.researchProducts) * 100);
queriedFunders4.forEach(queriedFunder => { }
let id = queriedFunder.index_id + '||' + queriedFunder.index_name + '||' + queriedFunder.index_shortName;
if(this.fundersMap.has(id) && ( if(!funder.logoUrl && this.staticLogos.has(funder.id)) {
!this.fundersMap.get(id).monitorDashboardStatus let split = funder.id.split("||");
|| let shortname = (split && split.length==3) ? funder.id.split("||")[2] : funder.id;
(this.fundersMap.get(id).monitorDashboardStatus == "RESTRICTED" && queriedFunder.visibility == "PUBLIC") funder.logoUrl = "assets/explore-assets/funders/logos/"+shortname+".png";
|| }
(this.fundersMap.get(id).monitorDashboardStatus == "PRIVATE" && (queriedFunder.visibility == "RESTRICTED" || queriedFunder.visibility == "PUBLIC")))) {
this.fundersMap.get(id).alias = queriedFunder.alias; this.fundersMap.set(queriedFunder.id, funder);
this.fundersMap.get(id).monitorDashboard = queriedFunder.alias; });
this.fundersMap.get(id).monitorDashboardStatus = queriedFunder.visibility; // convert funders map into an array
this.fundersMap.get(id).logoUrl = (queriedFunder.isUpload ? properties.utilsService + "/download/" : "")+ (queriedFunder.logoUrl); this.funders = Array.from(this.fundersMap.values());
} // calculate total numbers for intro content
}); this.fundersNumber = data.count;
this.fundersMap.forEach((value) => { this.researchProductsNumber = data.results;
if(value.openAccessResearchProducts > 0) { this.projectsNumber = data.projects;
value.openAccessPercentage = Math.round((value.openAccessResearchProducts / value.researchProducts) * 100); // sort funders
} this.funders.sort((a, b) => a['name'].localeCompare(b['name']));
if(!value.logoUrl && this.staticLogos.has(value.id)) { // initialize displayedFunders
let split = value.id.split("||"); this.displayedFunders = this.funders;
let shortname = (split && split.length==3) ? value.id.split("||")[2] : value.id; this.showLoading = false;
value.logoUrl = "assets/explore-assets/funders/logos/"+shortname+".png"; }))
}
}); // let refineParams = '&fq=resultbestaccessright%20exact%20%22Open%20Access%22';
// convert funders map into an array // this.subscriptions.push(
this.funders = Array.from(this.fundersMap.values()); // zip(
// calculate total numbers for intro content // this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties),
this.calculateNumbers(); // this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties, refineParams),
// sort funders // this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('funder', 'project', this.properties),
this.funders.sort((a, b) => a['name'].localeCompare(b['name'])); // this.stakeholderService.getStakeholders(this.properties.monitorServiceAPIURL, 'funder')
// initialize displayedFunders // ).subscribe((data: any[]) => {
this.displayedFunders = this.funders; // // storing all needed data to a map
this.showLoading = false; // // 1st call
}) // let queriedFunders1 = data[0][1][0].values;
); // queriedFunders1.forEach(queriedFunder => {
// this.fundersMap.set(queriedFunder.id, {
// "id": queriedFunder.id,
// "name": queriedFunder.name,
// "alias": '',
// "researchProducts": +queriedFunder.number,
// "openAccessResearchProducts": 0,
// "openAccessPercentage": 0,
// "projects": 0,
// "monitorDashboard": '',
// "monitorDashboardStatus": '',
// "logoUrl": ''
// });
// });
//
// // 2nd call
// let queriedFunders2 = data[1][1][0].values;
// queriedFunders2.forEach(queriedFunder => {
// if(this.fundersMap.has(queriedFunder.id)) {
// this.fundersMap.get(queriedFunder.id).openAccessResearchProducts = +queriedFunder.number;
// }
// });
//
// // 3rd call
// let queriedFunders3 = data[2][1][0].values;
// queriedFunders3.forEach(queriedFunder => {
// if(+queriedFunder.number > 1) {
// if (this.fundersMap.has(queriedFunder.id)) {
// this.fundersMap.get(queriedFunder.id).projects = +queriedFunder.number;
// } else {
// this.fundersMap.set(queriedFunder.id, {
// "id": queriedFunder.id,
// "name": queriedFunder.name,
// "alias": '',
// "researchProducts": 0,
// "openAccessResearchProducts": 0,
// "openAccessPercentage": 0,
// "projects": +queriedFunder.number,
// "monitorDashboard": '',
// "monitorDashboardStatus": '',
// "logoUrl": ''
// });
// }
// }
// });
//
// // 4th call
// let queriedFunders4 = data[3];
// queriedFunders4.forEach(queriedFunder => {
// let id = queriedFunder.index_id + '||' + queriedFunder.index_name + '||' + queriedFunder.index_shortName;
// if(this.fundersMap.has(id) && (
// !this.fundersMap.get(id).monitorDashboardStatus
// ||
// (this.fundersMap.get(id).monitorDashboardStatus == "RESTRICTED" && queriedFunder.visibility == "PUBLIC")
// ||
// (this.fundersMap.get(id).monitorDashboardStatus == "PRIVATE" && (queriedFunder.visibility == "RESTRICTED" || queriedFunder.visibility == "PUBLIC")))) {
// this.fundersMap.get(id).alias = queriedFunder.alias;
// this.fundersMap.get(id).monitorDashboard = queriedFunder.alias;
// this.fundersMap.get(id).monitorDashboardStatus = queriedFunder.visibility;
// this.fundersMap.get(id).logoUrl = (queriedFunder.isUpload ? properties.utilsService + "/download/" : "")+ (queriedFunder.logoUrl);
// }
// });
// this.fundersMap.forEach((value) => {
// if(value.openAccessResearchProducts > 0) {
// value.openAccessPercentage = Math.round((value.openAccessResearchProducts / value.researchProducts) * 100);
// }
// if(!value.logoUrl && this.staticLogos.has(value.id)) {
// let split = value.id.split("||");
// let shortname = (split && split.length==3) ? value.id.split("||")[2] : value.id;
// value.logoUrl = "assets/explore-assets/funders/logos/"+shortname+".png";
// }
// });
// // convert funders map into an array
// this.funders = Array.from(this.fundersMap.values());
// // calculate total numbers for intro content
// this.calculateNumbers();
// // sort funders
// this.funders.sort((a, b) => a['name'].localeCompare(b['name']));
// // initialize displayedFunders
// this.displayedFunders = this.funders;
// console.log(this.funders);
// this.showLoading = false;
// })
// );
} }
private calculateSum(array, property) { private calculateSum(array, property) {
@ -305,4 +362,12 @@ export class FundersComponent implements OnInit {
this.currentPage = $event.value; this.currentPage = $event.value;
HelperFunctions.scrollToId('target'); HelperFunctions.scrollToId('target');
} }
private removePartAfterCharacters(field, characters):string {
if(field && field.indexOf(characters) !=-1){
return field.split(characters)[0];
}
return field;
}
} }

3
explore/src/app/funders/funders.module.ts
View File

@ -20,6 +20,7 @@ import {closed_access} from '../openaireLibrary/utils/icons/icons';
import {InputModule} from '../openaireLibrary/sharedComponents/input/input.module'; import {InputModule} from '../openaireLibrary/sharedComponents/input/input.module';
import {PagingModule} from '../openaireLibrary/utils/paging.module'; import {PagingModule} from '../openaireLibrary/utils/paging.module';
import {SearchInputModule} from '../openaireLibrary/sharedComponents/search-input/search-input.module'; import {SearchInputModule} from '../openaireLibrary/sharedComponents/search-input/search-input.module';
import {GroupedRequestsServiceModule} from "../openaireLibrary/services/groupedRequestsService.module";
@NgModule({ @NgModule({
@ -27,7 +28,7 @@ import {SearchInputModule} from '../openaireLibrary/sharedComponents/search-inpu
FundersRoutingModule, CommonModule, RouterModule, AlertModalModule, FundersRoutingModule, CommonModule, RouterModule, AlertModalModule,
Schema2jsonldModule, SEOServiceModule, BreadcrumbsModule, LoadingModule, Schema2jsonldModule, SEOServiceModule, BreadcrumbsModule, LoadingModule,
RefineFieldResultsServiceModule, LogoUrlPipeModule, IconsModule, InputModule, RefineFieldResultsServiceModule, LogoUrlPipeModule, IconsModule, InputModule,
PagingModule, SearchInputModule PagingModule, SearchInputModule, GroupedRequestsServiceModule
], ],
declarations: [ declarations: [
FundersComponent FundersComponent

18
explore/src/app/home/home.component.html
View File

@ -47,7 +47,7 @@
<a class="uk-display-inline-block uk-text-uppercase uk-button uk-button-text uk-margin-small-bottom" <a class="uk-display-inline-block uk-text-uppercase uk-button uk-button-text uk-margin-small-bottom"
routerLinkActive="router-link-active" routerLink="/sdgs"> routerLinkActive="router-link-active" routerLink="/sdgs">
<span class="uk-flex uk-flex-middle"> <span class="uk-flex uk-flex-middle">
<img src="assets/common-assets/sdg/sdg-dot-img.svg" alt="SDGs logo" loading="lazy"> <img src="assets/common-assets/sdg/sdg-dot-img.svg" alt="SDGs logo" loading="lazy" width="17px" height="17px">
<span class="uk-margin-small-left">Sustainable Development Goals (SDG<span <span class="uk-margin-small-left">Sustainable Development Goals (SDG<span
class="uk-text-lowercase">s</span>)</span> class="uk-text-lowercase">s</span>)</span>
</span> </span>
@ -55,7 +55,7 @@
<a class="uk-display-inline-block uk-text-uppercase uk-button uk-button-text" <a class="uk-display-inline-block uk-text-uppercase uk-button uk-button-text"
routerLinkActive="router-link-active" routerLink="/fields-of-science"> routerLinkActive="router-link-active" routerLink="/fields-of-science">
<span class="uk-flex uk-flex-middle"> <span class="uk-flex uk-flex-middle">
<img src="assets/common-assets/fos/fos-icon.svg" alt="FOS logo" loading="lazy"> <img src="assets/common-assets/fos/fos-icon.svg" alt="FOS logo" loading="lazy" width="17px" height="8px">
<span class="uk-margin-small-left">Fields of Science (FoS)</span> <span class="uk-margin-small-left">Fields of Science (FoS)</span>
</span> </span>
</a> </a>
@ -64,8 +64,8 @@
</div> </div>
<div class="uk-visible@m uk-width-expand" uk-scrollspy-class> <div class="uk-visible@m uk-width-expand" uk-scrollspy-class>
<div class="uk-position-relative"> <div class="uk-position-relative">
<img src="assets/explore-assets/home/explore.jpg" uk-parallax="y: 500"> <img src="assets/explore-assets/home/explore.jpg" alt="researchers" uk-parallax="y: 500" loading="lazy">
<img class="uk-position-bottom-left" src="assets/explore-assets/home/mask.svg"> <img class="uk-position-bottom-left" src="assets/explore-assets/home/mask.svg" alt="search" loading="lazy">
</div> </div>
</div> </div>
</div> </div>
@ -156,16 +156,16 @@
alt="ipad" loading="lazy"> alt="ipad" loading="lazy">
</slider-item> </slider-item>
<slider-item type="slide" [start]="0"> <slider-item type="slide" [start]="0">
<img src="assets/explore-assets/home/1.png" alt="ipad" loading="lazy"> <img src="assets/explore-assets/home/1.png" alt="Deposit search" loading="lazy">
</slider-item> </slider-item>
<slider-item type="slide" [start]="0.5"> <slider-item type="slide" [start]="0.5">
<img src="assets/explore-assets/home/2.png" alt="ipad" loading="lazy"> <img src="assets/explore-assets/home/2.png" alt="Deposit in OpenAIRE or Zenodo" loading="lazy">
</slider-item> </slider-item>
<slider-item type="slide" [start]="1"> <slider-item type="slide" [start]="1">
<img src="assets/explore-assets/home/3.png" alt="ipad" loading="lazy"> <img src="assets/explore-assets/home/3.png" alt="Search links" loading="lazy">
</slider-item> </slider-item>
<slider-item type="slide" [start]="1.5"> <slider-item type="slide" [start]="1.5">
<img src="assets/explore-assets/home/4.png" alt="ipad" loading="lazy"> <img src="assets/explore-assets/home/4.png" alt="link" loading="lazy">
</slider-item> </slider-item>
</slider-column> </slider-column>
<slider-column type="nav" class="slider-nav"> <slider-column type="nav" class="slider-nav">
@ -238,7 +238,7 @@
</div> </div>
<div></div> <div></div>
<img class="uk-visible@m uk-height-1-1 uk-position-top-right" <img class="uk-visible@m uk-height-1-1 uk-position-top-right"
src="assets/common-assets/common/graph-nodes.svg" alt="ipad" loading="lazy"> src="assets/common-assets/common/graph-nodes.svg" alt="Graph nodes" loading="lazy">
</div> </div>
</div> </div>
</div> </div>

2
services/cache/mecache/beta-properties.file vendored
View File

@ -1,2 +1,4 @@
port = 4000 port = 4000
utilsService = https://beta.explore.openaire.eu/utils-service utilsService = https://beta.explore.openaire.eu/utils-service
environment = beta
preloadRequests =https://beta.services.openaire.eu/openaire/info/,https://beta.services.openaire.eu/openaire/contexts/,https://beta.services.openaire.eu/usagestats/allmetrics,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json

119
services/cache/mecache/cache.js vendored
View File

@ -2,23 +2,19 @@
const axios = require('axios'); const axios = require('axios');
let express = require('express'); let express = require('express');
let app = express(); let app = express();
let mcache = require('memory-cache'); const { LRUCache } = require('lru-cache');
const request = require('superagent'); const request = require('superagent');
const prom = require('prom-client'); const prom = require('prom-client');
const URL = require('url'); const URL = require('url');
var PropertiesReader = require('properties-reader'); const PropertiesReader = require('properties-reader');
var properties = PropertiesReader('./properties.file'); const properties = PropertiesReader('./properties.file');
const expireShort = 60 * 60 * 1000 // 1 hour //2 * 60 * 1000; //2mins const environment = properties.get("environment");
const expireLong = 24 * 60 * 60 * 1000; //24 hours const preloadRequests = properties.get("preloadRequests").split(',');;
const cacheMaxSize = 500;
const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", const cacheMaxSize = 300;
"/provision/mvc/vocabularies/", "/pages?page_route=",
"/allmetrics", "/countryusagestats/", "/openaire/info",
"/api/communities/", "/openaire/contexts/",
"/utils-service/explore/home", "/utils-service/explore/search", "/utils-service/explore/funders"];
let cors = require('cors'); let cors = require('cors');
app.use(cors()); app.use(cors());
const lruCache = new LRUCache({ max: cacheMaxSize });
const register = new prom.Registry(); const register = new prom.Registry();
prom.collectDefaultMetrics({register: register}); prom.collectDefaultMetrics({register: register});
@ -46,39 +42,40 @@ const histogram = new prom.Histogram({
let cache = () => { let cache = () => {
return (req, res, next) => { return (req, res, next) => {
if (req.query.url) { if (req.query.url) {
if (mcache.memsize() > cacheMaxSize) {
console.log("Max cache size reached!" + cacheMaxSize);
clearCache();
}
let key = '__express__' + req.query.url; let key = '__express__' + req.query.url;
let longCache = checkForLongCachedRequests(req.query.url);
let cachedBody = mcache.get(key);
const url = new URL.parse(req.query.url); const url = new URL.parse(req.query.url);
const target = url.host + '/' + url.pathname.split('/')[1]; const target = url.host + '/' + url.pathname.split('/')[1];
const scheme = url.protocol.replace(':', ''); const scheme = url.protocol.replace(':', '');
if (cachedBody) { if (lruCache.has(key)) {
// console.log( key, "hit")
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'}); const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});
res.send(JSON.parse(cachedBody)); res.send(JSON.parse(lruCache.get(key)));
responses.inc({scheme: scheme, target: target, code: res.statusCode}); responses.inc({scheme: scheme, target: target, code: res.statusCode});
end(); end();
} else { } else {
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'}); // console.log( key, "miss")
res.sendResponse = res.send; const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});
res.send = (body) => { res.sendResponse = res.send;
let alreadyCached = !!mcache.get(key); res.send = (body) => {
entries.set(mcache.memsize()); if(isAllowedToBeCached(scheme, target)) {
if (!alreadyCached) { let alreadyCached = lruCache.has(key);
responses.inc({scheme: scheme, target: target, code: res.statusCode}); entries.set(lruCache.size);
end(); if (!alreadyCached) {
} responses.inc({scheme: scheme, target: target, code: res.statusCode});
if (res.statusCode === 200) { end();
// console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url); }
mcache.put(key, body, longCache ? expireLong : expireShort, () => { if (res.statusCode === 200) {
entries.set(mcache.memsize()); lruCache.set(key, body);
}); entries.set(lruCache.size);
} }
res.sendResponse(body); res.sendResponse(body);
}; }else{
res.statusCode = 405;
res.sendResponse(JSON.stringify( {code: 405, message: "Method not Allowed"}));
}
};
next(); next();
} }
} else { } else {
@ -86,19 +83,26 @@ let cache = () => {
} }
}; };
}; };
function isAllowedToBeCached(scheme, target){
if(environment != "development"){
return scheme.indexOf("https://")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".zenodo.org/api/") !=-1 || target.indexOf("lab.idiap.ch/enermaps" != -1))
} else if(environment == "development"){
return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1 || target.indexOf("lab.idiap.ch/enermaps") != -1
}
return true;
}
app.get('/clear', (req, res) => { app.get('/clear', (req, res) => {
let c = mcache.size(); let c = lruCache.size;
const url = req.query.url; const url = req.query.url;
let message = ""; let message = "";
if (url) { if (url) {
let key = '__express__' + req.query.url; let key = '__express__' + req.query.url;
mcache.del(key); lruCache.delete(key);
message = "Delete entry with key " + url; message = "Delete entry with key " + url;
entries.set(mcache.size()); entries.set(lruCache.size);
} else { } else {
clearCache(); clearCache();
message = "Delete " + c + " entries. Now there are: " + mcache.size() message = "Delete " + c + " entries. Now there are: " + lruCache.size
} }
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
@ -119,7 +123,6 @@ app.get('/get', cache(), cors(), (req, res) => {
res.status(404).send(getResponse(404, "Not Found ")) //not found res.status(404).send(getResponse(404, "Not Found ")) //not found
} else { } else {
request.get(url, function (err, response) { request.get(url, function (err, response) {
// res.header("Access-Control-Allow-Origin", "http://localhost:3000");
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
@ -155,31 +158,29 @@ function getResponse(code, message) {
function clearCache() { function clearCache() {
console.log("cache is cleared!"); console.log("cache is cleared!");
mcache.clear(); lruCache.clear();
entries.set(mcache.size()); entries.set(lruCache.size);
initCache(); initCache();
} }
async function initCache() { async function initCache() {
try { try {
const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); const requests = await axios.get(properties.get('utilsService') + '/grouped-requests');
const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url));
const additionalDataResponses = await Promise.all(additionalDataPromises); const additionalDataResponses = await Promise.all(additionalDataPromises);
console.log("Cache initialized!") console.log("Cache initialized group queries!")
} catch (error) { } catch (error) {
console.error('Error fetching data: Cache initialize failed', error.message); console.error('Error fetching data: Cache initialize failed', error);
}
try{
const additionalDataPromisesPreloadRequests = preloadRequests.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + url));
const additionalDataResponsesPreloadRequests = await Promise.all(additionalDataPromisesPreloadRequests);
console.log("Cache initialized preload requests!")
} catch (error) {
console.error('Error fetching data: Cache initialize failed', error);
} }
} }
function checkForLongCachedRequests(url) {
let long = false;
longCachingRequests.forEach(partUrl => {
if (url.indexOf(partUrl) !== -1) {
long = true;
}
});
return long;
}
function resetAtMidnight() { function resetAtMidnight() {
console.log("Run Reset timer"); console.log("Run Reset timer");

2
services/cache/mecache/package.json vendored
View File

@ -13,7 +13,7 @@
"axios": "^1.6.7", "axios": "^1.6.7",
"cors": "^2.8.5", "cors": "^2.8.5",
"express": "^4.18.2", "express": "^4.18.2",
"memory-cache": "^0.2.0", "lru-cache": "^10.2.0",
"prom-client": "^11.3.0", "prom-client": "^11.3.0",
"properties-reader": "^2.2.0", "properties-reader": "^2.2.0",
"superagent": "^5.0.5" "superagent": "^5.0.5"

3
services/cache/mecache/production-properties.file vendored
View File

@ -1,2 +1,5 @@
port = 4000 port = 4000
utilsService = https://explore.openaire.eu/utils-service utilsService = https://explore.openaire.eu/utils-service
environment = production
preloadRequests =https://services.openaire.eu/openaire/info/,https://services.openaire.eu/openaire/contexts/,https://services.openaire.eu/usagestats/allmetrics,https://services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json

2
services/cache/mecache/properties.file vendored
View File

@ -1,2 +1,4 @@
port = 3200 port = 3200
utilsService= http://scoobydoo.di.uoa.gr:8000 utilsService= http://scoobydoo.di.uoa.gr:8000
environment = development
preloadRequests =https://dev-openaire.d4science.org/openaire/info/,https://dev-openaire.d4science.org/openaire/contexts/,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:provenanceActions.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:publication_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:languages.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:access_modes.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:countries.json