From 36f1ed105d89f4878c8d4f7d2cc33949fba8ed72 Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Wed, 7 Feb 2024 22:53:03 +0200 Subject: [PATCH 01/10] Updating openaireLibrary & common-assets --- explore/src/app/openaireLibrary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explore/src/app/openaireLibrary b/explore/src/app/openaireLibrary index b12877df..fbe514b7 160000 --- a/explore/src/app/openaireLibrary +++ b/explore/src/app/openaireLibrary @@ -1 +1 @@ -Subproject commit b12877df2a5fb4d9d136edec330b1c5c82e8eb34 +Subproject commit fbe514b74086c61dbfc3b8647b9eea458b34139d From af50558136c0be40ca97f6a568cd29bdb7199fcd Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Wed, 7 Feb 2024 22:56:11 +0200 Subject: [PATCH 02/10] [grouped-queries | DONE | CHANGED] funders.component.ts: Replaced search & stakeholders queries of funders (in Funders page) with grouped query for Funders page of EXPLORE --- explore/src/app/funders/funders.component.ts | 249 ++++++++++++------- explore/src/app/funders/funders.module.ts | 3 +- 2 files changed, 159 insertions(+), 93 deletions(-) diff --git a/explore/src/app/funders/funders.component.ts b/explore/src/app/funders/funders.component.ts index 800b8ced..f1de1989 100644 --- a/explore/src/app/funders/funders.component.ts +++ b/explore/src/app/funders/funders.component.ts @@ -17,6 +17,7 @@ import {NumberUtils} from '../openaireLibrary/utils/number-utils.class'; import {LayoutService} from '../openaireLibrary/dashboard/sharedComponents/sidebar/layout.service'; import {FormBuilder, FormControl} from '@angular/forms'; import {debounceTime, distinctUntilChanged} from 'rxjs/operators'; +import {GroupedRequestsService} from "../openaireLibrary/services/groupedRequests.service"; @Component({ selector: 'funders', @@ -85,7 +86,8 @@ export class FundersComponent implements OnInit { private stakeholderService: StakeholderService, private layoutService: LayoutService, private cdr: ChangeDetectorRef, - private fb: FormBuilder,) { + private fb: FormBuilder, + private groupedRequestsService: GroupedRequestsService) { } ngOnInit() { @@ -142,100 +144,155 @@ export class FundersComponent implements OnInit { } private getFunders() { - let refineParams = '&fq=resultbestaccessright%20exact%20%22Open%20Access%22'; - this.subscriptions.push( - zip( - this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties), - this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties, refineParams), - this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('funder', 'project', this.properties), - this.stakeholderService.getStakeholders(this.properties.monitorServiceAPIURL, 'funder') - ).subscribe((data: any[]) => { - // storing all needed data to a map - // 1st call - let queriedFunders1 = data[0][1][0].values; - queriedFunders1.forEach(queriedFunder => { - this.fundersMap.set(queriedFunder.id, { - "id": queriedFunder.id, - "name": queriedFunder.name, - "alias": '', - "researchProducts": +queriedFunder.number, - "openAccessResearchProducts": 0, - "openAccessPercentage": 0, - "projects": 0, - "monitorDashboard": '', - "monitorDashboardStatus": '', - "logoUrl": '' - }); - }); + this.subscriptions.push(this.groupedRequestsService.funders().subscribe(data => { + let funders = data["funders"]; + funders.forEach(queriedFunder => { + let id = queriedFunder.id; + let funder = { + "id": id, + "name": this.removePartAfterCharacters(queriedFunder.name, "||"), + "alias": '', + "researchProducts": +queriedFunder.results, + "openAccessResearchProducts": +queriedFunder.openResults, + "openAccessPercentage": 0, + "projects": +queriedFunder.projects, + "monitorDashboard": '', + "monitorDashboardStatus": '', + "logoUrl": '' + }; - // 2nd call - let queriedFunders2 = data[1][1][0].values; - queriedFunders2.forEach(queriedFunder => { - if(this.fundersMap.has(queriedFunder.id)) { - this.fundersMap.get(queriedFunder.id).openAccessResearchProducts = +queriedFunder.number; - } - }); + if((!funder.researchProducts || funder.researchProducts == 0) && (!funder.projects || funder.projects == 0)) { + return; + } - // 3rd call - let queriedFunders3 = data[2][1][0].values; - queriedFunders3.forEach(queriedFunder => { - if(+queriedFunder.number > 1) { - if (this.fundersMap.has(queriedFunder.id)) { - this.fundersMap.get(queriedFunder.id).projects = +queriedFunder.number; - } else { - this.fundersMap.set(queriedFunder.id, { - "id": queriedFunder.id, - "name": queriedFunder.name, - "alias": '', - "researchProducts": 0, - "openAccessResearchProducts": 0, - "openAccessPercentage": 0, - "projects": +queriedFunder.number, - "monitorDashboard": '', - "monitorDashboardStatus": '', - "logoUrl": '' - }); - } - } - }); + if(queriedFunder.stakeholder) { + let stakeholder = queriedFunder.stakeholder; + funder.alias = stakeholder.alias; + funder.monitorDashboard = stakeholder.alias; + funder.monitorDashboardStatus = stakeholder.visibility; + funder.logoUrl = (stakeholder.isUpload ? properties.utilsService + "/download/" : "")+ (stakeholder.logoUrl); + } - // 4th call - let queriedFunders4 = data[3]; - queriedFunders4.forEach(queriedFunder => { - let id = queriedFunder.index_id + '||' + queriedFunder.index_name + '||' + queriedFunder.index_shortName; - if(this.fundersMap.has(id) && ( - !this.fundersMap.get(id).monitorDashboardStatus - || - (this.fundersMap.get(id).monitorDashboardStatus == "RESTRICTED" && queriedFunder.visibility == "PUBLIC") - || - (this.fundersMap.get(id).monitorDashboardStatus == "PRIVATE" && (queriedFunder.visibility == "RESTRICTED" || queriedFunder.visibility == "PUBLIC")))) { - this.fundersMap.get(id).alias = queriedFunder.alias; - this.fundersMap.get(id).monitorDashboard = queriedFunder.alias; - this.fundersMap.get(id).monitorDashboardStatus = queriedFunder.visibility; - this.fundersMap.get(id).logoUrl = (queriedFunder.isUpload ? properties.utilsService + "/download/" : "")+ (queriedFunder.logoUrl); - } - }); - this.fundersMap.forEach((value) => { - if(value.openAccessResearchProducts > 0) { - value.openAccessPercentage = Math.round((value.openAccessResearchProducts / value.researchProducts) * 100); - } - if(!value.logoUrl && this.staticLogos.has(value.id)) { - let split = value.id.split("||"); - let shortname = (split && split.length==3) ? value.id.split("||")[2] : value.id; - value.logoUrl = "assets/explore-assets/funders/logos/"+shortname+".png"; - } - }); - // convert funders map into an array - this.funders = Array.from(this.fundersMap.values()); - // calculate total numbers for intro content - this.calculateNumbers(); - // sort funders - this.funders.sort((a, b) => a['name'].localeCompare(b['name'])); - // initialize displayedFunders - this.displayedFunders = this.funders; - this.showLoading = false; - }) - ); + if(funder.openAccessResearchProducts && funder.researchProducts) { + funder.openAccessPercentage = Math.round((funder.openAccessResearchProducts / funder.researchProducts) * 100); + } + + if(!funder.logoUrl && this.staticLogos.has(funder.id)) { + let split = funder.id.split("||"); + let shortname = (split && split.length==3) ? funder.id.split("||")[2] : funder.id; + funder.logoUrl = "assets/explore-assets/funders/logos/"+shortname+".png"; + } + + this.fundersMap.set(queriedFunder.id, funder); + }); + // convert funders map into an array + this.funders = Array.from(this.fundersMap.values()); + // calculate total numbers for intro content + this.fundersNumber = data.count; + this.researchProductsNumber = data.results; + this.projectsNumber = data.projects; + // sort funders + this.funders.sort((a, b) => a['name'].localeCompare(b['name'])); + // initialize displayedFunders + this.displayedFunders = this.funders; + this.showLoading = false; + })) + + // let refineParams = '&fq=resultbestaccessright%20exact%20%22Open%20Access%22'; + // this.subscriptions.push( + // zip( + // this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties), + // this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('relfunder', 'result', this.properties, refineParams), + // this.refineFieldResultsService.getAllRefineFieldResultsByFieldName('funder', 'project', this.properties), + // this.stakeholderService.getStakeholders(this.properties.monitorServiceAPIURL, 'funder') + // ).subscribe((data: any[]) => { + // // storing all needed data to a map + // // 1st call + // let queriedFunders1 = data[0][1][0].values; + // queriedFunders1.forEach(queriedFunder => { + // this.fundersMap.set(queriedFunder.id, { + // "id": queriedFunder.id, + // "name": queriedFunder.name, + // "alias": '', + // "researchProducts": +queriedFunder.number, + // "openAccessResearchProducts": 0, + // "openAccessPercentage": 0, + // "projects": 0, + // "monitorDashboard": '', + // "monitorDashboardStatus": '', + // "logoUrl": '' + // }); + // }); + // + // // 2nd call + // let queriedFunders2 = data[1][1][0].values; + // queriedFunders2.forEach(queriedFunder => { + // if(this.fundersMap.has(queriedFunder.id)) { + // this.fundersMap.get(queriedFunder.id).openAccessResearchProducts = +queriedFunder.number; + // } + // }); + // + // // 3rd call + // let queriedFunders3 = data[2][1][0].values; + // queriedFunders3.forEach(queriedFunder => { + // if(+queriedFunder.number > 1) { + // if (this.fundersMap.has(queriedFunder.id)) { + // this.fundersMap.get(queriedFunder.id).projects = +queriedFunder.number; + // } else { + // this.fundersMap.set(queriedFunder.id, { + // "id": queriedFunder.id, + // "name": queriedFunder.name, + // "alias": '', + // "researchProducts": 0, + // "openAccessResearchProducts": 0, + // "openAccessPercentage": 0, + // "projects": +queriedFunder.number, + // "monitorDashboard": '', + // "monitorDashboardStatus": '', + // "logoUrl": '' + // }); + // } + // } + // }); + // + // // 4th call + // let queriedFunders4 = data[3]; + // queriedFunders4.forEach(queriedFunder => { + // let id = queriedFunder.index_id + '||' + queriedFunder.index_name + '||' + queriedFunder.index_shortName; + // if(this.fundersMap.has(id) && ( + // !this.fundersMap.get(id).monitorDashboardStatus + // || + // (this.fundersMap.get(id).monitorDashboardStatus == "RESTRICTED" && queriedFunder.visibility == "PUBLIC") + // || + // (this.fundersMap.get(id).monitorDashboardStatus == "PRIVATE" && (queriedFunder.visibility == "RESTRICTED" || queriedFunder.visibility == "PUBLIC")))) { + // this.fundersMap.get(id).alias = queriedFunder.alias; + // this.fundersMap.get(id).monitorDashboard = queriedFunder.alias; + // this.fundersMap.get(id).monitorDashboardStatus = queriedFunder.visibility; + // this.fundersMap.get(id).logoUrl = (queriedFunder.isUpload ? properties.utilsService + "/download/" : "")+ (queriedFunder.logoUrl); + // } + // }); + // this.fundersMap.forEach((value) => { + // if(value.openAccessResearchProducts > 0) { + // value.openAccessPercentage = Math.round((value.openAccessResearchProducts / value.researchProducts) * 100); + // } + // if(!value.logoUrl && this.staticLogos.has(value.id)) { + // let split = value.id.split("||"); + // let shortname = (split && split.length==3) ? value.id.split("||")[2] : value.id; + // value.logoUrl = "assets/explore-assets/funders/logos/"+shortname+".png"; + // } + // }); + // // convert funders map into an array + // this.funders = Array.from(this.fundersMap.values()); + // // calculate total numbers for intro content + // this.calculateNumbers(); + // // sort funders + // this.funders.sort((a, b) => a['name'].localeCompare(b['name'])); + // // initialize displayedFunders + // this.displayedFunders = this.funders; + // console.log(this.funders); + // this.showLoading = false; + // }) + // ); } private calculateSum(array, property) { @@ -305,4 +362,12 @@ export class FundersComponent implements OnInit { this.currentPage = $event.value; HelperFunctions.scrollToId('target'); } + + private removePartAfterCharacters(field, characters):string { + if(field && field.indexOf(characters) !=-1){ + return field.split(characters)[0]; + + } + return field; + } } diff --git a/explore/src/app/funders/funders.module.ts b/explore/src/app/funders/funders.module.ts index c89e2d64..e95a5a5f 100644 --- a/explore/src/app/funders/funders.module.ts +++ b/explore/src/app/funders/funders.module.ts @@ -20,6 +20,7 @@ import {closed_access} from '../openaireLibrary/utils/icons/icons'; import {InputModule} from '../openaireLibrary/sharedComponents/input/input.module'; import {PagingModule} from '../openaireLibrary/utils/paging.module'; import {SearchInputModule} from '../openaireLibrary/sharedComponents/search-input/search-input.module'; +import {GroupedRequestsServiceModule} from "../openaireLibrary/services/groupedRequestsService.module"; @NgModule({ @@ -27,7 +28,7 @@ import {SearchInputModule} from '../openaireLibrary/sharedComponents/search-inpu FundersRoutingModule, CommonModule, RouterModule, AlertModalModule, Schema2jsonldModule, SEOServiceModule, BreadcrumbsModule, LoadingModule, RefineFieldResultsServiceModule, LogoUrlPipeModule, IconsModule, InputModule, - PagingModule, SearchInputModule + PagingModule, SearchInputModule, GroupedRequestsServiceModule ], declarations: [ FundersComponent From 5b5000b2fe92be99923a96496a3e86b3cdf549d5 Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Wed, 7 Feb 2024 23:06:52 +0200 Subject: [PATCH 03/10] Updating openaireLibrary --- explore/src/app/openaireLibrary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explore/src/app/openaireLibrary b/explore/src/app/openaireLibrary index b12877df..13727852 160000 --- a/explore/src/app/openaireLibrary +++ b/explore/src/app/openaireLibrary @@ -1 +1 @@ -Subproject commit b12877df2a5fb4d9d136edec330b1c5c82e8eb34 +Subproject commit 137278522e0d3577b222dc6bab3dbc3842c013a0 From 1738dfcccc86c3212e09b881ac6e0d2b314d8af5 Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Thu, 8 Feb 2024 12:40:48 +0200 Subject: [PATCH 04/10] Updating openaireLibrary --- explore/src/app/openaireLibrary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explore/src/app/openaireLibrary b/explore/src/app/openaireLibrary index 13727852..fbe514b7 160000 --- a/explore/src/app/openaireLibrary +++ b/explore/src/app/openaireLibrary @@ -1 +1 @@ -Subproject commit 137278522e0d3577b222dc6bab3dbc3842c013a0 +Subproject commit fbe514b74086c61dbfc3b8647b9eea458b34139d From 07681241c006225c67e1d771652afeaa8f2335a8 Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Thu, 8 Feb 2024 14:20:13 +0200 Subject: [PATCH 05/10] [develop | DONE | CHANGED]: contact.component.ts: Do not request page help contents (helptexts). --- explore/src/app/contact/contact.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explore/src/app/contact/contact.component.ts b/explore/src/app/contact/contact.component.ts index 410d9a18..d421ce75 100644 --- a/explore/src/app/contact/contact.component.ts +++ b/explore/src/app/contact/contact.component.ts @@ -67,7 +67,7 @@ export class ContactComponent implements OnInit { this.updateTitle(this.pageTitle); this.updateDescription(this.description); this.reset(); - this.getPageContents(); + // this.getPageContents(); this.sending = false; } From b0db89f826149bff2d7941abb825f963557f645f Mon Sep 17 00:00:00 2001 From: argirok Date: Thu, 8 Feb 2024 15:58:43 +0200 Subject: [PATCH 06/10] [ develop | DONE | CHANGED] Cache: change to LRU cache, remove short/long cache related code and properties, add preload requests in properties to load on init cache (on start, on clear and on daily reset) --- services/cache/mecache/beta-properties.file | 2 + services/cache/mecache/cache.js | 118 +++++++++--------- services/cache/mecache/package.json | 2 +- .../cache/mecache/production-properties.file | 3 + services/cache/mecache/properties.file | 2 + 5 files changed, 68 insertions(+), 59 deletions(-) diff --git a/services/cache/mecache/beta-properties.file b/services/cache/mecache/beta-properties.file index d39459e8..d34c784c 100644 --- a/services/cache/mecache/beta-properties.file +++ b/services/cache/mecache/beta-properties.file @@ -1,2 +1,4 @@ port = 4000 utilsService = https://beta.explore.openaire.eu/utils-service +environment = beta +preloadRequests =https://beta.services.openaire.eu/openaire/info/,https://beta.services.openaire.eu/openaire/contexts/,https://beta.services.openaire.eu/usagestats/allmetrics,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index 11710b4e..e457a522 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -2,22 +2,19 @@ const axios = require('axios'); let express = require('express'); let app = express(); -let mcache = require('memory-cache'); +const { LRUCache } = require('lru-cache'); const request = require('superagent'); const prom = require('prom-client'); const URL = require('url'); -var PropertiesReader = require('properties-reader'); -var properties = PropertiesReader('./properties.file'); -const expireShort = 60 * 60 * 1000 // 1 hour //2 * 60 * 1000; //2mins -const expireLong = 24 * 60 * 60 * 1000; //24 hours -const cacheMaxSize = 500; -const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", - "/provision/mvc/vocabularies/", "/pages?page_route=", - "/allmetrics", "/countryusagestats/", "/openaire/info", - "/api/communities/", "/openaire/contexts/", "/utils-service/explore/home"]; +const PropertiesReader = require('properties-reader'); +const properties = PropertiesReader('./properties.file'); +const environment = properties.get("environment"); +const preloadRequests = properties.get("preloadRequests").split(',');; + +const cacheMaxSize = 300; let cors = require('cors'); app.use(cors()); - +const lruCache = new LRUCache({ max: cacheMaxSize }); const register = new prom.Registry(); prom.collectDefaultMetrics({register: register}); @@ -45,39 +42,40 @@ const histogram = new prom.Histogram({ let cache = () => { return (req, res, next) => { if (req.query.url) { - if (mcache.memsize() > cacheMaxSize) { - console.log("Max cache size reached!" + cacheMaxSize); - clearCache(); - } let key = '__express__' + req.query.url; - let longCache = checkForLongCachedRequests(req.query.url); - let cachedBody = mcache.get(key); const url = new URL.parse(req.query.url); const target = url.host + '/' + url.pathname.split('/')[1]; const scheme = url.protocol.replace(':', ''); - if (cachedBody) { + if (lruCache.has(key)) { +// console.log( key, "hit") const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'}); - res.send(JSON.parse(cachedBody)); + res.send(JSON.parse(lruCache.get(key))); responses.inc({scheme: scheme, target: target, code: res.statusCode}); end(); } else { - const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'}); - res.sendResponse = res.send; - res.send = (body) => { - let alreadyCached = !!mcache.get(key); - entries.set(mcache.memsize()); - if (!alreadyCached) { - responses.inc({scheme: scheme, target: target, code: res.statusCode}); - end(); - } - if (res.statusCode === 200) { - // console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url); - mcache.put(key, body, longCache ? expireLong : expireShort, () => { - entries.set(mcache.memsize()); - }); - } - res.sendResponse(body); - }; + // console.log( key, "miss") + const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'}); + res.sendResponse = res.send; + res.send = (body) => { + if(isAllowedToBeCached(scheme, target)) { + let alreadyCached = lruCache.has(key); + entries.set(lruCache.size); + if (!alreadyCached) { + responses.inc({scheme: scheme, target: target, code: res.statusCode}); + end(); + } + if (res.statusCode === 200) { + lruCache.set(key, body); + entries.set(lruCache.size); + } + res.sendResponse(body); + }else{ + res.statusCode = 405; + res.sendResponse(JSON.stringify( {code: 405, message: "Method not Allowed"})); + } + + }; + next(); } } else { @@ -85,19 +83,26 @@ let cache = () => { } }; }; - +function isAllowedToBeCached(scheme, target){ + if(environment != "development"){ + return scheme.indexOf("https://")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".zenodo.org/api/") !=-1) + } else if(environment == "development"){ + return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1 + } + return true; +} app.get('/clear', (req, res) => { - let c = mcache.size(); + let c = lruCache.size; const url = req.query.url; let message = ""; if (url) { let key = '__express__' + req.query.url; - mcache.del(key); + lruCache.delete(key); message = "Delete entry with key " + url; - entries.set(mcache.size()); + entries.set(lruCache.size); } else { clearCache(); - message = "Delete " + c + " entries. Now there are: " + mcache.size() + message = "Delete " + c + " entries. Now there are: " + lruCache.size } res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); @@ -118,7 +123,6 @@ app.get('/get', cache(), cors(), (req, res) => { res.status(404).send(getResponse(404, "Not Found ")) //not found } else { request.get(url, function (err, response) { - // res.header("Access-Control-Allow-Origin", "http://localhost:3000"); res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); @@ -154,31 +158,29 @@ function getResponse(code, message) { function clearCache() { console.log("cache is cleared!"); - mcache.clear(); - entries.set(mcache.size()); + lruCache.clear(); + entries.set(lruCache.size); initCache(); } async function initCache() { try { - const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); - const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); - const additionalDataResponses = await Promise.all(additionalDataPromises); - console.log("Cache initialized!") + const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); + const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); + const additionalDataResponses = await Promise.all(additionalDataPromises); + console.log("Cache initialized group queries!") } catch (error) { - console.error('Error fetching data: Cache initialize failed', error.message); + console.error('Error fetching data: Cache initialize failed', error); + } + try{ + const additionalDataPromisesPreloadRequests = preloadRequests.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + url)); + const additionalDataResponsesPreloadRequests = await Promise.all(additionalDataPromisesPreloadRequests); + console.log("Cache initialized preload requests!") + } catch (error) { + console.error('Error fetching data: Cache initialize failed', error); } } -function checkForLongCachedRequests(url) { - let long = false; - longCachingRequests.forEach(partUrl => { - if (url.indexOf(partUrl) !== -1) { - long = true; - } - }); - return long; -} function resetAtMidnight() { console.log("Run Reset timer"); diff --git a/services/cache/mecache/package.json b/services/cache/mecache/package.json index 1118af1a..c5b9a32e 100644 --- a/services/cache/mecache/package.json +++ b/services/cache/mecache/package.json @@ -13,7 +13,7 @@ "axios": "^1.6.7", "cors": "^2.8.5", "express": "^4.18.2", - "memory-cache": "^0.2.0", + "lru-cache": "^10.2.0", "prom-client": "^11.3.0", "properties-reader": "^2.2.0", "superagent": "^5.0.5" diff --git a/services/cache/mecache/production-properties.file b/services/cache/mecache/production-properties.file index 40248189..7ff8eb33 100644 --- a/services/cache/mecache/production-properties.file +++ b/services/cache/mecache/production-properties.file @@ -1,2 +1,5 @@ port = 4000 utilsService = https://explore.openaire.eu/utils-service +environment = production +preloadRequests =https://services.openaire.eu/openaire/info/,https://services.openaire.eu/openaire/contexts/,https://services.openaire.eu/usagestats/allmetrics,https://services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json + diff --git a/services/cache/mecache/properties.file b/services/cache/mecache/properties.file index 6734892a..2ff599db 100644 --- a/services/cache/mecache/properties.file +++ b/services/cache/mecache/properties.file @@ -1,2 +1,4 @@ port = 3200 utilsService= http://scoobydoo.di.uoa.gr:8000 +environment = development +preloadRequests =https://dev-openaire.d4science.org/openaire/info/,https://dev-openaire.d4science.org/openaire/contexts/,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:provenanceActions.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:publication_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:languages.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:access_modes.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:countries.json From 9952b55f2c740095aeab46d26ca69258c76076d9 Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Thu, 8 Feb 2024 17:55:22 +0200 Subject: [PATCH 07/10] [develop | DONE | CHANGED]: cache.js: Added "lab.idiap.ch/enermaps" in "isAllowedToBeCached()" method. --- services/cache/mecache/cache.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index e457a522..0c209f20 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -85,9 +85,9 @@ let cache = () => { }; function isAllowedToBeCached(scheme, target){ if(environment != "development"){ - return scheme.indexOf("https://")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".zenodo.org/api/") !=-1) + return scheme.indexOf("https://")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".zenodo.org/api/") !=-1 || target.indexOf("lab.idiap.ch/enermaps" != -1)) } else if(environment == "development"){ - return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1 + return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1 || target.indexOf("lab.idiap.ch/enermaps") != -1 } return true; } From a7d0cc5d4e8b3a48b7a8d2414120a9b227ab85b7 Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Thu, 8 Feb 2024 17:56:42 +0200 Subject: [PATCH 08/10] [develop | DONE | CHANGED]: app.module.ts: Removed CacheInterceptorService from EXPLORE. --- explore/src/app/app.module.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/explore/src/app/app.module.ts b/explore/src/app/app.module.ts index 81d561fc..1704868f 100755 --- a/explore/src/app/app.module.ts +++ b/explore/src/app/app.module.ts @@ -39,11 +39,11 @@ import {AlertModalModule} from './openaireLibrary/utils/modal/alertModal.module' exports: [AppComponent], providers: [ {provide: APP_ID, useValue: 'explore'}, - { - provide: HTTP_INTERCEPTORS, - useClass: CacheInterceptorService, - multi: true - }, + // { + // provide: HTTP_INTERCEPTORS, + // useClass: CacheInterceptorService, + // multi: true + // }, { provide: HTTP_INTERCEPTORS, useClass: HttpInterceptorService, From d639261fbeb199252bd391351d3f4261dab8f67c Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Thu, 8 Feb 2024 18:46:45 +0200 Subject: [PATCH 09/10] [develop | WIP | FIXED]: Added correct alt on images, width and height in FoS and SDG images and lazy loading everywhere. --- explore/src/app/home/home.component.html | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/explore/src/app/home/home.component.html b/explore/src/app/home/home.component.html index 1f41806a..8c8655fd 100644 --- a/explore/src/app/home/home.component.html +++ b/explore/src/app/home/home.component.html @@ -47,7 +47,7 @@ - SDGs logo + SDGs logo Sustainable Development Goals (SDGs) @@ -55,7 +55,7 @@ - FOS logo + FOS logo Fields of Science (FoS) @@ -64,8 +64,8 @@
- - + researchers + search
@@ -156,16 +156,16 @@ alt="ipad" loading="lazy"> - ipad + Deposit search - ipad + Deposit in OpenAIRE or Zenodo - ipad + Search links - ipad + link @@ -238,7 +238,7 @@
ipad + src="assets/common-assets/common/graph-nodes.svg" alt="Graph nodes" loading="lazy"> From 1d7a7ae2c4adc7f32a0606f00e3e6a53502393fc Mon Sep 17 00:00:00 2001 From: "konstantina.galouni" Date: Thu, 8 Feb 2024 18:48:22 +0200 Subject: [PATCH 10/10] Updating openaireLibrary & openaire-theme --- explore/src/app/openaireLibrary | 2 +- explore/src/assets/openaire-theme | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/explore/src/app/openaireLibrary b/explore/src/app/openaireLibrary index fbe514b7..a7294685 160000 --- a/explore/src/app/openaireLibrary +++ b/explore/src/app/openaireLibrary @@ -1 +1 @@ -Subproject commit fbe514b74086c61dbfc3b8647b9eea458b34139d +Subproject commit a7294685fb4ad530d1d1fbca75edd5bb0f691eff diff --git a/explore/src/assets/openaire-theme b/explore/src/assets/openaire-theme index 52eca9ab..419d7e20 160000 --- a/explore/src/assets/openaire-theme +++ b/explore/src/assets/openaire-theme @@ -1 +1 @@ -Subproject commit 52eca9abe84c941a7e78a5690f273d8c8e3194f9 +Subproject commit 419d7e200b60639ebcb9660c0e77b06a02504caa