From b0db89f826149bff2d7941abb825f963557f645f Mon Sep 17 00:00:00 2001 From: argirok Date: Thu, 8 Feb 2024 15:58:43 +0200 Subject: [PATCH] [ develop | DONE | CHANGED] Cache: change to LRU cache, remove short/long cache related code and properties, add preload requests in properties to load on init cache (on start, on clear and on daily reset) --- services/cache/mecache/beta-properties.file | 2 + services/cache/mecache/cache.js | 118 +++++++++--------- services/cache/mecache/package.json | 2 +- .../cache/mecache/production-properties.file | 3 + services/cache/mecache/properties.file | 2 + 5 files changed, 68 insertions(+), 59 deletions(-) diff --git a/services/cache/mecache/beta-properties.file b/services/cache/mecache/beta-properties.file index d39459e8..d34c784c 100644 --- a/services/cache/mecache/beta-properties.file +++ b/services/cache/mecache/beta-properties.file @@ -1,2 +1,4 @@ port = 4000 utilsService = https://beta.explore.openaire.eu/utils-service +environment = beta +preloadRequests =https://beta.services.openaire.eu/openaire/info/,https://beta.services.openaire.eu/openaire/contexts/,https://beta.services.openaire.eu/usagestats/allmetrics,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json diff --git a/services/cache/mecache/cache.js b/services/cache/mecache/cache.js index 11710b4e..e457a522 100644 --- a/services/cache/mecache/cache.js +++ b/services/cache/mecache/cache.js @@ -2,22 +2,19 @@ const axios = require('axios'); let express = require('express'); let app = express(); -let mcache = require('memory-cache'); +const { LRUCache } = require('lru-cache'); const request = require('superagent'); const prom = require('prom-client'); const URL = require('url'); -var PropertiesReader = require('properties-reader'); -var properties = PropertiesReader('./properties.file'); -const expireShort = 60 * 60 * 1000 // 1 hour //2 * 60 * 1000; //2mins -const expireLong = 24 * 60 * 60 * 1000; //24 hours -const cacheMaxSize = 500; -const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", - "/provision/mvc/vocabularies/", "/pages?page_route=", - "/allmetrics", "/countryusagestats/", "/openaire/info", - "/api/communities/", "/openaire/contexts/", "/utils-service/explore/home"]; +const PropertiesReader = require('properties-reader'); +const properties = PropertiesReader('./properties.file'); +const environment = properties.get("environment"); +const preloadRequests = properties.get("preloadRequests").split(',');; + +const cacheMaxSize = 300; let cors = require('cors'); app.use(cors()); - +const lruCache = new LRUCache({ max: cacheMaxSize }); const register = new prom.Registry(); prom.collectDefaultMetrics({register: register}); @@ -45,39 +42,40 @@ const histogram = new prom.Histogram({ let cache = () => { return (req, res, next) => { if (req.query.url) { - if (mcache.memsize() > cacheMaxSize) { - console.log("Max cache size reached!" + cacheMaxSize); - clearCache(); - } let key = '__express__' + req.query.url; - let longCache = checkForLongCachedRequests(req.query.url); - let cachedBody = mcache.get(key); const url = new URL.parse(req.query.url); const target = url.host + '/' + url.pathname.split('/')[1]; const scheme = url.protocol.replace(':', ''); - if (cachedBody) { + if (lruCache.has(key)) { +// console.log( key, "hit") const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'}); - res.send(JSON.parse(cachedBody)); + res.send(JSON.parse(lruCache.get(key))); responses.inc({scheme: scheme, target: target, code: res.statusCode}); end(); } else { - const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'}); - res.sendResponse = res.send; - res.send = (body) => { - let alreadyCached = !!mcache.get(key); - entries.set(mcache.memsize()); - if (!alreadyCached) { - responses.inc({scheme: scheme, target: target, code: res.statusCode}); - end(); - } - if (res.statusCode === 200) { - // console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url); - mcache.put(key, body, longCache ? expireLong : expireShort, () => { - entries.set(mcache.memsize()); - }); - } - res.sendResponse(body); - }; + // console.log( key, "miss") + const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'}); + res.sendResponse = res.send; + res.send = (body) => { + if(isAllowedToBeCached(scheme, target)) { + let alreadyCached = lruCache.has(key); + entries.set(lruCache.size); + if (!alreadyCached) { + responses.inc({scheme: scheme, target: target, code: res.statusCode}); + end(); + } + if (res.statusCode === 200) { + lruCache.set(key, body); + entries.set(lruCache.size); + } + res.sendResponse(body); + }else{ + res.statusCode = 405; + res.sendResponse(JSON.stringify( {code: 405, message: "Method not Allowed"})); + } + + }; + next(); } } else { @@ -85,19 +83,26 @@ let cache = () => { } }; }; - +function isAllowedToBeCached(scheme, target){ + if(environment != "development"){ + return scheme.indexOf("https://")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".zenodo.org/api/") !=-1) + } else if(environment == "development"){ + return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1 + } + return true; +} app.get('/clear', (req, res) => { - let c = mcache.size(); + let c = lruCache.size; const url = req.query.url; let message = ""; if (url) { let key = '__express__' + req.query.url; - mcache.del(key); + lruCache.delete(key); message = "Delete entry with key " + url; - entries.set(mcache.size()); + entries.set(lruCache.size); } else { clearCache(); - message = "Delete " + c + " entries. Now there are: " + mcache.size() + message = "Delete " + c + " entries. Now there are: " + lruCache.size } res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); @@ -118,7 +123,6 @@ app.get('/get', cache(), cors(), (req, res) => { res.status(404).send(getResponse(404, "Not Found ")) //not found } else { request.get(url, function (err, response) { - // res.header("Access-Control-Allow-Origin", "http://localhost:3000"); res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); @@ -154,31 +158,29 @@ function getResponse(code, message) { function clearCache() { console.log("cache is cleared!"); - mcache.clear(); - entries.set(mcache.size()); + lruCache.clear(); + entries.set(lruCache.size); initCache(); } async function initCache() { try { - const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); - const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); - const additionalDataResponses = await Promise.all(additionalDataPromises); - console.log("Cache initialized!") + const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); + const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); + const additionalDataResponses = await Promise.all(additionalDataPromises); + console.log("Cache initialized group queries!") } catch (error) { - console.error('Error fetching data: Cache initialize failed', error.message); + console.error('Error fetching data: Cache initialize failed', error); + } + try{ + const additionalDataPromisesPreloadRequests = preloadRequests.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + url)); + const additionalDataResponsesPreloadRequests = await Promise.all(additionalDataPromisesPreloadRequests); + console.log("Cache initialized preload requests!") + } catch (error) { + console.error('Error fetching data: Cache initialize failed', error); } } -function checkForLongCachedRequests(url) { - let long = false; - longCachingRequests.forEach(partUrl => { - if (url.indexOf(partUrl) !== -1) { - long = true; - } - }); - return long; -} function resetAtMidnight() { console.log("Run Reset timer"); diff --git a/services/cache/mecache/package.json b/services/cache/mecache/package.json index 1118af1a..c5b9a32e 100644 --- a/services/cache/mecache/package.json +++ b/services/cache/mecache/package.json @@ -13,7 +13,7 @@ "axios": "^1.6.7", "cors": "^2.8.5", "express": "^4.18.2", - "memory-cache": "^0.2.0", + "lru-cache": "^10.2.0", "prom-client": "^11.3.0", "properties-reader": "^2.2.0", "superagent": "^5.0.5" diff --git a/services/cache/mecache/production-properties.file b/services/cache/mecache/production-properties.file index 40248189..7ff8eb33 100644 --- a/services/cache/mecache/production-properties.file +++ b/services/cache/mecache/production-properties.file @@ -1,2 +1,5 @@ port = 4000 utilsService = https://explore.openaire.eu/utils-service +environment = production +preloadRequests =https://services.openaire.eu/openaire/info/,https://services.openaire.eu/openaire/contexts/,https://services.openaire.eu/usagestats/allmetrics,https://services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json + diff --git a/services/cache/mecache/properties.file b/services/cache/mecache/properties.file index 6734892a..2ff599db 100644 --- a/services/cache/mecache/properties.file +++ b/services/cache/mecache/properties.file @@ -1,2 +1,4 @@ port = 3200 utilsService= http://scoobydoo.di.uoa.gr:8000 +environment = development +preloadRequests =https://dev-openaire.d4science.org/openaire/info/,https://dev-openaire.d4science.org/openaire/contexts/,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:provenanceActions.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:publication_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:languages.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:access_modes.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:countries.json