[ develop | DONE | CHANGED] Cache: change to LRU cache, remove short/long cache related code and properties, add preload requests in properties to load on init cache (on start, on clear and on daily reset)
This commit is contained in:
parent
7761eeeedc
commit
b0db89f826
|
@ -1,2 +1,4 @@
|
||||||
port = 4000
|
port = 4000
|
||||||
utilsService = https://beta.explore.openaire.eu/utils-service
|
utilsService = https://beta.explore.openaire.eu/utils-service
|
||||||
|
environment = beta
|
||||||
|
preloadRequests =https://beta.services.openaire.eu/openaire/info/,https://beta.services.openaire.eu/openaire/contexts/,https://beta.services.openaire.eu/usagestats/allmetrics,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://beta.services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json
|
||||||
|
|
|
@ -2,22 +2,19 @@
|
||||||
const axios = require('axios');
|
const axios = require('axios');
|
||||||
let express = require('express');
|
let express = require('express');
|
||||||
let app = express();
|
let app = express();
|
||||||
let mcache = require('memory-cache');
|
const { LRUCache } = require('lru-cache');
|
||||||
const request = require('superagent');
|
const request = require('superagent');
|
||||||
const prom = require('prom-client');
|
const prom = require('prom-client');
|
||||||
const URL = require('url');
|
const URL = require('url');
|
||||||
var PropertiesReader = require('properties-reader');
|
const PropertiesReader = require('properties-reader');
|
||||||
var properties = PropertiesReader('./properties.file');
|
const properties = PropertiesReader('./properties.file');
|
||||||
const expireShort = 60 * 60 * 1000 // 1 hour //2 * 60 * 1000; //2mins
|
const environment = properties.get("environment");
|
||||||
const expireLong = 24 * 60 * 60 * 1000; //24 hours
|
const preloadRequests = properties.get("preloadRequests").split(',');;
|
||||||
const cacheMaxSize = 500;
|
|
||||||
const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent",
|
const cacheMaxSize = 300;
|
||||||
"/provision/mvc/vocabularies/", "/pages?page_route=",
|
|
||||||
"/allmetrics", "/countryusagestats/", "/openaire/info",
|
|
||||||
"/api/communities/", "/openaire/contexts/", "/utils-service/explore/home"];
|
|
||||||
let cors = require('cors');
|
let cors = require('cors');
|
||||||
app.use(cors());
|
app.use(cors());
|
||||||
|
const lruCache = new LRUCache({ max: cacheMaxSize });
|
||||||
const register = new prom.Registry();
|
const register = new prom.Registry();
|
||||||
prom.collectDefaultMetrics({register: register});
|
prom.collectDefaultMetrics({register: register});
|
||||||
|
|
||||||
|
@ -45,39 +42,40 @@ const histogram = new prom.Histogram({
|
||||||
let cache = () => {
|
let cache = () => {
|
||||||
return (req, res, next) => {
|
return (req, res, next) => {
|
||||||
if (req.query.url) {
|
if (req.query.url) {
|
||||||
if (mcache.memsize() > cacheMaxSize) {
|
|
||||||
console.log("Max cache size reached!" + cacheMaxSize);
|
|
||||||
clearCache();
|
|
||||||
}
|
|
||||||
let key = '__express__' + req.query.url;
|
let key = '__express__' + req.query.url;
|
||||||
let longCache = checkForLongCachedRequests(req.query.url);
|
|
||||||
let cachedBody = mcache.get(key);
|
|
||||||
const url = new URL.parse(req.query.url);
|
const url = new URL.parse(req.query.url);
|
||||||
const target = url.host + '/' + url.pathname.split('/')[1];
|
const target = url.host + '/' + url.pathname.split('/')[1];
|
||||||
const scheme = url.protocol.replace(':', '');
|
const scheme = url.protocol.replace(':', '');
|
||||||
if (cachedBody) {
|
if (lruCache.has(key)) {
|
||||||
|
// console.log( key, "hit")
|
||||||
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});
|
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});
|
||||||
res.send(JSON.parse(cachedBody));
|
res.send(JSON.parse(lruCache.get(key)));
|
||||||
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
||||||
end();
|
end();
|
||||||
} else {
|
} else {
|
||||||
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});
|
// console.log( key, "miss")
|
||||||
res.sendResponse = res.send;
|
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});
|
||||||
res.send = (body) => {
|
res.sendResponse = res.send;
|
||||||
let alreadyCached = !!mcache.get(key);
|
res.send = (body) => {
|
||||||
entries.set(mcache.memsize());
|
if(isAllowedToBeCached(scheme, target)) {
|
||||||
if (!alreadyCached) {
|
let alreadyCached = lruCache.has(key);
|
||||||
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
entries.set(lruCache.size);
|
||||||
end();
|
if (!alreadyCached) {
|
||||||
}
|
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
||||||
if (res.statusCode === 200) {
|
end();
|
||||||
// console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url);
|
}
|
||||||
mcache.put(key, body, longCache ? expireLong : expireShort, () => {
|
if (res.statusCode === 200) {
|
||||||
entries.set(mcache.memsize());
|
lruCache.set(key, body);
|
||||||
});
|
entries.set(lruCache.size);
|
||||||
}
|
}
|
||||||
res.sendResponse(body);
|
res.sendResponse(body);
|
||||||
};
|
}else{
|
||||||
|
res.statusCode = 405;
|
||||||
|
res.sendResponse(JSON.stringify( {code: 405, message: "Method not Allowed"}));
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
next();
|
next();
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
@ -85,19 +83,26 @@ let cache = () => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
function isAllowedToBeCached(scheme, target){
|
||||||
|
if(environment != "development"){
|
||||||
|
return scheme.indexOf("https://")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".zenodo.org/api/") !=-1)
|
||||||
|
} else if(environment == "development"){
|
||||||
|
return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
app.get('/clear', (req, res) => {
|
app.get('/clear', (req, res) => {
|
||||||
let c = mcache.size();
|
let c = lruCache.size;
|
||||||
const url = req.query.url;
|
const url = req.query.url;
|
||||||
let message = "";
|
let message = "";
|
||||||
if (url) {
|
if (url) {
|
||||||
let key = '__express__' + req.query.url;
|
let key = '__express__' + req.query.url;
|
||||||
mcache.del(key);
|
lruCache.delete(key);
|
||||||
message = "Delete entry with key " + url;
|
message = "Delete entry with key " + url;
|
||||||
entries.set(mcache.size());
|
entries.set(lruCache.size);
|
||||||
} else {
|
} else {
|
||||||
clearCache();
|
clearCache();
|
||||||
message = "Delete " + c + " entries. Now there are: " + mcache.size()
|
message = "Delete " + c + " entries. Now there are: " + lruCache.size
|
||||||
}
|
}
|
||||||
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
||||||
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
||||||
|
@ -118,7 +123,6 @@ app.get('/get', cache(), cors(), (req, res) => {
|
||||||
res.status(404).send(getResponse(404, "Not Found ")) //not found
|
res.status(404).send(getResponse(404, "Not Found ")) //not found
|
||||||
} else {
|
} else {
|
||||||
request.get(url, function (err, response) {
|
request.get(url, function (err, response) {
|
||||||
// res.header("Access-Control-Allow-Origin", "http://localhost:3000");
|
|
||||||
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
||||||
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
||||||
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
||||||
|
@ -154,31 +158,29 @@ function getResponse(code, message) {
|
||||||
|
|
||||||
function clearCache() {
|
function clearCache() {
|
||||||
console.log("cache is cleared!");
|
console.log("cache is cleared!");
|
||||||
mcache.clear();
|
lruCache.clear();
|
||||||
entries.set(mcache.size());
|
entries.set(lruCache.size);
|
||||||
initCache();
|
initCache();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function initCache() {
|
async function initCache() {
|
||||||
try {
|
try {
|
||||||
const requests = await axios.get(properties.get('utilsService') + '/grouped-requests');
|
const requests = await axios.get(properties.get('utilsService') + '/grouped-requests');
|
||||||
const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url));
|
const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url));
|
||||||
const additionalDataResponses = await Promise.all(additionalDataPromises);
|
const additionalDataResponses = await Promise.all(additionalDataPromises);
|
||||||
console.log("Cache initialized!")
|
console.log("Cache initialized group queries!")
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error fetching data: Cache initialize failed', error.message);
|
console.error('Error fetching data: Cache initialize failed', error);
|
||||||
|
}
|
||||||
|
try{
|
||||||
|
const additionalDataPromisesPreloadRequests = preloadRequests.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + url));
|
||||||
|
const additionalDataResponsesPreloadRequests = await Promise.all(additionalDataPromisesPreloadRequests);
|
||||||
|
console.log("Cache initialized preload requests!")
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error fetching data: Cache initialize failed', error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkForLongCachedRequests(url) {
|
|
||||||
let long = false;
|
|
||||||
longCachingRequests.forEach(partUrl => {
|
|
||||||
if (url.indexOf(partUrl) !== -1) {
|
|
||||||
long = true;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return long;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resetAtMidnight() {
|
function resetAtMidnight() {
|
||||||
console.log("Run Reset timer");
|
console.log("Run Reset timer");
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
"axios": "^1.6.7",
|
"axios": "^1.6.7",
|
||||||
"cors": "^2.8.5",
|
"cors": "^2.8.5",
|
||||||
"express": "^4.18.2",
|
"express": "^4.18.2",
|
||||||
"memory-cache": "^0.2.0",
|
"lru-cache": "^10.2.0",
|
||||||
"prom-client": "^11.3.0",
|
"prom-client": "^11.3.0",
|
||||||
"properties-reader": "^2.2.0",
|
"properties-reader": "^2.2.0",
|
||||||
"superagent": "^5.0.5"
|
"superagent": "^5.0.5"
|
||||||
|
|
|
@ -1,2 +1,5 @@
|
||||||
port = 4000
|
port = 4000
|
||||||
utilsService = https://explore.openaire.eu/utils-service
|
utilsService = https://explore.openaire.eu/utils-service
|
||||||
|
environment = production
|
||||||
|
preloadRequests =https://services.openaire.eu/openaire/info/,https://services.openaire.eu/openaire/contexts/,https://services.openaire.eu/usagestats/allmetrics,https://services.openaire.eu/provision/mvc/vocabularies/dnet:provenanceActions.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:relation_relClass.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:publication_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:languages.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:access_modes.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://services.openaire.eu/provision/mvc/vocabularies/dnet:countries.json
|
||||||
|
|
||||||
|
|
|
@ -1,2 +1,4 @@
|
||||||
port = 3200
|
port = 3200
|
||||||
utilsService= http://scoobydoo.di.uoa.gr:8000
|
utilsService= http://scoobydoo.di.uoa.gr:8000
|
||||||
|
environment = development
|
||||||
|
preloadRequests =https://dev-openaire.d4science.org/openaire/info/,https://dev-openaire.d4science.org/openaire/contexts/,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:provenanceActions.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:subject_classification_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:publication_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:dataCite_resource.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:languages.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:access_modes.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasource_typologies.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:datasourceCompatibilityLevel.json,https://dev-openaire.d4science.org/provision/mvc/vocabularies/dnet:countries.json
|
||||||
|
|
Loading…
Reference in New Issue