234 lines
9.2 KiB
JavaScript
234 lines
9.2 KiB
JavaScript
'use strict';
|
|
const axios = require('axios');
|
|
let express = require('express');
|
|
let app = express();
|
|
const { LRUCache } = require('lru-cache');
|
|
const request = require('superagent');
|
|
const prom = require('prom-client');
|
|
const URL = require('url');
|
|
let compression = require("compression");
|
|
const PropertiesReader = require('properties-reader');
|
|
const properties = PropertiesReader('./properties.file');
|
|
const environment = properties.get("environment");
|
|
const preloadRequests = properties.get("preloadRequests").split(',');;
|
|
var accesslog = require('access-log');
|
|
|
|
const cacheMaxSize = 1000;
|
|
let cors = require('cors');
|
|
app.use(cors());
|
|
app.use(compression());
|
|
const lruCache = new LRUCache({ max: cacheMaxSize });
|
|
const register = new prom.Registry();
|
|
prom.collectDefaultMetrics({register: register});
|
|
|
|
const responses = new prom.Counter({
|
|
name: 'cache_http_responses_total',
|
|
help: 'A counter for cache response codes for every API request.',
|
|
labelNames: ['scheme', 'target', 'code'],
|
|
registers: [register]
|
|
});
|
|
|
|
const entries = new prom.Gauge({
|
|
name: 'cache_used_entries',
|
|
help: 'A counter to count cache entries',
|
|
registers: [register]
|
|
});
|
|
|
|
const histogram = new prom.Histogram({
|
|
name: 'cache_http_request_duration_seconds',
|
|
help: 'A Histogram for cache. Providing information about a cache request and load latency in seconds.',
|
|
labelNames: ['scheme', 'target', 'cache'],
|
|
registers: [register],
|
|
buckets: [0.1, 0.2, 0.5, 1, 2]
|
|
});
|
|
|
|
let cache = () => {
|
|
return (req, res, next) => {
|
|
if (req.query.url) {
|
|
let key = req.query.url;
|
|
const url = new URL.parse(req.query.url);
|
|
// console.log(req.headers.origin, req.headers.referrer, req.headers.origin)
|
|
const cacheControlHeader = req.headers['cache-control'];
|
|
// Log the Cache-Control header
|
|
// console.log('Cache-Control header sent by client:', cacheControlHeader);
|
|
let forceReload = req.query.forceReload && req.query.forceReload == 'true'?true:false;
|
|
forceReload = forceReload || (cacheControlHeader && (cacheControlHeader.indexOf("no-cache") || cacheControlHeader.indexOf("no-store") || cacheControlHeader.indexOf("must-revalidate")))?true:false;
|
|
|
|
const target = url.host + '/' + url.pathname.split('/')[1];
|
|
const scheme = url.protocol.replace(':', '');
|
|
if (lruCache.has(key) && !forceReload) {
|
|
// console.log( key, "hit")
|
|
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});
|
|
res.send(JSON.parse(lruCache.get(key)));
|
|
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
|
end();
|
|
} else {
|
|
// console.log( key, "miss", forceReload)
|
|
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});
|
|
res.sendResponse = res.send;
|
|
res.send = (body) => {
|
|
if(isAllowedToBeCached(decodeURI(scheme), decodeURI(target))) {
|
|
let alreadyCached = lruCache.has(key);
|
|
entries.set(lruCache.size);
|
|
if (!alreadyCached) {
|
|
responses.inc({scheme: scheme, target: target, code: res.statusCode});
|
|
end();
|
|
}
|
|
if (res.statusCode === 200) {
|
|
lruCache.set(key, body);
|
|
entries.set(lruCache.size);
|
|
}
|
|
res.sendResponse(body);
|
|
}else{
|
|
res.statusCode = 405;
|
|
res.sendResponse(JSON.stringify( {code: 405, message: "Method not Allowed"}));
|
|
}
|
|
|
|
};
|
|
accesslogCustomFormat(req, res);
|
|
next();
|
|
}
|
|
} else {
|
|
accesslogCustomFormat(req, res);
|
|
next();
|
|
}
|
|
};
|
|
};
|
|
function isAllowedToBeCached(scheme, target){
|
|
if(environment != "development"){
|
|
return scheme.indexOf("https")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf("zenodo.org/api") !=-1 || target.indexOf("lab.idiap.ch/enermaps" != -1))
|
|
} else if(environment == "development"){
|
|
return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("zenodo.org/api") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1 || target.indexOf("lab.idiap.ch/enermaps") != -1
|
|
}
|
|
return true;
|
|
}
|
|
app.get('/clear', (req, res) => {
|
|
let c = lruCache.size;
|
|
const url = req.query.url;
|
|
let message = "";
|
|
if (url) {
|
|
let key = req.query.url;
|
|
lruCache.delete(key);
|
|
message = "Delete entry with key " + url;
|
|
entries.set(lruCache.size);
|
|
} else {
|
|
clearCache();
|
|
message = "Delete " + c + " entries. Now there are: " + lruCache.size
|
|
}
|
|
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
res.header("Content-Type", "application/json");
|
|
accesslogCustomFormat(req, res);
|
|
res.status(200).send(getResponse(200, message));
|
|
|
|
});
|
|
app.get('/metrics', (req, res) => {
|
|
res.set('Content-Type', register.contentType);
|
|
accesslogCustomFormat(req, res);
|
|
res.end(register.metrics());
|
|
});
|
|
|
|
app.get('/info', (req, res) => {
|
|
accesslogCustomFormat(req, res);
|
|
res.status(200).send(getResponse(200, {size:lruCache.size, keys: Array.from(lruCache.keys())}));
|
|
});
|
|
|
|
app.get('/get', cache(), cors(), (req, res) => {
|
|
setTimeout(() => {
|
|
const url = (req.query) ? req.query.url : null;
|
|
if (!url) {
|
|
res.status(404).send(getResponse(404, "Not Found ")) //not found
|
|
} else {
|
|
request.get(url, function (err, response) {
|
|
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
res.header("Content-Type", "application/json");
|
|
if (!response && err) {
|
|
res.status(500).send(getResponse(500, "An error occurred for " + url))
|
|
} else {
|
|
res.status(response.status).send(response.body);
|
|
}
|
|
})
|
|
}
|
|
})
|
|
});
|
|
|
|
|
|
app.use((req, res) => {
|
|
res.status(404).send(getResponse(404, "Not Found")); //not found
|
|
});
|
|
|
|
const server = app.listen(properties.get('port'), function () {
|
|
console.log(`Example app listening on port`, server.address().port)
|
|
//run the timer
|
|
resetAtMidnight();
|
|
initCache();
|
|
});
|
|
|
|
function getResponse(code, message) {
|
|
var response = {};
|
|
response["code"] = code;
|
|
response["message"] = message;
|
|
return response;
|
|
}
|
|
|
|
function clearCache() {
|
|
console.log("cache is cleared!");
|
|
lruCache.clear();
|
|
entries.set(lruCache.size);
|
|
initCache();
|
|
}
|
|
|
|
async function initCache() {
|
|
try {
|
|
const requests = await axios.get(properties.get('utilsService') + '/grouped-requests');
|
|
const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url));
|
|
const additionalDataResponses = await Promise.all(additionalDataPromises);
|
|
console.log("Cache initialized group queries!")
|
|
} catch (error) {
|
|
console.error('Error fetching data: Cache initialize failed', error);
|
|
}
|
|
try{
|
|
const additionalDataPromisesPreloadRequests = preloadRequests.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + url));
|
|
const additionalDataResponsesPreloadRequests = await Promise.all(additionalDataPromisesPreloadRequests);
|
|
console.log("Cache initialized preload requests!")
|
|
} catch (error) {
|
|
console.error('Error fetching data: Cache initialize failed', error);
|
|
}
|
|
}
|
|
|
|
|
|
function resetAtMidnight() {
|
|
console.log("Run Reset timer");
|
|
var now = new Date();
|
|
var night = new Date(
|
|
now.getFullYear(),
|
|
now.getMonth(),
|
|
now.getDate() + 1, // the next day, ...
|
|
0, 0, 0 // ...at 00:00:00 hours
|
|
);
|
|
var msToMidnight = night.getTime() - now.getTime();
|
|
setTimeout(function () {
|
|
clearCache(); // <-- This is the function being called at midnight.
|
|
resetAtMidnight(); // Then, reset again next midnight.
|
|
}, msToMidnight);
|
|
}
|
|
|
|
function accesslogCustomFormat(req, res){
|
|
// default format
|
|
accesslog(req, res);
|
|
// ::ffff:195.134.66.178 - - [15/Jul/2024:11:24:35 +0300] "GET /clear HTTP/1.1" 200 59 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0"
|
|
|
|
//custom format
|
|
/*
|
|
accesslog(req, res,
|
|
{
|
|
userID: function (req) { return req.user; },
|
|
format : 'url=":url" method=":method" statusCode=":statusCode" delta=":delta" ip=":ip"'
|
|
});
|
|
*/
|
|
// url="/clear" method="GET" statusCode="200" delta="2" ip="::ffff:195.134.66.178"
|
|
}
|