explore-services/services/cache/mecache/cache.js

185 lines
6.4 KiB
JavaScript

'use strict';
let express = require('express');
let app = express();
let mcache = require('memory-cache');
const request = require('superagent');
const prom = require('prom-client');
const URL = require('url');
var PropertiesReader = require('properties-reader');
var properties = PropertiesReader('./properties.file');
const expireShort = 2 * 60 * 1000; //2mins
const expireLong = 24 * 60 * 60 * 1000; //24 hours
const cacheMaxSize = 500;
const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent",
"/provision/mvc/vocabularies/", "/pages?page_route=",
"/allmetrics", "/countryusagestats/", "/openaire/info",
"/api/communities/", "/openaire/contexts/"];
let cors = require('cors');
app.use(cors());
const register = new prom.Registry();
prom.collectDefaultMetrics({register: register});
const responses = new prom.Counter({
name: 'cache_http_responses_total',
help: 'A counter for cache response codes for every API request.',
labelNames: ['scheme', 'target', 'code'],
registers: [register]
});
const entries = new prom.Gauge({
name: 'cache_used_entries',
help: 'A counter to count cache entries',
registers: [register]
});
const histogram = new prom.Histogram({
name: 'cache_http_request_duration_seconds',
help: 'A Histogram for cache. Providing information about a cache request and load latency in seconds.',
labelNames: ['scheme', 'target', 'cache'],
registers: [register],
buckets: [0.1, 0.2, 0.5, 1, 2]
});
let cache = () => {
return (req, res, next) => {
if (req.query.url) {
if (mcache.memsize() > cacheMaxSize) {
console.log("Max cache size reached!" + cacheMaxSize);
clearCache();
}
let key = '__express__' + req.query.url;
let longCache = checkForLongCachedRequests(req.query.url);
let cachedBody = mcache.get(key);
const url = new URL.parse(req.query.url);
const target = url.host + '/' + url.pathname.split('/')[1];
const scheme = url.protocol.replace(':', '');
if (cachedBody) {
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});
res.send(JSON.parse(cachedBody));
responses.inc({scheme: scheme, target: target, code: res.statusCode});
end();
} else {
const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});
res.sendResponse = res.send;
res.send = (body) => {
let alreadyCached = !!mcache.get(key);
entries.set(mcache.memsize());
if (!alreadyCached) {
responses.inc({scheme: scheme, target: target, code: res.statusCode});
end();
}
if (res.statusCode === 200) {
// console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url);
mcache.put(key, body, longCache ? expireLong : expireShort, () => {
entries.set(mcache.memsize());
});
}
res.sendResponse(body);
};
next();
}
} else {
next();
}
};
};
app.get('/clear', (req, res) => {
let c = mcache.size();
const url = req.query.url;
let message = "";
if (url) {
let key = '__express__' + req.query.url;
mcache.del(key);
message = "Delete entry with key " + url;
entries.set(mcache.size());
} else {
clearCache();
message = "Delete " + c + " entries. Now there are: " + mcache.size()
}
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
res.header("Content-Type", "application/json");
res.status(200).send(getResponse(200, message));
});
app.get('/metrics', (req, res) => {
res.set('Content-Type', register.contentType);
res.end(register.metrics());
});
app.get('/get', cache(), cors(), (req, res) => {
setTimeout(() => {
const url = (req.query) ? req.query.url : null;
if (!url) {
res.status(404).send(getResponse(404, "Not Found ")) //not found
} else {
request.get(url, function (err, response) {
// res.header("Access-Control-Allow-Origin", "http://localhost:3000");
res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length");
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
res.header("Access-Control-Allow-Methods", "GET, OPTIONS");
res.header("Content-Type", "application/json");
if (!response && err) {
res.status(500).send(getResponse(500, "An error occurred for " + url))
} else {
res.status(response.status).send(response.body);
}
})
}
})
});
app.use((req, res) => {
res.status(404).send(getResponse(404, "Not Found")); //not found
});
const server = app.listen(properties.get('port'), function () {
console.log(`Example app listening on port`, server.address().port)
//run the timer
resetAtMidnight();
});
function getResponse(code, message) {
var response = {};
response["code"] = code;
response["message"] = message;
return response;
}
function clearCache() {
console.log("cache is cleared!");
mcache.clear();
entries.set(mcache.size());
}
function checkForLongCachedRequests(url) {
let long = false;
longCachingRequests.forEach(partUrl => {
if (url.indexOf(partUrl) !== -1) {
long = true;
}
});
return long;
}
function resetAtMidnight() {
console.log("Run Reset timer");
var now = new Date();
var night = new Date(
now.getFullYear(),
now.getMonth(),
now.getDate() + 1, // the next day, ...
0, 0, 0 // ...at 00:00:00 hours
);
var msToMidnight = night.getTime() - now.getTime();
setTimeout(function () {
clearCache(); // <-- This is the function being called at midnight.
resetAtMidnight(); // Then, reset again next midnight.
}, msToMidnight);
}