'use strict'; let express = require('express'); let app = express(); let mcache = require('memory-cache'); const request = require('superagent'); const prom = require('prom-client'); const URL = require('url'); const expireShort = 2 * 60 * 1000; //2mins const expireLong = 24 * 60 * 60 * 1000; //24 hours const cacheMaxSize = 500; const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent", "/provision/mvc/vocabularies/", "/pages?page_route=", "/allmetrics", "/countryusagestats/", "/openaire/info", "/api/communities/", "/openaire/contexts/"]; let cors = require('cors'); app.use(cors()); const register = new prom.Registry(); prom.collectDefaultMetrics({register: register}); const responses = new prom.Counter({ name: 'cache_http_responses_total', help: 'A counter for cache response codes for every API request.', labelNames: ['scheme', 'target', 'code'], registers: [register] }); const entries = new prom.Gauge({ name: 'cache_used_entries', help: 'A counter to count cache entries', registers: [register] }); const histogram = new prom.Histogram({ name: 'cache_http_request_duration_seconds', help: 'A Histogram for cache. Providing information about a cache request and load latency in seconds.', labelNames: ['scheme', 'target', 'cache'], registers: [register], buckets: [0.1, 0.2, 0.5, 1, 2] }); let cache = () => { return (req, res, next) => { if (req.query.url) { if (mcache.memsize() > cacheMaxSize) { console.log("Max cache size reached!" + cacheMaxSize); clearCache(); } let key = '__express__' + req.query.url; let longCache = checkForLongCachedRequests(req.query.url); let cachedBody = mcache.get(key); const url = new URL.parse(req.query.url); const target = url.host + '/' + url.pathname.split('/')[1]; const scheme = url.protocol.replace(':', ''); if (cachedBody) { const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'}); res.send(JSON.parse(cachedBody)); responses.inc({scheme: scheme, target: target, code: res.statusCode}); end(); } else { const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'}); res.sendResponse = res.send; res.send = (body) => { let alreadyCached = !!mcache.get(key); entries.set(mcache.memsize()); if (!alreadyCached) { responses.inc({scheme: scheme, target: target, code: res.statusCode}); end(); } if (res.statusCode === 200) { // console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url); mcache.put(key, body, longCache ? expireLong : expireShort, () => { entries.set(mcache.memsize()); }); } res.sendResponse(body); }; next(); } } else { next(); } }; }; app.get('/clear', (req, res) => { let c = mcache.size(); const url = req.query.url; let message = ""; if (url) { let key = '__express__' + req.query.url; mcache.del(key); message = "Delete entry with key " + url; entries.set(mcache.size()); } else { clearCache(); message = "Delete " + c + " entries. Now there are: " + mcache.size() } res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Content-Type", "application/json"); res.status(200).send(getResponse(200, message)); }); app.get('/metrics', (req, res) => { res.set('Content-Type', register.contentType); res.end(register.metrics()); }); app.get('/get', cache(), cors(), (req, res) => { setTimeout(() => { const url = (req.query) ? req.query.url : null; if (!url) { res.status(404).send(getResponse(404, "Not Found ")) //not found } else { request.get(url, function (err, response) { // res.header("Access-Control-Allow-Origin", "http://localhost:3000"); res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Content-Type", "application/json"); if (!response && err) { res.status(500).send(getResponse(500, "An error occurred for " + url)) } else { res.status(response.status).send(response.body); } }) } }) }); app.use((req, res) => { res.status(404).send(getResponse(404, "Not Found")); //not found }); app.listen((process.env.PORT) ? process.env.PORT : 3000, function () { console.log(`Example app listening on port ${(process.env.PORT) ? process.env.PORT : 3000}!`) //run the timer resetAtMidnight(); }); function getResponse(code, message) { var response = {}; response["code"] = code; response["message"] = message; return response; } function clearCache() { console.log("cache is cleared!"); mcache.clear(); entries.set(mcache.size()); } function checkForLongCachedRequests(url) { let long = false; longCachingRequests.forEach(partUrl => { if (url.indexOf(partUrl) !== -1) { long = true; } }); return long; } function resetAtMidnight() { console.log("Run Reset timer"); var now = new Date(); var night = new Date( now.getFullYear(), now.getMonth(), now.getDate() + 1, // the next day, ... 0, 0, 0 // ...at 00:00:00 hours ); var msToMidnight = night.getTime() - now.getTime(); setTimeout(function () { clearCache(); // <-- This is the function being called at midnight. resetAtMidnight(); // Then, reset again next midnight. }, msToMidnight); }