'use strict'; const axios = require('axios'); let express = require('express'); let app = express(); const { LRUCache } = require('lru-cache'); const request = require('superagent'); const prom = require('prom-client'); const URL = require('url'); const PropertiesReader = require('properties-reader'); const properties = PropertiesReader('./properties.file'); const environment = properties.get("environment"); const preloadRequests = properties.get("preloadRequests").split(',');; const cacheMaxSize = 300; let cors = require('cors'); app.use(cors()); const lruCache = new LRUCache({ max: cacheMaxSize }); const register = new prom.Registry(); prom.collectDefaultMetrics({register: register}); const responses = new prom.Counter({ name: 'cache_http_responses_total', help: 'A counter for cache response codes for every API request.', labelNames: ['scheme', 'target', 'code'], registers: [register] }); const entries = new prom.Gauge({ name: 'cache_used_entries', help: 'A counter to count cache entries', registers: [register] }); const histogram = new prom.Histogram({ name: 'cache_http_request_duration_seconds', help: 'A Histogram for cache. Providing information about a cache request and load latency in seconds.', labelNames: ['scheme', 'target', 'cache'], registers: [register], buckets: [0.1, 0.2, 0.5, 1, 2] }); let cache = () => { return (req, res, next) => { if (req.query.url) { let key = '__express__' + req.query.url; const url = new URL.parse(req.query.url); const target = url.host + '/' + url.pathname.split('/')[1]; const scheme = url.protocol.replace(':', ''); if (lruCache.has(key)) { // console.log( key, "hit") const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'}); res.send(JSON.parse(lruCache.get(key))); responses.inc({scheme: scheme, target: target, code: res.statusCode}); end(); } else { // console.log( key, "miss") const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'}); res.sendResponse = res.send; res.send = (body) => { if(isAllowedToBeCached(scheme, target)) { let alreadyCached = lruCache.has(key); entries.set(lruCache.size); if (!alreadyCached) { responses.inc({scheme: scheme, target: target, code: res.statusCode}); end(); } if (res.statusCode === 200) { lruCache.set(key, body); entries.set(lruCache.size); } res.sendResponse(body); }else{ res.statusCode = 405; res.sendResponse(JSON.stringify( {code: 405, message: "Method not Allowed"})); } }; next(); } } else { next(); } }; }; function isAllowedToBeCached(scheme, target){ if(environment != "development"){ return scheme.indexOf("https://")!=-1 && ( target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".zenodo.org/api/") !=-1) } else if(environment == "development"){ return target.indexOf(".openaire.eu/") !=-1 || target.indexOf(".di.uoa.gr") !=-1 || target.indexOf("dev-openaire.d4science.org") !=-1 } return true; } app.get('/clear', (req, res) => { let c = lruCache.size; const url = req.query.url; let message = ""; if (url) { let key = '__express__' + req.query.url; lruCache.delete(key); message = "Delete entry with key " + url; entries.set(lruCache.size); } else { clearCache(); message = "Delete " + c + " entries. Now there are: " + lruCache.size } res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Content-Type", "application/json"); res.status(200).send(getResponse(200, message)); }); app.get('/metrics', (req, res) => { res.set('Content-Type', register.contentType); res.end(register.metrics()); }); app.get('/get', cache(), cors(), (req, res) => { setTimeout(() => { const url = (req.query) ? req.query.url : null; if (!url) { res.status(404).send(getResponse(404, "Not Found ")) //not found } else { request.get(url, function (err, response) { res.header("Access-Control-Allow-Headers", "Origin, Content-Type, Content-Length"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Access-Control-Allow-Methods", "GET, OPTIONS"); res.header("Content-Type", "application/json"); if (!response && err) { res.status(500).send(getResponse(500, "An error occurred for " + url)) } else { res.status(response.status).send(response.body); } }) } }) }); app.use((req, res) => { res.status(404).send(getResponse(404, "Not Found")); //not found }); const server = app.listen(properties.get('port'), function () { console.log(`Example app listening on port`, server.address().port) //run the timer resetAtMidnight(); initCache(); }); function getResponse(code, message) { var response = {}; response["code"] = code; response["message"] = message; return response; } function clearCache() { console.log("cache is cleared!"); lruCache.clear(); entries.set(lruCache.size); initCache(); } async function initCache() { try { const requests = await axios.get(properties.get('utilsService') + '/grouped-requests'); const additionalDataPromises = requests.data.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + properties.get('utilsService') + url)); const additionalDataResponses = await Promise.all(additionalDataPromises); console.log("Cache initialized group queries!") } catch (error) { console.error('Error fetching data: Cache initialize failed', error); } try{ const additionalDataPromisesPreloadRequests = preloadRequests.map((url) => axios.get('http://localhost:'+properties.get('port') + '/get?url=' + url)); const additionalDataResponsesPreloadRequests = await Promise.all(additionalDataPromisesPreloadRequests); console.log("Cache initialized preload requests!") } catch (error) { console.error('Error fetching data: Cache initialize failed', error); } } function resetAtMidnight() { console.log("Run Reset timer"); var now = new Date(); var night = new Date( now.getFullYear(), now.getMonth(), now.getDate() + 1, // the next day, ... 0, 0, 0 // ...at 00:00:00 hours ); var msToMidnight = night.getTime() - now.getTime(); setTimeout(function () { clearCache(); // <-- This is the function being called at midnight. resetAtMidnight(); // Then, reset again next midnight. }, msToMidnight); }