explore-services/services/cache/mecache/cache.js

'use strict';

let express = require('express');
let app = express();
let mcache = require('memory-cache');
const request = require('superagent');
const prom = require('prom-client');
const URL = require('url');
const expireShort = 2 * 60 * 1000; //2mins
const expireLong = 24 * 60 * 60 * 1000; //24 hours
const cacheMaxSize =500;
const longCachingRequests = ["/communityFull", "/full", "/pagehelpcontent","/provision/mvc/vocabularies/","/pages?page_route=", "/allmetrics", "/countryusagestats/", "/openaire/info"];
let cors = require('cors');
app.use(cors());

const register = new prom.Registry();
prom.collectDefaultMetrics({register: register});

const responses = new prom.Counter({
    name: 'cache_http_responses_total',
    help: 'A counter for cache response codes for every API request.',
    labelNames: ['scheme', 'target', 'code'],
    registers: [register]
});

const entries = new prom.Gauge({
    name: 'cache_used_entries',
    help: 'A counter to count cache entries',
    registers: [register]
});

const histogram = new prom.Histogram({
    name: 'cache_http_request_duration_seconds',
    help: 'A Histogram for cache. Providing information about a cache request and load latency in seconds.',
    labelNames: ['scheme', 'target', 'cache'],
    registers: [register],
    buckets: [0.1, 0.2, 0.5, 1, 2]
});

let cache = () => {
    return (req, res, next) => {
        if(req.query.url) {
            if(mcache.memsize() > cacheMaxSize){
                console.log("Max cache size reached!"+cacheMaxSize);
                clearCache();
            }
            let key = '__express__' + req.query.url;
            let longCache = checkForLongCachedRequests(req.query.url);
            let cachedBody = mcache.get(key);
            const url = new URL.parse(req.query.url);
            const target = url.host + '/' + url.pathname.split('/')[1];
            const scheme = url.protocol.replace(':', '');
            if (cachedBody) {
                const end = histogram.startTimer({scheme: scheme, target: target, cache: 'hit'});
                res.send(JSON.parse(cachedBody));
                responses.inc({scheme: scheme, target: target, code: res.statusCode});
                end();
            } else {
                const end = histogram.startTimer({scheme: scheme, target: target, cache: 'miss'});
                res.sendResponse = res.send;
                res.send = (body) => {
                    let alreadyCached = !!mcache.get(key);
                    entries.set(mcache.memsize());
                    if(!alreadyCached) {
                        responses.inc({scheme: scheme, target: target, code: res.statusCode});
                        end();
                    }
                    if (res.statusCode === 200) {
                        // console.log("Expire in " +(longCache?expireLong:expireShort) + " " +req.query.url);
                        mcache.put(key, body,longCache?expireLong:expireShort, () => {
                            entries.set(mcache.memsize());
                        });
                    }
                    res.sendResponse(body);
                };
                next();
            }
        } else {
            next();
        }
    };
};

app.get('/clear', (req, res) => {
    let c = mcache.size();
    const url = req.query.url;
    let message ="";
    if(url){
        let key = '__express__' + req.query.url;
        mcache.del(key);
        message = "Delete entry with key "+ url;
        entries.set(mcache.size());
    }else{
        clearCache();
         message ="Delete "+c + " entries. Now there are: "+mcache.size()
     }
    res.header("Access-Control-Allow-Headers", "Origin, Content-Type,  Content-Length");
    res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");
    res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");
    res.header("Content-Type", "application/json");
    res.status(200).send(getResponse(200,message));

});
app.get('/metrics', (req, res) => {
    res.set('Content-Type', register.contentType);
    res.end(register.metrics());
});

app.get('/get', cache(), cors(), (req, res) => {
    setTimeout(() => {
        const url = (req.query) ? req.query.url : null;
        if (!url) {
            res.status(404).send(getResponse(404, "Not Found ")) //not found
        } else {
            request.get(url, function (err, response) {
                // res.header("Access-Control-Allow-Origin", "http://localhost:3000");
                res.header("Access-Control-Allow-Headers", "Origin, Content-Type,  Content-Length");
                res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");
                res.header("Access-Control-Allow-Methods", "GET,  OPTIONS");
                res.header("Content-Type", "application/json");
                if (!response && err) {
                    res.status(500).send(getResponse(500, "An error occurred for " + url))
                } else {
                    res.status(response.status).send(response.body);
                }
            })
        }
    })
});


app.use((req, res) => {
    res.status(404).send(getResponse(404, "Not Found")); //not found
});

app.listen((process.env.PORT) ? process.env.PORT : 3000, function () {
    console.log(`Example app listening on port ${(process.env.PORT) ? process.env.PORT : 3000}!`)
    //run the timer
    resetAtMidnight();
});

function getResponse(code, message) {
    var response = {};
    response["code"] = code;
    response["message"] = message;
    return response;

}
function clearCache(){
    console.log("cache is cleared!");
    mcache.clear();
    entries.set(mcache.size());
}
function checkForLongCachedRequests(url){
    let long =false;
    longCachingRequests.forEach(partUrl => {
        if(url.indexOf(partUrl) !==-1){
            long = true;
        }
    });
    return long;
}
function resetAtMidnight() {
    console.log("Run Reset timer");
    var now = new Date();
    var night = new Date(
        now.getFullYear(),
        now.getMonth(),
        now.getDate() + 1 , //  the next day, ...
         0, 0, 0 // ...at 00:00:00 hours
    );
    var msToMidnight = night.getTime() - now.getTime();

    setTimeout(function() {
        clearCache();              //      <-- This is the function being called at midnight.
        resetAtMidnight();    //      Then, reset again next midnight.
    }, msToMidnight);
}