From bb7f63cb04665abe58519bb457b025149ebf95f4 Mon Sep 17 00:00:00 2001 From: dcore94 Date: Mon, 4 Mar 2024 15:25:27 +0100 Subject: [PATCH] first share --- pepexample/Dockerfile | 20 ++ pepexample/build-image.sh | 3 + pepexample/config.js | 37 +++ pepexample/default.conf | 91 +++++++ pepexample/docker-compose.yaml | 19 ++ pepexample/docker-entrypoint.sh | 4 + pepexample/nginx.conf | 46 ++++ pepexample/pep.js | 411 ++++++++++++++++++++++++++++++++ pepexample/server.py | 62 +++++ 9 files changed, 693 insertions(+) create mode 100644 pepexample/Dockerfile create mode 100755 pepexample/build-image.sh create mode 100644 pepexample/config.js create mode 100644 pepexample/default.conf create mode 100644 pepexample/docker-compose.yaml create mode 100644 pepexample/docker-entrypoint.sh create mode 100644 pepexample/nginx.conf create mode 100644 pepexample/pep.js create mode 100644 pepexample/server.py diff --git a/pepexample/Dockerfile b/pepexample/Dockerfile new file mode 100644 index 0000000..075841c --- /dev/null +++ b/pepexample/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y curl gnupg2 ca-certificates lsb-release ubuntu-keyring +RUN curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null +RUN echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/ubuntu `lsb_release -cs` nginx" | tee /etc/apt/sources.list.d/nginx.list +RUN echo "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" | tee /etc/apt/preferences.d/99nginx +RUN apt-get update && apt-get install -y nginx nginx-module-njs + +COPY nginx.conf /etc/nginx/nginx.conf +COPY default.conf /etc/nginx/conf.d/default.conf +COPY pep.js config.js /etc/nginx/ +COPY server.py /opt/ +COPY docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod 755 /docker-entrypoint.sh && chown -R www-data: /docker-entrypoint.sh + +#RUN apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list + +# forward request and error logs to docker log collector +RUN ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log + +CMD ["/docker-entrypoint.sh"] \ No newline at end of file diff --git a/pepexample/build-image.sh b/pepexample/build-image.sh new file mode 100755 index 0000000..3dcb3a7 --- /dev/null +++ b/pepexample/build-image.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker build -t peptest:latest . \ No newline at end of file diff --git a/pepexample/config.js b/pepexample/config.js new file mode 100644 index 0000000..1eceb5a --- /dev/null +++ b/pepexample/config.js @@ -0,0 +1,37 @@ +export default { config }; + +var config = { + "hosts": [ + { + "host": ["peptest"], + "audience": "peptest", + "allow-basic-auth": true, + "paths": [ + { + "name": "operator", + "path": "^/operator/?.*$", + "methods": [ + { + "method": "GET" + }, + { + "method": "POST" + } + ] + }, + { + "name": "researcher", + "path": "^/researcher/?.*$", + "methods": [ + { + "method": "GET" + }, + { + "method": "POST" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pepexample/default.conf b/pepexample/default.conf new file mode 100644 index 0000000..08f1754 --- /dev/null +++ b/pepexample/default.conf @@ -0,0 +1,91 @@ +# backend service +upstream service { + ip_hash; + server 127.0.0.1:8000; +} + +# variables computed by njs and which may possibly be passed among locations +js_var $auth_token; +js_var $account_record; +js_var $pep_credentials; + +server { + + listen *:80; + listen [::]:80; + + # the vhost name + server_name peptest; + + # for services that return large maounts of data + subrequest_output_buffer_size 200k; + + location /health { + add_header Content-Length 0; + add_header Content-Type "text/plain"; + return 200; + # or just call the backend service's healt endpoint + } + + location / { + js_content pep.enforce; + } + + # internal location that redirects to backend will only be called from PEP JS code when all checks are passed + location /_backend { + internal; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + proxy_pass http://service/; + } + + + # internal location that redirects to Keycloak in order to verify a token's validity. This will be called only from PEP JS code + location /jwt_verify_request { + internal; + proxy_method POST; + proxy_http_version 1.1; + proxy_set_header Authorization $pep_credentials; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_pass "https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token/introspect"; + + proxy_ignore_headers Cache-Control Expires Set-Cookie; + gunzip on; + } + + # internal location that redirects to Keycloak in order to exchange Basic auth credentials with token. This will be called only from PEP JS code + location /jwt_request { + internal; + proxy_method POST; + proxy_http_version 1.1; + proxy_set_header Authorization $pep_credentials; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_pass "https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token"; + gunzip on; + } + + # internal location that redirects to Keycloak in order to perform a specific authorization request. This will be called only from PEP JS code + location /permission_request { + internal; + proxy_method POST; + proxy_http_version 1.1; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_set_header Authorization "Bearer $auth_token"; + proxy_pass "https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token"; + gunzip on; + } + + # internal location that sends a record to accounting service. This will be called only from PEP JS code if requested. + location /accounting { + internal; + proxy_method POST; + proxy_http_version 1.1; + proxy_set_header Authorization "$auth_token"; + proxy_set_header Content-Type "application/json"; + proxy_pass https://accounting-service.d4science.org/accounting-service/record; + } +} \ No newline at end of file diff --git a/pepexample/docker-compose.yaml b/pepexample/docker-compose.yaml new file mode 100644 index 0000000..0d9a92e --- /dev/null +++ b/pepexample/docker-compose.yaml @@ -0,0 +1,19 @@ +version: '3.6' + +services: + pep: + image: peptest:latest + ports: + - "80:80" + healthcheck: + test: curl http://localhost/health + interval: 5s + timeout: 5s + retries: 12 + volumes: + - ./server.py:/opt/server.py + - ./default.conf:/etc/nginx/conf.d/default.conf + - ./pep.js:/etc/nginx/pep.js + - ./config.js:/etc/nginx/config.js + environment: + PEP_CREDENTIALS: ${pep_credentials} \ No newline at end of file diff --git a/pepexample/docker-entrypoint.sh b/pepexample/docker-entrypoint.sh new file mode 100644 index 0000000..373cb96 --- /dev/null +++ b/pepexample/docker-entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +python3 /opt/server.py & +nginx -g 'daemon off;' \ No newline at end of file diff --git a/pepexample/nginx.conf b/pepexample/nginx.conf new file mode 100644 index 0000000..bb68ece --- /dev/null +++ b/pepexample/nginx.conf @@ -0,0 +1,46 @@ +# Added to load njs module +load_module modules/ngx_http_js_module.so; + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + +# declare environmental variable to get credentials requiredd to access keycloak +env PEP_CREDENTIALS; + +http { + + # added to import pep script + js_import pep.js; + + # added to bind enforce function + js_set $authorization pep.enforce; + + # added to create cache for tokens and auth calls + proxy_cache_path /var/cache/nginx/pep keys_zone=token_responses:1m max_size=2m; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/pepexample/pep.js b/pepexample/pep.js new file mode 100644 index 0000000..a1cacd6 --- /dev/null +++ b/pepexample/pep.js @@ -0,0 +1,411 @@ +export default { enforce }; + +import defaultExport from './config.js'; + +function log(c, s){ + c.request.error(s) +} + +var _debug = true +function debug(c, s){ + if(_debug === true){ + log(c, s) + } +} + +function enforce(r) { + + var context = { + request: r , + config : defaultExport["config"], + backend : (defaultExport.backend ? defaultExport.backend : "/_backend") + } + + log(context, "Inside NJS enforce for " + r.method + " @ " + r.headersIn.host + "/" + r.uri) + + context = computeProtection(context) + + wkf.run(wkf.build(context), context) +} + +// ######## WORKFLOW FUNCTIONS ############### +var wkf = { + + build : (context)=>{ + var actions = [ + "export_pep_credentials", + "parse_authentication", + "check_authentication", + "export_authn_token", +// "pip", + "pdp", +// "export_backend_headers", + "pass" + ] + /*var actions = [ + "export_pep_credentials", + "parse_authentication", + "check_authentication", + "export_authn_token", + "pip", + "pdp", + "export_backend_headers", + "start_accounting", + "pass_and_wait", + "close_accounting", + "respond_to_client" + ]*/ + return actions + }, + + run : (actions, context) => { + context.request.error("Starting workflow with " + njs.dump(actions)) + var w = actions.reduce( + (acc, f) => { return acc.then(typeof(f) === "function" ? f : wkf[f]) }, + Promise.resolve().then(()=>context) + ) + w.catch(e => { context.request.error(njs.dump(e)); context.request.return(401)} ) + }, + + export_pep_credentials : exportPepCredentials, + export_authn_token : exportAuthToken, + export_backend_headers : exportBackendHeaders, + parse_authentication : parseAuthentication, + check_authentication : checkAuthentication, + verify_token : verifyToken, + request_token : requestToken, + pip : pipExecutor, + pdp : pdpExecutor, + pass : pass, + pass_and_wait : pass_and_wait, + respond_to_client : respondToClient, + start_accounting : buildAccountingRecord, + close_accounting : closeAccountingRecord, + send_accounting : sendAccountingRecord +} + +function getTokenField(context, f){ + return context.authn.verified_token[f] +} + +function exportVariable(context, name, value){ + context.request.variables[name] = value + return context +} + +function exportBackendHeaders(context){ + return context +} + +function exportPepCredentials(context){ + if(process.env["pep_credentials"] || process.env["PEP_CREDENTIALS"]){ + return exportVariable(context, "pep_credentials", "Basic " + process.env["PEP_CREDENTIALS"]) + }else if(context.config["pep_credentials"]){ + return exportVariable(context, "pep_credentials", "Basic " + context.config["pep_credentials"]) + }else{ + throw new Error("Need PEP credentials") + } +} + +function exportAuthToken(context){ + return exportVariable(context, "auth_token", context.authn.token) +} + +function checkAuthentication(context){ + return context.authn.type === "bearer" ? wkf.verify_token(context) : wkf.request_token(context) +} + +function parseAuthentication(context){ + context.request.log("Inside parseAuthentication") + var incomingauth = context.request.headersIn["Authorization"] + + if(!incomingauth) throw new Error("Authentication required"); + + var arr = incomingauth.trim().replace(/\s\s+/g, " ").split(" ") + if(arr.length != 2) throw new Error("Unknown authentication scheme"); + + var type = arr[0].toLowerCase() + if(type === "basic" && context.authz.host && context.authz.host["allow-basic-auth"]){ + var unamepass = Buffer.from(arr[1], 'base64').toString().split(":") + if(unamepass.length != 2) return null; + context.authn = { type : type, raw : arr[1], user : unamepass[0], password : unamepass[1]} + return context + }else if(type === "bearer"){ + context.authn = { type : type, raw : arr[1], token : arr[1]} + return context + } + throw new Error("Unknown authentication scheme"); +} + +function verifyToken(context){ + log(context, "Inside verifyToken") + debug(context, "Token is " + context.authn.token) + var options = { + "body" : "token=" + context.authn.token + "&token_type_hint=access_token" + } + return context.request.subrequest("/jwt_verify_request", options) + .then(reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseText); + if (response.active === true) { + return response + } else { + throw new Error("Unauthorized: " + reply.responseText) + } + } else { + throw new Error("Unauthorized: " + reply.responseText) + } + }).then(verified_token => { + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + }) +} + +function requestToken(context){ + log(context, "Inside requestToken") + var options = { + "body" : "grant_type=client_credentials&client_id="+context.authn.user+"&client_secret="+context.authn.password + } + return context.request.subrequest("/jwt_request", options) + .then(reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseText); + context.authn.token = response.access_token + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + } else if (reply.status === 400 || reply.status === 401){ + var options = { + "body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password + } + return context.request.subrequest("/jwt_request", options) + .then( reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseText); + context.authn.token = response.access_token + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + } else{ + throw new Error("Unauthorized " + reply.status) + } + }) + } else { + throw new Error("Unauthorized " + reply.status) + } + }) +} + +function pipExecutor(context){ + log(context, "Inside extra claims PIP") + context.authz.pip.forEach(extra =>{ + //call extra claim pip function + try{ + var operator = extra.operator + var result = wkf[operator](context, extra.args) + //ensure array and add to extra_claims + if(!(result instanceof Array)) result = [result] + if(!context.extra_claims) context.extra_claims = {}; + context.extra_claims[extra.claim] = result + } catch (error){ + log(context, "Skipping invalid extra claim " + njs.dump(error)) + } + }) + log(context, "Extra claims are " + njs.dump(context.extra_claims)) + return context +} + +function pdpExecutor(context){ + log(context, "Inside PDP") + return context.authz.pdp(context) +} + +function umaCall(context){ + log(context, "Inside UMA call") + var options = { "body" : computePermissionRequestBody(context) }; + return context.request.subrequest("/permission_request", options) + .then(reply =>{ + if(reply.status === 200){ + debug(context, "UMA call reply is " + reply.status) + return context + }else{ + throw new Error("Response for authorization request is not ok " + reply.status + " " + njs.dump(reply.responseText)) + } + }) +} + +// Call backend and return reply to client directly +async function pass(context){ + log(context, "Inside pass and wait"); + const r = context.request + const reply = await r.subrequest((context.backend + r.variables.uri), { method : r.method, args : r.variables.args, headers : r.headersIn}) + debug(context, "[BACKEND] response status: " + reply.status) + context.backendresponse = reply + return respondToClient(context) +} + +// Pass to backend but instead of returning to client wait for the reply. Thi intercepts the possibility of performing extra action like closing and sending accounting record. +async function pass_and_wait(context){ + log(context, "Inside pass and wait"); + const r = context.request + const reply = await r.subrequest((context.backend + r.variables.uri), { method : r.method, args : r.variables.args, headers : r.headersIn}) + debug(context, "[BACKEND] response status: " + reply.status) + context.backendresponse = reply + return context +} + +// Definetely return backend response +function respondToClient(context){ + log(context, "Inside respond to client" + njs.dump(context.backendresponse)); + for(let k in context.backendresponse.headersOut){ + context.request.headersOut[k] = context.backendresponse.headersOut[k] + } + context.request.return(context.backendresponse.status, context.backendresponse.responseText) + return context +} + +// ######## AUTHORIZATION PART ############### +function computePermissionRequestBody(context){ + + if(!context.authz.host || !context.authz.path ){ + throw new Error("Enforcemnt mode is always enforcing. Host or path not found...") + } + + var audience = computeAudience(context) + var grant = "grant_type=urn:ietf:params:oauth:grant-type:uma-ticket" + var mode = "response_mode=decision" + var permissions = computePermissions(context) + var extra = "" + if(context.extra_claims){ + extra = + "claim_token_format=urn:ietf:params:oauth:token-type:jwt&claim_token=" + + JSON.stringify(context.extra_claims).toString("base64url") + } + var body = audience + "&" + grant + "&" + permissions + "&" + mode + "&" + extra + context.request.error("Computed permission request body is " + body) + return body +} + +function computeAudience(context){ + var aud = context.request.headersIn.host + if(context.authz.host){ + aud = context.authz.host.audience||context.authz.host.host + } + return "audience=" + aud +} + +function computePermissions(context){ + var resource = context.request.uri + if(context.authz.path){ + resource = context.authz.path.name||context.authz.path.path + } + var scopes = [] + if(context.authz.method && context.authz.method.scopes){ + scopes = context.authz.method.scopes + } + if(scopes.length > 0){ + return scopes.map(s=>"permission=" + resource + "#" + s).join("&") + } + return "permission=" + resource +} + +function getPath(hostconfig, incomingpath, incomingmethod){ + var paths = hostconfig.paths || [] + var matchingpaths = paths + .filter(p => {return incomingpath.match(p.path) != null}) + .reduce((acc, p) => { + if (!p.methods || p.methods.length === 0) acc.weak.push({ path: p}); + else{ + var matchingmethods = p.methods.filter(m=>m.method.toUpperCase() === incomingmethod) + if(matchingmethods.length > 0) acc.strong.push({ method : matchingmethods[0], path: p}); + } + return acc; + }, { strong: [], weak: []}) + return matchingpaths.strong.concat(matchingpaths.weak)[0] +} + +function getHost(config, host){ + var matching = config.hosts.filter(h=>{ + //compare for both string and array of strings + return ((h.host.filter && h.host.indexOf(host) !== -1) || h.host === host) + }) + return matching.length > 0 ? matching[0] : null +} + +function computeProtection(context){ + debug(context, "Getting by host " + context.request.headersIn.host) + context.authz = {} + context.authz.host = getHost(context.config, context.request.headersIn.host) + if(context.authz.host !== null){ + log(context, "Host found:" + context.authz.host) + context.authz.pip = context.authz.host.pip ? context.authz.host.pip : []; + context.authz.pdp = context.authz.host.pdp ? context.authz.host.pdp : umaCall; + var pathandmethod = getPath(context.authz.host, context.request.uri, context.request.method); + if(pathandmethod){ + context.authz.path = pathandmethod.path; + context.authz.pip = context.authz.path.pip ? context.authz.pip.concat(context.authz.path.pip) : context.authz.pip; + context.authz.pdp = context.authz.path.pdp ? context.authz.path.pdp : context.authz.pdp; + context.authz.method = pathandmethod.method; + if(context.authz.method){ + context.authz.pip = context.authz.method.pip ? context.authz.pip.concat(context.authz.method.pip) : context.authz.pip; + context.authz.pdp = context.authz.method.pdp ? context.authz.method.pdp : context.authz.pdp; + } + } + } + debug(context, "Leaving protection computation: ") + return context +} + +// ####### ACCOUNTING PART ##################### +function buildAccountingRecord(context){ + log(context, "Inside build accounting record"); + const t = (new Date()).getTime() + context.record = { + "recordType": "ServiceUsageRecord", + "operationCount": 1, + "creationTime": t, + "callerHost": context.request.remoteAddress, + "serviceClass": "Application", + "callerQualifier": "TOKEN", + "consumerId": getTokenField(context, "preferred_username"), + "aggregated": true, + "serviceName": "{{ SERVICENAME }}", + "duration": 0, + "maxInvocationTime": 0, + "scope": "{{ SCOPES }}", + "host": "{{ SERVICEHOST }}", + "startTime": t, + "id": uuid(), + "calledMethod": context.request.method + " " + context.request.uri, + "endTime": 0, + "minInvocationTime": 0, + "operationResult": null + } + debug(context, "Record is " + JSON.stringify(context.record)) + return context + } + + function closeAccountingRecord(context, success){ + log(context, "Inside close accounting"); + const t = (new Date()).getTime() + context.record.duration = t - context.record.startTime + context.record.endTime = t + context.record.minInvocationTime = context.record.duration + context.record.operationResult = success ? "SUCCESS" : "FAILED"; + debug(context, "Record is " + JSON.stringify(context.record)) + return context + } + + function sendAccountingRecord(context){ + log(context, "Inside send accounting"); + context.request.subrequest("/accounting", { detached : true, body : JSON.stringify(context.record) }) + return context + } + + function uuid() { + return 'xxxxxxxx-xxxx-4xxx-8xxx-xxxxxxxxxxxx'.replace(/[x]/g, function (c) { + const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + } \ No newline at end of file diff --git a/pepexample/server.py b/pepexample/server.py new file mode 100644 index 0000000..faf9e0b --- /dev/null +++ b/pepexample/server.py @@ -0,0 +1,62 @@ +# webapp.py + +from functools import cached_property +from http.cookies import SimpleCookie +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qsl, urlparse +import subprocess +import os +import json + +class WebRequestHandler(BaseHTTPRequestHandler): + @cached_property + def url(self): + return urlparse(self.path) + + @cached_property + def query_data(self): + return dict(parse_qsl(self.url.query)) + + @cached_property + def post_data(self): + content_length = int(self.headers.get("Content-Length", 0)) + return self.rfile.read(content_length) + + @cached_property + def json_data(self): + return json.loads(self.post_data.decode("utf-8")) + + @cached_property + def authorized(self): + return self.headers.get("Authorization") == os.environ["Authorization"] + + def do_GET(self): + message = "GET from {} with params: {} and headers {}".format(self.url.path, str(self.query_data),str(self.headers)) + print(message) + self.protocol_version = 'HTTP/1.0' + if self.url.path in ["/researcher/notexists", "/operator/notexists"] : + print("returning 404 Not found") + self.send_error(404, "Not found") + else: + print("returning 200") + self.send_response(200, "OK") + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(message.encode("UTF-8")) + self.wfile.flush() + + + def do_POST(self): + message = "POST to {} with data: {} and headers {}".format(self.url.path, str(self.post_data),str(self.headers)) + print(message) + self.protocol_version = 'HTTP/1.0' + self.send_response(200, "OK") + self.send_header("Content-Type", "text/plain") + self.send_header("X-Custom", "Hello world") + self.end_headers() + self.wfile.write(message.encode("UTF-8")) + self.wfile.flush() + +if __name__ == "__main__": + server = HTTPServer(("0.0.0.0", 8000), WebRequestHandler) + server.serve_forever() \ No newline at end of file