From abb136d996157f0b36ea0b51edf1a9a84ca35c61 Mon Sep 17 00:00:00 2001 From: dcore94 Date: Fri, 7 Jan 2022 11:55:00 +0100 Subject: [PATCH] first share --- .gitignore | 9 + README.md | 29 +++ build-docker-images-sh | 1 + conf/pep/config.js.j2 | 57 ++++++ conf/pep/default.conf | 83 ++++++++ conf/pep/nginx.conf | 44 ++++ conf/pep/pep.js | 323 ++++++++++++++++++++++++++++++ conf/secrets.yaml | 20 ++ conf/service/auth.json.j2 | 16 ++ conf/service/d4s-vre-manager.json | 13 ++ images/Dockerfile | 12 ++ site.yaml | 25 +++ swarm.yaml | 61 ++++++ 13 files changed, 693 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100755 build-docker-images-sh create mode 100644 conf/pep/config.js.j2 create mode 100644 conf/pep/default.conf create mode 100644 conf/pep/nginx.conf create mode 100644 conf/pep/pep.js create mode 100644 conf/secrets.yaml create mode 100644 conf/service/auth.json.j2 create mode 100644 conf/service/d4s-vre-manager.json create mode 100644 images/Dockerfile create mode 100644 site.yaml create mode 100644 swarm.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a984a3c --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +/conf/pep/config.js +/conf/service/auth.json +/.muscle diff --git a/README.md b/README.md new file mode 100644 index 0000000..39af52c --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +# README # + +This README would normally document whatever steps are necessary to get your application up and running. + +### What is this repository for? ### + +* Quick summary +* Version +* [Learn Markdown](https://bitbucket.org/tutorials/markdowndemo) + +### How do I get set up? ### + +* Summary of set up +* Configuration +* Dependencies +* Database configuration +* How to run tests +* Deployment instructions + +### Contribution guidelines ### + +* Writing tests +* Code review +* Other guidelines + +### Who do I talk to? ### + +* Repo owner or admin +* Other community or team contact \ No newline at end of file diff --git a/build-docker-images-sh b/build-docker-images-sh new file mode 100755 index 0000000..b20458a --- /dev/null +++ b/build-docker-images-sh @@ -0,0 +1 @@ +docker build --rm --no-cache -t nubisware/d4s-vre-manager -f images/Dockerfile . diff --git a/conf/pep/config.js.j2 b/conf/pep/config.js.j2 new file mode 100644 index 0000000..b86f92c --- /dev/null +++ b/conf/pep/config.js.j2 @@ -0,0 +1,57 @@ +export default { config }; + +var config = { + "pep-credentials" : "{{pep_credentials}}", + "hosts" : [ + { + "host": "d4s-vre-manager-pep", + "audience" : "d4science-cdn", + "allow-basic-auth" : false, + "paths" : [ + { + "name" : "Default Resource", + "path" : "^/visuals/d4s-vre-manager/.*$", + "methods" : [ + { + "method" : "GET" + } + ] + }, + { + "name" : "Default Resource", + "path" : "^/d4s-vre-manager/.+$", + "methods" : [ + { + "method" : "GET" + } + ] + }, + { + "name" : "Default Resource", + "path" : "^/services/d4s-vre-manager/.*$", + "methods" : [ + { + "method" : "OPTIONS" + }, + { + "method" : "POST" + }, + { + "method" : "HEAD" + }, + { + "method" : "PUT" + }, + { + "method" : "DELETE" + }, + { + "method" : "GET" + } + ] + } + ] + } + ] +} + diff --git a/conf/pep/default.conf b/conf/pep/default.conf new file mode 100644 index 0000000..5544942 --- /dev/null +++ b/conf/pep/default.conf @@ -0,0 +1,83 @@ +upstream d4s-vre-manager { + ip_hash; + server d4s-vre-manager:8984; +} + +js_var $auth_token; +js_var $pep_credentials; +js_var $user; +js_var $user_display; + +map $http_authorization $source_auth { + default ""; +} + +server { + + listen *:80; + listen [::]:80; + server_name d4s-vre-manager-pep; + + subrequest_output_buffer_size 200k; + + location / { + js_content pep.enforce; + } + + location /resources/ { + proxy_pass http://d4s-vre-manager; + } + + location @backend { + proxy_set_header Authorization "Bearer $auth_token"; + proxy_set_header X-User "$user"; + proxy_set_header X-User-display "$user_display"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Original-URI $request_uri; + proxy_pass http://d4s-vre-manager; + } + + location /jwt_verify_request { + internal; + gunzip on; + proxy_method POST; + proxy_http_version 1.1; + proxy_set_header Authorization $pep_credentials; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token/introspect; + + proxy_cache token_responses; # Enable caching + proxy_cache_key $source_auth; # Cache for each source authentication + proxy_cache_lock on; # Duplicate tokens must wait + proxy_cache_valid 200 10s; # How long to use each response + proxy_ignore_headers Cache-Control Expires Set-Cookie; + } + + location /jwt_request { + internal; + gunzip on; + proxy_method POST; + proxy_http_version 1.1; + proxy_set_header Authorization $pep_credentials; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token; + } + + location /permission_request { + internal; + gunzip on; + proxy_method POST; + proxy_http_version 1.1; + proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_set_header Authorization "Bearer $auth_token"; + proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token; + } + +} + diff --git a/conf/pep/nginx.conf b/conf/pep/nginx.conf new file mode 100644 index 0000000..5796bf6 --- /dev/null +++ b/conf/pep/nginx.conf @@ -0,0 +1,44 @@ +# Added to load njs module +load_module modules/ngx_http_js_module.so; + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + + # added to import pep script + js_import pep.js; + + # added to bind enforce function + js_set $authorization pep.enforce; + + # added to create cache for tokens and auth calls + proxy_cache_path /var/cache/nginx/pep keys_zone=token_responses:1m max_size=2m; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/conf/pep/pep.js b/conf/pep/pep.js new file mode 100644 index 0000000..184177d --- /dev/null +++ b/conf/pep/pep.js @@ -0,0 +1,323 @@ +export default { enforce }; + +import defaultExport from './config.js'; + +function log(c, s){ + c.request.error(s) +} + +function enforce(r) { + + var context = { + request: r , + config : defaultExport["config"], + backend : (defaultExport.backend ? defaultExport.backend : "@backend") + } + + log(context, "Inside NJS enforce for " + r.method + " @ " + r.headersIn.host + "/" + r.uri) + + context = computeProtection(context) + + wkf.run(wkf.build(context), context) +} + +// ######## WORKFLOW FUNCTIONS ############### +var wkf = { + + build : (context)=>{ + var actions = [ + "export_pep_credentials", + "parse_authentication", + "check_authentication", + "export_authn_token", + "pip", + "pdp", + "export_backend_headers", + "pass" + ] + return actions + }, + + run : (actions, context) => { + context.request.error("Starting workflow with " + njs.dump(actions)) + var w = actions.reduce( + (acc, f) => { return acc.then(typeof(f) === "function" ? f : wkf[f]) }, + Promise.resolve().then(()=>context) + ) + w.catch(e => { context.request.error(njs.dump(e)); context.request.return(401)} ) + }, + + export_pep_credentials : exportPepCredentials, + export_authn_token : exportAuthToken, + export_backend_headers : exportBackendHeaders, + parse_authentication : parseAuthentication, + check_authentication : checkAuthentication, + verify_token : verifyToken, + request_token : requestToken, + pip : pipExecutor, + pdp : pdpExecutor, + pass : pass, + + //PIP utilities + "get-path-component" : (c, i) => c.request.uri.split("/")[i], + "get-token-field" : getTokenField, + "get-contexts" : (c) => { + var ra = c.authn.verified_token["resource_access"] + if(ra){ + var out = []; + for(var k in ra){ + if(ra[k].roles && ra[k].roles.length !== 0) out.push(k) + } + } + return out; + } +} + +function getTokenField(context, f){ + return context.authn.verified_token[f] +} + +function exportVariable(context, name, value){ + context.request.variables[name] = value + log(context, "Exported variables:" + njs.dump(context.request.variables)) + return context +} + +function exportBackendHeaders(context){ + exportVariable(context, "user", context.authn.verified_token.preferred_username) + exportVariable(context, "user_display", context.authn.verified_token.name) + return context +} + +function exportPepCredentials(context){ + if(!context.config["pep-credentials"]){ + throw new Error("Need PEP credentials") + } + return exportVariable(context, "pep_credentials", "Basic " + context.config["pep-credentials"]) +} + +function exportAuthToken(context){ + return exportVariable(context, "auth_token", context.authn.token) +} + +function checkAuthentication(context){ + return context.authn.type === "bearer" ? wkf.verify_token(context) : wkf.request_token(context) +} + +function parseAuthentication(context){ + context.request.log("Inside parseAuthentication") + var incomingauth = context.request.headersIn["Authorization"] + + if(!incomingauth) throw new Error("Authentication required"); + + var arr = incomingauth.trim().replace(/\s\s+/g, " ").split(" ") + if(arr.length != 2) throw new Error("Unknown authentication scheme"); + + var type = arr[0].toLowerCase() + if(type === "basic" && context.authz.host && context.authz.host["allow-basic-auth"]){ + var unamepass = Buffer.from(arr[1], 'base64').toString().split(":") + if(unamepass.length != 2) return null; + context.authn = { type : type, raw : arr[1], user : unamepass[0], password : unamepass[1]} + return context + }else if(type === "bearer"){ + context.authn = { type : type, raw : arr[1], token : arr[1]} + return context + } + throw new Error("Unknown authentication scheme"); +} + +function verifyToken(context){ + log(context, "Inside verifyToken") + log(context, "Token is " + context.authn.token) + var options = { + "body" : "token=" + context.authn.token + "&token_type_hint=access_token" + } + return context.request.subrequest("/jwt_verify_request", options) + .then(reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseBody); + if (response.active === true) { + return response + } else { + throw new Error("Unauthorized: " + reply.responseBody) + } + } else { + throw new Error("Unauthorized: " + reply.responseBody) + } + }).then(verified_token => { + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + }) +} + +function requestToken(context){ + log(context, "Inside requestToken") + var options = { + "body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password + } + return context.request.subrequest("/jwt_request", options) + .then(reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseBody); + context.authn.token = response.access_token + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + } else if (reply.status === 400 || reply.status === 401){ + var options = { + "body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password + } + return context.request.subrequest("/jwt_request", options) + .then( reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseBody); + context.authn.token = response.access_token + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + } else{ + throw new Error("Unauthorized " + reply.status) + } + }) + } else { + throw new Error("Unauthorized " + reply.status) + } + }) +} + +function pipExecutor(context){ + log(context, "Inside extra claims PIP") + context.authz.pip.forEach(extra =>{ + //call extra claim pip function + try{ + var operator = extra.operator + var result = wkf[operator](context, extra.args) + //ensure array and add to extra_claims + if(!(result instanceof Array)) result = [result] + if(!context.extra_claims) context.extra_claims = {}; + context.extra_claims[extra.claim] = result + } catch (error){ + log(context, "Skipping invalid extra claim " + njs.dump(error)) + } + }) + log(context, "Extra claims are " + njs.dump(context.extra_claims)) + return context +} + +function pdpExecutor(context){ + log(context, "Inside PDP") + return context.authz.pdp(context) +} + +function umaCall(context){ + log(context, "Inside UMA call") + var options = { "body" : computePermissionRequestBody(context) }; + return context.request.subrequest("/permission_request", options) + .then(reply =>{ + if(reply.status === 200){ + return context + }else{ + throw new Error("Response for authorization request is not ok " + reply.status + " " + njs.dump(reply.responseBody)) + } + }) +} + +function pass(context){ + log(context, "Inside pass"); + if(typeof(context.backend) === "string") context.request.internalRedirect(context.backend); + else if (typeof(context.backend) === "function") context.request.internalRedirect(context.backend(context)) + return context; +} + +// ######## AUTHORIZATION PART ############### +function computePermissionRequestBody(context){ + + if(!context.authz.host || !context.authz.path ){ + throw new Error("Enforcemnt mode is always enforcing. Host or path not found...") + } + + var audience = computeAudience(context) + var grant = "grant_type=urn:ietf:params:oauth:grant-type:uma-ticket" + var mode = "response_mode=decision" + var permissions = computePermissions(context) + var extra = "" + if(context.extra_claims){ + extra = + "claim_token_format=urn:ietf:params:oauth:token-type:jwt&claim_token=" + + JSON.stringify(context.extra_claims).toString("base64url") + } + var body = audience + "&" + grant + "&" + permissions + "&" + mode + "&" + extra + context.request.error("Computed permission request body is " + body) + return body +} + +function computeAudience(context){ + var aud = context.request.headersIn.host + if(context.authz.host){ + aud = context.authz.host.audience||context.authz.host.host + } + return "audience=" + aud +} + +function computePermissions(context){ + var resource = context.request.uri + if(context.authz.path){ + resource = context.authz.path.name||context.authz.path.path + } + var scopes = [] + if(context.authz.method && context.authz.method.scopes){ + scopes = context.authz.method.scopes + } + if(scopes.length > 0){ + return scopes.map(s=>"permission=" + resource + "#" + s).join("&") + } + return "permission=" + resource +} + +function getPath(hostconfig, incomingpath, incomingmethod){ + var paths = hostconfig.paths || [] + var matchingpaths = paths + .filter(p => {return incomingpath.match(p.path) != null}) + .reduce((acc, p) => { + if (!p.methods || p.methods.length === 0) acc.weak.push({ path: p}); + else{ + var matchingmethods = p.methods.filter(m=>m.method.toUpperCase() === incomingmethod) + if(matchingmethods.length > 0) acc.strong.push({ method : matchingmethods[0], path: p}); + } + return acc; + }, { strong: [], weak: []}) + return matchingpaths.strong.concat(matchingpaths.weak)[0] +} + +function getHost(config, host){ + var matching = config.hosts.filter(h=>{ + return h.host === host + }) + return matching.length > 0 ? matching[0] : null +} + +function computeProtection(context){ + log(context, "Getting by host " + context.request.headersIn.host) + context.authz = {} + context.authz.host = getHost(context.config, context.request.headersIn.host) + if(context.authz.host !== null){ + log(context, "Host found:" + context.authz.host) + context.authz.pip = context.authz.host.pip ? context.authz.host.pip : []; + context.authz.pdp = context.authz.host.pdp ? context.authz.host.pdp : umaCall; + var pathandmethod = getPath(context.authz.host, context.request.uri, context.request.method); + if(pathandmethod){ + context.authz.path = pathandmethod.path; + context.authz.pip = context.authz.path.pip ? context.authz.pip.concat(context.authz.path.pip) : context.authz.pip; + context.authz.pdp = context.authz.path.pdp ? context.authz.path.pdp : context.authz.pdp; + context.authz.method = pathandmethod.method; + if(context.authz.method){ + context.authz.pip = context.authz.method.pip ? context.authz.pip.concat(context.authz.method.pip) : context.authz.pip; + context.authz.pdp = context.authz.method.pdp ? context.authz.method.pdp : context.authz.pdp; + } + } + } + log(context, "Leaving protection computation: ") + return context +} + diff --git a/conf/secrets.yaml b/conf/secrets.yaml new file mode 100644 index 0000000..85acd4d --- /dev/null +++ b/conf/secrets.yaml @@ -0,0 +1,20 @@ +$ANSIBLE_VAULT;1.1;AES256 +30306331333731366336666633306337363464653165373831383766373763626433363030646132 +3165653730336139363863373731386636663338323835340a333563366634366536376461393530 +35643738366662666335363439373535623166313031393032303530373033373864613733303863 +6363333535343464640a646436613661663063666165386139356632373330616464313737343533 +33336538343265646634333539613139613261663231376130613732313361393238346631633164 +36643563316332346335316430643230373161363237316137383237363533643032313161366631 +37663761376135636137653566333832313266383634643837393936303935373238656539373734 +30373236343933643331383836633032343132306439653761643233323931323830333563326330 +35343039393662313561396564353863323833306265376364303933396163613365353066626534 +63306131616337393733636665623961626335303564306230653233616437613635643432326465 +62316632336466393362373831376565623936376535306466326666353834306630666139313266 +38636630653034346561323532623239333163323631393737386536633435653866383763616638 +37326330326566366666353430353736656233633330636338373765613461326335333734386230 +32643331663236323466663235636433346133383938326434646566636261313161313433363339 +37643266306231363636386266356631653930373137623162333966643863336566616462333135 +30303862306235306139633663383965366330343461333932356164316636656363616635323930 +61656532663938346135376233633732366439333162633361343734663432306436613333303737 +32613034663834623038656339653361363939643830626630333430383365373436323262663333 +383737323832643830393361336532363137 diff --git a/conf/service/auth.json.j2 b/conf/service/auth.json.j2 new file mode 100644 index 0000000..c33c1cd --- /dev/null +++ b/conf/service/auth.json.j2 @@ -0,0 +1,16 @@ +{ + + "keycloakurl" : "https://accounts.dev.d4science.org", + "keycloakrealm" : "d4science", + + "user-manager-client-id" : "orchestrator", + "user-manager-client-secret" : "{{ user_manager_client_secret }}", + + "client-manager-client-id" : "orchestrator", + "client-manager-client-secret" : "{{ client_manager_client_secret }}", + + "group-manager-client-id" : "orchestrator", + "group-manager-client-secret" : "{{ group_manager_client_secret }}" + +} + diff --git a/conf/service/d4s-vre-manager.json b/conf/service/d4s-vre-manager.json new file mode 100644 index 0000000..eb26c59 --- /dev/null +++ b/conf/service/d4s-vre-manager.json @@ -0,0 +1,13 @@ +{ + "d4science_realm" : "d4science", + "d4science_iam_base" : "https://accounts.dev.d4science.org/auth", + "d4science_cdn_base" : "http://cdn.dev.d4science.org", + "external_apps" : { + "/gcube/devsec/devVRE" : [ + { + "name" : "_myextapp", + "description" : "An application for testing purposes" + } + ] + } +} diff --git a/images/Dockerfile b/images/Dockerfile new file mode 100644 index 0000000..9912f26 --- /dev/null +++ b/images/Dockerfile @@ -0,0 +1,12 @@ +### 1. Get muscle classic +FROM nubisware/muscle-classic:latest + +# 2. Install applicative fibers +ADD .muscle/ /root/.muscle/ +RUN git fiber install -p G3 -n d4s-vre-manager \ + && git service create -n service -w auth -w utils -w config -w error -w utils -w inspect -w resources -w d4s-vre-manager \ + && rm -rf /root/.muscle/ + +WORKDIR service +EXPOSE 8984 +ENTRYPOINT ./start.sh diff --git a/site.yaml b/site.yaml new file mode 100644 index 0000000..71a1525 --- /dev/null +++ b/site.yaml @@ -0,0 +1,25 @@ +--- +- hosts: localhost + vars_files: + - conf/secrets.yaml + vars: + infrastructure: "local" + dry: false + tasks: + - name: Patch PEP config + template: + src: "conf/pep/config.js.j2" + dest: "conf/pep/config.js" + + - name: Patch service auth config + template: + src: "conf/service/auth.json.j2" + dest: "conf/service/auth.json" + + - name: Start swarm + docker_stack: + name: 'cdn-{{ infrastructure }}' + state: present + compose: + - "swarm.yaml" + when: dry is not defined or not dry|bool diff --git a/swarm.yaml b/swarm.yaml new file mode 100644 index 0000000..13728e2 --- /dev/null +++ b/swarm.yaml @@ -0,0 +1,61 @@ +version: '3.6' + +services: + + d4s-vre-manager-pep: + image: nginx:stable-alpine + networks: + - cdn-network + deploy: + mode: replicated + replicas: 2 + restart_policy: + condition: on-failure + delay: 10s + window: 120s + configs: + - source: vre_manager_pep_conf + target: /etc/nginx/templates/default.conf.template + - source: vre_manager_pep_baseconf + target: /etc/nginx/nginx.conf + - source: vre_manager_pep + target: /etc/nginx/pep.js + - source: vre_manager_pepconfig + target: /etc/nginx/config.js + + + d4s-vre-manager: + image: nubisware/d4s-vre-manager + networks: + - cdn-network + deploy: + mode: replicated + replicas: 2 + restart_policy: + condition: on-failure + delay: 10s + window: 200s + configs: + - source: vre_manager_conf + target: /opt/app/service/conf/d4s-vre-manager.json + - source: vre_manager_auth_conf + target: /opt/app/service/conf/auth.json + +networks: + cdn-network: + +configs: + vre_manager_pep_conf: + file: ./conf/pep/default.conf + vre_manager_pep_baseconf: + file: ./conf/pep/nginx.conf + vre_manager_pep: + file: ./conf/pep/pep.js + vre_manager_pepconfig: + file: ./conf/pep/config.js + + vre_manager_conf: + file: ./conf/service/d4s-vre-manager.json + vre_manager_auth_conf: + file: ./conf/service/auth.json +