first share

main
dcore94 2 years ago
commit 8d8bd9ca4f

6
.gitignore vendored

@ -0,0 +1,6 @@
# files generated by ansible
/conf/pep/config.js
/conf/service/auth.json
# keys for accessing muscle repos
/.muscle

@ -0,0 +1,64 @@
# README #
### What is this repository for? ###
* Overview
This repository contains instructions to setup the D4S Navigation island
* Version
0.0.1
* Dependencies
This repository depends on D4S CDN (https://code-repo.d4science.org/gCubeSystem/d4s-cdn-setup.git).
### How do I get set up? ###
* Summary of set up
This repository contains all information to build a D4S navigation image. Most importantly it contains the ansible instructions to deploy a swarm, CDN navigation island and PEP.
* Deployment instructions
There are 4 branches.
__main__ branch contains configurations to start a local instance useful for development. It is configured to use the D4SCience dev IAM.
__dev__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the DEV infrastructure.
__pre__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the PRE infrastructure.
__prod__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the PROD infrastructure.
In order to run a local site the following commands need to be executed:
```
git clone https://code-repo.d4science.org/gCubeSystem/d4s-navigation-setup.git
ansible-playbook site.yaml --ask-vault-pass
```
This will create a basic stack with an NGINX based PEP and base navigation service.
If you want to deploy on a specific D4S infrastructure please clone the corresponding branch.
```
git clone -b {infra} https://code-repo.d4science.org/gCubeSystem/d4s-navigation-setup.git
ansible-playbook site.yaml --ask-vault-pass
```
* Commit guidelines
DO NOT commit the following files:
conf/pep/config.js
conf/service/auth.js
They will be generated by the ansible based deployment procedure in order to inject the secrets.
### Who do I talk to? ###
* Repo owner or admin
Marco Lettere
marco.lettere@nubisware.com

@ -0,0 +1 @@
docker build --rm --no-cache -t nubisware/d4s-navigation -f images/Dockerfile .

@ -0,0 +1,57 @@
export default { config };
var config = {
"pep-credentials" : "{{ pep_credentials }}",
"hosts" : [
{
"host": "d4s-navigation-pep",
"audience" : "d4science-cdn",
"allow-basic-auth" : false,
"paths" : [
{
"name" : "Default Resource",
"path" : "^/visuals/d4s-navigation/.*$",
"methods" : [
{
"method" : "GET"
}
]
},
{
"name" : "Default Resource",
"path" : "^/d4s-navigation/.+$",
"methods" : [
{
"method" : "GET"
}
]
},
{
"name" : "Default Resource",
"path" : "^/services/d4s-navigation/.*$",
"methods" : [
{
"method" : "OPTIONS"
},
{
"method" : "POST"
},
{
"method" : "HEAD"
},
{
"method" : "PUT"
},
{
"method" : "DELETE"
},
{
"method" : "GET"
}
]
}
]
}
]
}

@ -0,0 +1,83 @@
upstream d4s-navigation {
ip_hash;
server d4s-navigation:8984;
}
js_var $auth_token;
js_var $pep_credentials;
js_var $user;
js_var $user_display;
map $http_authorization $source_auth {
default "";
}
server {
listen *:80;
listen [::]:80;
server_name d4s-navigation-pep;
subrequest_output_buffer_size 200k;
location / {
js_content pep.enforce;
}
location /resources/ {
proxy_pass http://d4s-navigation;
}
location @backend {
proxy_set_header Authorization "Bearer $auth_token";
proxy_set_header X-User "$user";
proxy_set_header X-User-display "$user_display";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Original-URI $request_uri;
proxy_pass http://d4s-navigation;
}
location /jwt_verify_request {
internal;
gunzip on;
proxy_method POST;
proxy_http_version 1.1;
proxy_set_header Authorization $pep_credentials;
proxy_set_header Content-Type "application/x-www-form-urlencoded";
proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token/introspect;
proxy_cache token_responses; # Enable caching
proxy_cache_key $source_auth; # Cache for each source authentication
proxy_cache_lock on; # Duplicate tokens must wait
proxy_cache_valid 200 10s; # How long to use each response
proxy_ignore_headers Cache-Control Expires Set-Cookie;
}
location /jwt_request {
internal;
gunzip on;
proxy_method POST;
proxy_http_version 1.1;
proxy_set_header Authorization $pep_credentials;
proxy_set_header Content-Type "application/x-www-form-urlencoded";
proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token;
}
location /permission_request {
internal;
gunzip on;
proxy_method POST;
proxy_http_version 1.1;
proxy_set_header Content-Type "application/x-www-form-urlencoded";
proxy_set_header Authorization "Bearer $auth_token";
proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token;
}
}

@ -0,0 +1,44 @@
# Added to load njs module
load_module modules/ngx_http_js_module.so;
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
# added to import pep script
js_import pep.js;
# added to bind enforce function
js_set $authorization pep.enforce;
# added to create cache for tokens and auth calls
proxy_cache_path /var/cache/nginx/pep keys_zone=token_responses:1m max_size=2m;
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}

@ -0,0 +1,323 @@
export default { enforce };
import defaultExport from './config.js';
function log(c, s){
c.request.error(s)
}
function enforce(r) {
var context = {
request: r ,
config : defaultExport["config"],
backend : (defaultExport.backend ? defaultExport.backend : "@backend")
}
log(context, "Inside NJS enforce for " + r.method + " @ " + r.headersIn.host + "/" + r.uri)
context = computeProtection(context)
wkf.run(wkf.build(context), context)
}
// ######## WORKFLOW FUNCTIONS ###############
var wkf = {
build : (context)=>{
var actions = [
"export_pep_credentials",
"parse_authentication",
"check_authentication",
"export_authn_token",
"pip",
"pdp",
"export_backend_headers",
"pass"
]
return actions
},
run : (actions, context) => {
context.request.error("Starting workflow with " + njs.dump(actions))
var w = actions.reduce(
(acc, f) => { return acc.then(typeof(f) === "function" ? f : wkf[f]) },
Promise.resolve().then(()=>context)
)
w.catch(e => { context.request.error(njs.dump(e)); context.request.return(401)} )
},
export_pep_credentials : exportPepCredentials,
export_authn_token : exportAuthToken,
export_backend_headers : exportBackendHeaders,
parse_authentication : parseAuthentication,
check_authentication : checkAuthentication,
verify_token : verifyToken,
request_token : requestToken,
pip : pipExecutor,
pdp : pdpExecutor,
pass : pass,
//PIP utilities
"get-path-component" : (c, i) => c.request.uri.split("/")[i],
"get-token-field" : getTokenField,
"get-contexts" : (c) => {
var ra = c.authn.verified_token["resource_access"]
if(ra){
var out = [];
for(var k in ra){
if(ra[k].roles && ra[k].roles.length !== 0) out.push(k)
}
}
return out;
}
}
function getTokenField(context, f){
return context.authn.verified_token[f]
}
function exportVariable(context, name, value){
context.request.variables[name] = value
log(context, "Exported variables:" + njs.dump(context.request.variables))
return context
}
function exportBackendHeaders(context){
exportVariable(context, "user", context.authn.verified_token.preferred_username)
exportVariable(context, "user_display", context.authn.verified_token.name)
return context
}
function exportPepCredentials(context){
if(!context.config["pep-credentials"]){
throw new Error("Need PEP credentials")
}
return exportVariable(context, "pep_credentials", "Basic " + context.config["pep-credentials"])
}
function exportAuthToken(context){
return exportVariable(context, "auth_token", context.authn.token)
}
function checkAuthentication(context){
return context.authn.type === "bearer" ? wkf.verify_token(context) : wkf.request_token(context)
}
function parseAuthentication(context){
context.request.log("Inside parseAuthentication")
var incomingauth = context.request.headersIn["Authorization"]
if(!incomingauth) throw new Error("Authentication required");
var arr = incomingauth.trim().replace(/\s\s+/g, " ").split(" ")
if(arr.length != 2) throw new Error("Unknown authentication scheme");
var type = arr[0].toLowerCase()
if(type === "basic" && context.authz.host && context.authz.host["allow-basic-auth"]){
var unamepass = Buffer.from(arr[1], 'base64').toString().split(":")
if(unamepass.length != 2) return null;
context.authn = { type : type, raw : arr[1], user : unamepass[0], password : unamepass[1]}
return context
}else if(type === "bearer"){
context.authn = { type : type, raw : arr[1], token : arr[1]}
return context
}
throw new Error("Unknown authentication scheme");
}
function verifyToken(context){
log(context, "Inside verifyToken")
log(context, "Token is " + context.authn.token)
var options = {
"body" : "token=" + context.authn.token + "&token_type_hint=access_token"
}
return context.request.subrequest("/jwt_verify_request", options)
.then(reply=>{
if (reply.status === 200) {
var response = JSON.parse(reply.responseBody);
if (response.active === true) {
return response
} else {
throw new Error("Unauthorized: " + reply.responseBody)
}
} else {
throw new Error("Unauthorized: " + reply.responseBody)
}
}).then(verified_token => {
context.authn.verified_token =
JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString())
return context
})
}
function requestToken(context){
log(context, "Inside requestToken")
var options = {
"body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password
}
return context.request.subrequest("/jwt_request", options)
.then(reply=>{
if (reply.status === 200) {
var response = JSON.parse(reply.responseBody);
context.authn.token = response.access_token
context.authn.verified_token =
JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString())
return context
} else if (reply.status === 400 || reply.status === 401){
var options = {
"body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password
}
return context.request.subrequest("/jwt_request", options)
.then( reply=>{
if (reply.status === 200) {
var response = JSON.parse(reply.responseBody);
context.authn.token = response.access_token
context.authn.verified_token =
JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString())
return context
} else{
throw new Error("Unauthorized " + reply.status)
}
})
} else {
throw new Error("Unauthorized " + reply.status)
}
})
}
function pipExecutor(context){
log(context, "Inside extra claims PIP")
context.authz.pip.forEach(extra =>{
//call extra claim pip function
try{
var operator = extra.operator
var result = wkf[operator](context, extra.args)
//ensure array and add to extra_claims
if(!(result instanceof Array)) result = [result]
if(!context.extra_claims) context.extra_claims = {};
context.extra_claims[extra.claim] = result
} catch (error){
log(context, "Skipping invalid extra claim " + njs.dump(error))
}
})
log(context, "Extra claims are " + njs.dump(context.extra_claims))
return context
}
function pdpExecutor(context){
log(context, "Inside PDP")
return context.authz.pdp(context)
}
function umaCall(context){
log(context, "Inside UMA call")
var options = { "body" : computePermissionRequestBody(context) };
return context.request.subrequest("/permission_request", options)
.then(reply =>{
if(reply.status === 200){
return context
}else{
throw new Error("Response for authorization request is not ok " + reply.status + " " + njs.dump(reply.responseBody))
}
})
}
function pass(context){
log(context, "Inside pass");
if(typeof(context.backend) === "string") context.request.internalRedirect(context.backend);
else if (typeof(context.backend) === "function") context.request.internalRedirect(context.backend(context))
return context;
}
// ######## AUTHORIZATION PART ###############
function computePermissionRequestBody(context){
if(!context.authz.host || !context.authz.path ){
throw new Error("Enforcemnt mode is always enforcing. Host or path not found...")
}
var audience = computeAudience(context)
var grant = "grant_type=urn:ietf:params:oauth:grant-type:uma-ticket"
var mode = "response_mode=decision"
var permissions = computePermissions(context)
var extra = ""
if(context.extra_claims){
extra =
"claim_token_format=urn:ietf:params:oauth:token-type:jwt&claim_token=" +
JSON.stringify(context.extra_claims).toString("base64url")
}
var body = audience + "&" + grant + "&" + permissions + "&" + mode + "&" + extra
context.request.error("Computed permission request body is " + body)
return body
}
function computeAudience(context){
var aud = context.request.headersIn.host
if(context.authz.host){
aud = context.authz.host.audience||context.authz.host.host
}
return "audience=" + aud
}
function computePermissions(context){
var resource = context.request.uri
if(context.authz.path){
resource = context.authz.path.name||context.authz.path.path
}
var scopes = []
if(context.authz.method && context.authz.method.scopes){
scopes = context.authz.method.scopes
}
if(scopes.length > 0){
return scopes.map(s=>"permission=" + resource + "#" + s).join("&")
}
return "permission=" + resource
}
function getPath(hostconfig, incomingpath, incomingmethod){
var paths = hostconfig.paths || []
var matchingpaths = paths
.filter(p => {return incomingpath.match(p.path) != null})
.reduce((acc, p) => {
if (!p.methods || p.methods.length === 0) acc.weak.push({ path: p});
else{
var matchingmethods = p.methods.filter(m=>m.method.toUpperCase() === incomingmethod)
if(matchingmethods.length > 0) acc.strong.push({ method : matchingmethods[0], path: p});
}
return acc;
}, { strong: [], weak: []})
return matchingpaths.strong.concat(matchingpaths.weak)[0]
}
function getHost(config, host){
var matching = config.hosts.filter(h=>{
return h.host === host
})
return matching.length > 0 ? matching[0] : null
}
function computeProtection(context){
log(context, "Getting by host " + context.request.headersIn.host)
context.authz = {}
context.authz.host = getHost(context.config, context.request.headersIn.host)
if(context.authz.host !== null){
log(context, "Host found:" + context.authz.host)
context.authz.pip = context.authz.host.pip ? context.authz.host.pip : [];
context.authz.pdp = context.authz.host.pdp ? context.authz.host.pdp : umaCall;
var pathandmethod = getPath(context.authz.host, context.request.uri, context.request.method);
if(pathandmethod){
context.authz.path = pathandmethod.path;
context.authz.pip = context.authz.path.pip ? context.authz.pip.concat(context.authz.path.pip) : context.authz.pip;
context.authz.pdp = context.authz.path.pdp ? context.authz.path.pdp : context.authz.pdp;
context.authz.method = pathandmethod.method;
if(context.authz.method){
context.authz.pip = context.authz.method.pip ? context.authz.pip.concat(context.authz.method.pip) : context.authz.pip;
context.authz.pdp = context.authz.method.pdp ? context.authz.method.pdp : context.authz.pdp;
}
}
}
log(context, "Leaving protection computation: ")
return context
}

@ -0,0 +1,13 @@
$ANSIBLE_VAULT;1.1;AES256
30623131396461633063383439663039623665613038313636303861336432366664303239616339
3736366333333665396537636661383432633338663362340a616361393739343737336366333932
30643639333333626634653561383936633933613130376461306363376433326634623234343035
6431376663343165620a623664623236653264363232646238333331663433376533316361363936
38303366633438623330306464316462646536633732623938306131363735316431646133333632
63653532646566613032303138303335396366363063343666306632656465393035303563376461
30343361373338663135343436333335303632333062323430333933343937333365363164366534
39396163646432363036356339353833613431353034666237353265653966373631336364633533
33633462366638373731623139363232646164356662333530393665396532383135353565363235
35386531656636333933336333303030623232373062646165396530366263626336633462333164
35616134636166353231303166393631653636623030353738316238343764656436346338393432
36653563393836363131

@ -0,0 +1,10 @@
{
"keycloakurl" : "https://accounts.dev.d4science.org",
"keycloakrealm" : "d4science",
"user-manager-client-id" : "orchestrator",
"user-manager-client-secret" : "{{ user_manager_client_secret }}"
}

@ -0,0 +1,127 @@
{
"d4science_realm" : "d4science",
"d4science_iam_base" : "https://accounts.dev.d4science.org/auth",
"d4science_cdn_base" : "http://cdn.dev.d4science.org",
"infrastructure" : {
"id" : "%2Fgcube",
"name" : "/gcube",
"type" : "root-vo",
"children" : [
{
"id" : "%2Fgcube%2Fdevsec",
"name" : "/gcube/devsec",
"type" : "vo",
"about" : "<h5>devsec VO</h5><p>This VO of the D4Science development infrastructure.</p><p> In this VO ...</p>",
"apps" : [
{
"id" : "about",
"name" : "About",
"uri" : "d4s-navigation/about",
"description" : "Welcome message"
},
{
"id" : "vo-manage",
"name" : "Manage",
"type" : "category",
"apps" : [
{
"id" : "user-manager",
"name" : "Users",
"uri" : "d4s-vre-manager/user-manager",
"description" : "Manage users of VO and roles"
},
{
"id" : "ext-app-manager",
"name" : "External applications",
"uri" : "d4s-vre-manager/ext-app-manager",
"description" : "Manage external applications and roles"
}
],
"description" : "Manage Virtual Organization"
}
],
"children" : [
{
"id" : "%2Fgcube%2Fdevsec%2FdevVRE",
"name" : "/gcube/devsec/devVRE",
"type" : "vre",
"about" : "<h5>/gcube/devsec/devVRE</h5><p>on gcube dev infrastructure used for ...</p>",
"apps" : [
{
"id" : "about",
"name" : "About",
"uri" : "d4s-navigation/about",
"description" : "Welcome message"
},
{
"id" : "vre-manage",
"type" : "category",
"name" : "Manage",
"description" : "Manage Virtual Research Environment",
"apps" : [
{
"id" : "user-manager",
"name" : "Users",
"uri" : "d4s-vre-manager/user-manager",
"description" : "Manage users of VRE and roles"
},
{
"id" : "ext-app-manager",
"name" : "External applications",
"uri" : "d4s-vre-manager/ext-app-manager",
"description" : "Manage external applications and roles"
}
]
},
{
"id" : "analytics",
"type" : "category",
"name" : "Analytics 2",
"description" : "The new analytics framework",
"icon" : "images/batman.png",
"apps" : [
{
"id" : "ccp_workflow-admin",
"name" : "Workflows",
"uri" : "ccp/workflow-admin",
"description" : "Manage workflows of analytics engine 2"
}
]
}
]
},
{
"id" : "%2Fgcube%2Fdevsec%2FSAITEST",
"name" : "/gcube/devsec/SAITEST",
"type" : "vre",
"about" : "<h5>/gcube/devsec/SAITEST</h5><p>/gcube/devsec/SAITEST on gcube dev infrastructure used for ...</p>",
"apps" : [
{
"id" : "about",
"name" : "About",
"uri" : "d4s-navigation/about",
"description" : "Welcome message"
}
]
},
{
"id" : "%2Fgcube%2Fdevsec%2FanotherVRE4",
"name" : "/gcube/devsec/anotherVRE4",
"type" : "vre"
}
]
},
{
"id" : "%2Fgcube%2FdevNext",
"name" : "/gcube/devNext",
"children" : [
{
"id" : "%2Fgcube%2FdevNext%2FNextNext",
"name" : "/gcube/devNext/NextNext",
"type" : "vre"
}
]
}
]
}
}

@ -0,0 +1,12 @@
### 1. Get muscle classic
FROM nubisware/muscle-classic:latest
# 2. Install applicative fibers
ADD .muscle/ /root/.muscle/
RUN git fiber install -p G3 -n d4s-navigation \
&& git service create -n service -w auth -w utils -w config -w error -w utils -w inspect -w resources -w d4s-navigation \
&& rm -rf /root/.muscle
WORKDIR service
EXPOSE 8984
ENTRYPOINT ./start.sh

@ -0,0 +1,25 @@
---
- hosts: localhost
vars_files:
- conf/secrets.yaml
vars:
infrastructure: "local"
dry: false
tasks:
- name: Patch PEP config
template:
src: "conf/pep/config.js.j2"
dest: "conf/pep/config.js"
- name: Patch service auth config
template:
src: "conf/service/auth.json.j2"
dest: "conf/service/auth.json"
- name: Start swarm
docker_stack:
name: 'cdn-{{ infrastructure }}'
state: present
compose:
- "swarm.yaml"
when: dry is not defined or not dry|bool

@ -0,0 +1,61 @@
version: '3.6'
services:
d4s-navigation-pep:
image: nginx:stable-alpine
networks:
- cdn-network
deploy:
mode: replicated
replicas: 2
restart_policy:
condition: on-failure
delay: 10s
window: 120s
configs:
- source: navigation_pep_nginx_conf
target: /etc/nginx/templates/default.conf.template
- source: navigation_pep_nginx_baseconf
target: /etc/nginx/nginx.conf
- source: navigation_pep
target: /etc/nginx/pep.js
- source: navigation_pepconfig
target: /etc/nginx/config.js
d4s-navigation:
image: nubisware/d4s-navigation
networks:
- cdn-network
deploy:
mode: replicated
replicas: 2
restart_policy:
condition: on-failure
delay: 10s
window: 200s
configs:
- source: navigation_conf
target: /opt/app/service/conf/d4s-navigation.json
- source: navigation_auth_conf
target: /opt/app/service/conf/auth.json
networks:
cdn-network:
configs:
navigation_pep_nginx_conf:
file: ./conf/pep/default.conf
navigation_pep_nginx_baseconf:
file: ./conf/pep/nginx.conf
navigation_pep:
file: ./conf/pep/pep.js
navigation_pepconfig:
file: ./conf/pep/config.js
navigation_conf:
file: ./conf/service/d4s-navigation.json
navigation_auth_conf:
file: ./conf/service/auth.json
Loading…
Cancel
Save