diff --git a/v3.4/Dockerfile-frontend b/v3.4/Dockerfile-frontend new file mode 100644 index 0000000..a60a35b --- /dev/null +++ b/v3.4/Dockerfile-frontend @@ -0,0 +1,9 @@ +FROM nginx:alpine + +LABEL maintainer="Nubisware " + +# Bake common configurations for Conductor PEP +COPY config/nginx/nginx.conf /etc/nginx/nginx.conf +COPY config/nginx/pep.js /etc/nginx/pep.js +COPY config/nginx/config.js /etc/nginx/config.js + diff --git a/v3.4/Dockerfile-frontend-dev b/v3.4/Dockerfile-frontend-dev index 5f25e50..a677d17 100644 --- a/v3.4/Dockerfile-frontend-dev +++ b/v3.4/Dockerfile-frontend-dev @@ -1,7 +1,15 @@ -FROM nubisware/conductor-frontend +FROM nubisware/conductor-frontend:common LABEL maintainer="Nubisware " +# Ensure that cache is invalidated +ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache + +# Copy compiled UI assets to nginx www directory +WORKDIR /usr/share/nginx/html +RUN rm -rf ./* +COPY build/ . + # Copy NGINX default configuration -COPY ./config/nginx/conf.d/default-dev.conf /etc/nginx/conf.d/default.conf +COPY ./config.dev/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf diff --git a/v3.4/Dockerfile-frontend-pre b/v3.4/Dockerfile-frontend-pre index 05309ea..8777208 100644 --- a/v3.4/Dockerfile-frontend-pre +++ b/v3.4/Dockerfile-frontend-pre @@ -1,7 +1,15 @@ -FROM nubisware/conductor-frontend +FROM nubisware/conductor-frontend:common LABEL maintainer="Nubisware " +# Ensure that cache is invalidated +ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache + +# Copy compiled UI assets to nginx www directory +WORKDIR /usr/share/nginx/html +RUN rm -rf ./* +COPY build/ . + # Copy NGINX default configuration -COPY ./config/nginx/conf.d/default-pre.conf /etc/nginx/conf.d/default.conf +COPY ./config.pre/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf diff --git a/v3.4/build-dev-images.sh b/v3.4/build-dev-images.sh index 62f56ef..77fc2e3 100755 --- a/v3.4/build-dev-images.sh +++ b/v3.4/build-dev-images.sh @@ -1,5 +1,13 @@ -#docker build -t nubisware/conductor-server3:dev -f Dockerfile-server-dev . +docker build -t nubisware/conductor-server3:dev -f Dockerfile-server-dev . #docker push nubisware/conductor-server3:dev +docker build -t nubisware/conductor-frontend:common -f Dockerfile-frontend . + +cd /home/lettere/git/conductor/ui/ +./build-dev-code.sh +cd - +cp -r /home/lettere/git/conductor/ui/build . docker build -t nubisware/conductor-frontend:dev -f Dockerfile-frontend-dev . -docker push nubisware/conductor-frontend:dev +rm -rf build + +#docker push nubisware/conductor-frontend:dev diff --git a/v3.4/build-pre-images.sh b/v3.4/build-pre-images.sh index 9a33bd2..7b6a106 100755 --- a/v3.4/build-pre-images.sh +++ b/v3.4/build-pre-images.sh @@ -1,5 +1,12 @@ -docker build -t nubisware/conductor-server3:pre -f Dockerfile-server-pre . -docker push nubisware/conductor-server3:pre +#docker build -t nubisware/conductor-server3:pre -f Dockerfile-server-pre . +#docker push nubisware/conductor-server3:pre +docker build -t nubisware/conductor-frontend:common -f Dockerfile-frontend . + +cd /home/lettere/git/conductor/ui/ +./build-pre-code.sh +cd - +cp -r /home/lettere/git/conductor/ui/build . docker build -t nubisware/conductor-frontend:pre -f Dockerfile-frontend-pre . -docker push nubisware/conductor-frontend:pre +rm -rf build +#docker push nubisware/conductor-frontend:pre diff --git a/v3.4/config/base-workers.cfg b/v3.4/config.dev/base-workers.cfg similarity index 100% rename from v3.4/config/base-workers.cfg rename to v3.4/config.dev/base-workers.cfg diff --git a/v3.4/config/nginx/conf.d/default-dev.conf b/v3.4/config.dev/nginx/conf.d/default.conf similarity index 97% rename from v3.4/config/nginx/conf.d/default-dev.conf rename to v3.4/config.dev/nginx/conf.d/default.conf index 7b11c9e..067549e 100644 --- a/v3.4/config/nginx/conf.d/default-dev.conf +++ b/v3.4/config.dev/nginx/conf.d/default.conf @@ -12,7 +12,7 @@ js_var $pep_credentials; server { listen 80; - server_name conductor; + server_name conductor conductor-dev.int.d4science.net; location / { # This would be the directory where your React app's static files are stored at diff --git a/v3.4/config/base-workers-dev.cfg b/v3.4/config.pre/base-workers.cfg similarity index 86% rename from v3.4/config/base-workers-dev.cfg rename to v3.4/config.pre/base-workers.cfg index 00c11e2..1a38177 100644 --- a/v3.4/config/base-workers-dev.cfg +++ b/v3.4/config.pre/base-workers.cfg @@ -5,7 +5,7 @@ pollrate = 1 [pymail] server = smtp-relay.d4science.org -user = conductor_dev +user = conductor_pre password = protocol = starttls port = 587 diff --git a/v3.4/config.pre/config-pg-es7.properties b/v3.4/config.pre/config-pg-es7.properties new file mode 100755 index 0000000..5e69649 --- /dev/null +++ b/v3.4/config.pre/config-pg-es7.properties @@ -0,0 +1,25 @@ +# Database persistence type. +conductor.db.type=postgres + +spring.datasource.url=jdbc:postgresql://postgres:5432/conductor +spring.datasource.username=conductor +spring.datasource.password=conductor + +# Hikari pool sizes are -1 by default and prevent startup +spring.datasource.hikari.maximum-pool-size=10 +spring.datasource.hikari.minimum-idle=2 + +# Elastic search instance indexing is disabled. +conductor.indexing.enabled=true +conductor.elasticsearch.version=7 +conductor.elasticsearch.url=http://es:9200 + +#Enable Prometheus +conductor.metrics-prometheus.enabled=true +management.endpoints.web.exposure.include=prometheus,health,info,metrics + +# GRPC disabled +conductor.grpc-server.enabled=false + +# Load sample kitchen sink disabled +loadSample=false diff --git a/v3.4/config.pre/config-simple.properties b/v3.4/config.pre/config-simple.properties new file mode 100755 index 0000000..9a040e4 --- /dev/null +++ b/v3.4/config.pre/config-simple.properties @@ -0,0 +1,12 @@ +# In memory database persistence model. +conductor.db.type=memory + +# Elastic search instance indexing is disabled. +conductor.indexing.enabled=false +conductor.elasticsearch.clusterHealthColor=yellow + +# GRPC disabled +conductor.grpc-server.enabled=false + +# Load sample kitchen sink disabled +loadSample=false diff --git a/v3.4/config/nginx/conf.d/default-pre.conf b/v3.4/config.pre/nginx/conf.d/default.conf similarity index 92% rename from v3.4/config/nginx/conf.d/default-pre.conf rename to v3.4/config.pre/nginx/conf.d/default.conf index 6644379..ac286dc 100644 --- a/v3.4/config/nginx/conf.d/default-pre.conf +++ b/v3.4/config.pre/nginx/conf.d/default.conf @@ -12,7 +12,7 @@ js_var $pep_credentials; server { listen 80; - server_name conductor; + server_name conductor.pre.d4science.org; location / { # This would be the directory where your React app's static files are stored at @@ -50,10 +50,10 @@ server { proxy_http_version 1.1; proxy_set_header Authorization $pep_credentials; proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_set_header Accept-Encoding identity; proxy_pass "https://accounts.pre.d4science.org/auth/realms/d4science/protocol/openid-connect/token/introspect"; proxy_ignore_headers Cache-Control Expires Set-Cookie; - gunzip on; proxy_cache token_responses; # Enable caching proxy_cache_key $source_auth; # Cache for each source authentication @@ -67,8 +67,9 @@ server { proxy_http_version 1.1; proxy_set_header Authorization $pep_credentials; proxy_set_header Content-Type "application/x-www-form-urlencoded"; + proxy_set_header Accept-Encoding identity; + proxy_pass "https://accounts.pre.d4science.org/auth/realms/d4science/protocol/openid-connect/token"; - gunzip on; } location /permission_request { @@ -77,8 +78,8 @@ server { proxy_http_version 1.1; proxy_set_header Content-Type "application/x-www-form-urlencoded"; proxy_set_header Authorization "Bearer $auth_token"; + proxy_set_header Accept-Encoding identity; proxy_pass "https://accounts.pre.d4science.org/auth/realms/d4science/protocol/openid-connect/token"; - gunzip on; } } diff --git a/v3.4/config/nginx/config.js b/v3.4/config/nginx/config.js new file mode 100644 index 0000000..94a4ed3 --- /dev/null +++ b/v3.4/config/nginx/config.js @@ -0,0 +1,139 @@ +export default { config }; + +var config = { + "hosts" : [ + { + "host": ["conductor.d4science.org", "conductor.pre.d4science.org", "conductor.dev.d4science.org", "conductor.int.d4science.net", "conductor"], + "audience" : "conductor-server", + "allow-basic-auth" : true, + "paths" : [ + { + "name" : "metadata", + "path" : "^/api/metadata/(taskdefs|workflow)/?.*$", + "methods" : [ + { + "method" : "GET", + "scopes" : ["get","list"] + } + ] + }, + { + "name" : "metadata.taskdefs", + "path" : "^/api/metadata/taskdefs/?.*$", + "methods" : [ + { + "method" : "POST", + "scopes" : ["create"] + }, + { + "method" : "DELETE", + "scopes" : ["delete"], + }, + { + "method" : "PUT", + "scopes" : ["update"], + } + ] + }, + { + "name" : "metadata.workflow", + "path" : "^/api/metadata/workflow/?.*$", + "methods" : [ + { + "method" : "POST", + "scopes" : ["create"] + }, + { + "method" : "DELETE", + "scopes" : ["delete"], + }, + { + "method" : "PUT", + "scopes" : ["update"], + } + ] + }, + { + "name" : "workflow", + "path" : "^/api/workflow/?.*$", + "methods" : [ + { + "method" : "GET", + "scopes" : ["get"], + }, + { + "method" : "POST", + "scopes" : ["start"], + }, + { + "method" : "DELETE", + "scopes" : ["terminate"], + } + ] + }, + { + "name" : "event", + "path" : "^/api/event/?.*$", + "methods" : [ + { + "method" : "GET", + "scopes" : ["get"], + }, + { + "method" : "POST", + "scopes" : ["create"], + }, + { + "method" : "DELETE", + "scopes" : ["delete"], + }, + { + "method" : "PUT", + "scopes" : ["update"], + } + ] + }, + { + "name" : "task", + "path" : "^/api/tasks/poll/.+$", + "methods" : [ + { + "method" : "GET", + "scopes" : ["poll"], + } + ] + }, + { + "name" : "queue", + "path" : "^/api/tasks/queue/.+$", + "methods" : [ + { + "method" : "GET", + "scopes" : ["get"], + } + ] + }, + { + "name" : "task", + "path" : "^/api/tasks[/]?$", + "methods" : [ + { + "method" : "POST", + "scopes" : ["update"], + } + ] + }, + { + "name" : "log", + "path" : "^/api/tasks/.+/log$", + "methods" : [ + { + "method" : "GET", + "scopes" : ["get"], + } + ] + } + ] + } + ] +} diff --git a/v3.4/config/nginx/nginx.conf b/v3.4/config/nginx/nginx.conf new file mode 100644 index 0000000..5796bf6 --- /dev/null +++ b/v3.4/config/nginx/nginx.conf @@ -0,0 +1,44 @@ +# Added to load njs module +load_module modules/ngx_http_js_module.so; + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + + # added to import pep script + js_import pep.js; + + # added to bind enforce function + js_set $authorization pep.enforce; + + # added to create cache for tokens and auth calls + proxy_cache_path /var/cache/nginx/pep keys_zone=token_responses:1m max_size=2m; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/v3.4/config/nginx/pep.js b/v3.4/config/nginx/pep.js new file mode 100644 index 0000000..d9bbed0 --- /dev/null +++ b/v3.4/config/nginx/pep.js @@ -0,0 +1,332 @@ +export default { enforce }; + +import defaultExport from './config.js'; + +function log(c, s){ + c.request.error(s) +} + +var _debug = true +function debug(c, s){ + if(_debug === true){ + log(c, s) + } +} + +function enforce(r) { + + var context = { + request: r , + config : defaultExport["config"], + backend : (defaultExport.backend ? defaultExport.backend : "@backend") + } + + log(context, "Inside NJS enforce for " + r.method + " @ " + r.headersIn.host + "/" + r.uri) + + context = computeProtection(context) + + wkf.run(wkf.build(context), context) +} + +// ######## WORKFLOW FUNCTIONS ############### +var wkf = { + + build : (context)=>{ + var actions = [ + "export_pep_credentials", + "parse_authentication", + "check_authentication", + "export_authn_token", + "pip", + "pdp", + "export_backend_headers", + "pass" + ] + return actions + }, + + run : (actions, context) => { + context.request.error("Starting workflow with " + njs.dump(actions)) + var w = actions.reduce( + (acc, f) => { return acc.then(typeof(f) === "function" ? f : wkf[f]) }, + Promise.resolve().then(()=>context) + ) + w.catch(e => { context.request.error(njs.dump(e)); context.request.return(401)} ) + }, + + export_pep_credentials : exportPepCredentials, + export_authn_token : exportAuthToken, + export_backend_headers : exportBackendHeaders, + parse_authentication : parseAuthentication, + check_authentication : checkAuthentication, + verify_token : verifyToken, + request_token : requestToken, + pip : pipExecutor, + pdp : pdpExecutor, + pass : pass, + + //PIP utilities + "get-path-component" : (c, i) => c.request.uri.split("/")[i], + "get-token-field" : getTokenField, + "get-contexts" : (c) => { + var ra = c.authn.verified_token["resource_access"] + if(ra){ + var out = []; + for(var k in ra){ + if(ra[k].roles && ra[k].roles.length !== 0) out.push(k) + } + } + return out; + } +} + +function getTokenField(context, f){ + return context.authn.verified_token[f] +} + +function exportVariable(context, name, value){ + context.request.variables[name] = value + return context +} + +function exportBackendHeaders(context){ + return context +} + +function exportPepCredentials(context){ + if(process.env["pep_credentials"]){ + return exportVariable(context, "pep_credentials", "Basic " + process.env["pep_credentials"]) + }else if(context.config["pep_credentials"]){ + return exportVariable(context, "pep_credentials", "Basic " + context.config["pep_credentials"]) + }else{ + throw new Error("Need PEP credentials") + } +} + +function exportAuthToken(context){ + return exportVariable(context, "auth_token", context.authn.token) +} + +function checkAuthentication(context){ + return context.authn.type === "bearer" ? wkf.verify_token(context) : wkf.request_token(context) +} + +function parseAuthentication(context){ + context.request.log("Inside parseAuthentication") + var incomingauth = context.request.headersIn["Authorization"] + + if(!incomingauth) throw new Error("Authentication required"); + + var arr = incomingauth.trim().replace(/\s\s+/g, " ").split(" ") + if(arr.length != 2) throw new Error("Unknown authentication scheme"); + + var type = arr[0].toLowerCase() + if(type === "basic" && context.authz.host && context.authz.host["allow-basic-auth"]){ + var unamepass = Buffer.from(arr[1], 'base64').toString().split(":") + if(unamepass.length != 2) return null; + context.authn = { type : type, raw : arr[1], user : unamepass[0], password : unamepass[1]} + return context + }else if(type === "bearer"){ + context.authn = { type : type, raw : arr[1], token : arr[1]} + return context + } + throw new Error("Unknown authentication scheme"); +} + +function verifyToken(context){ + log(context, "Inside verifyToken") + debug(context, "Token is " + context.authn.token) + var options = { + "body" : "token=" + context.authn.token + "&token_type_hint=access_token" + } + return context.request.subrequest("/jwt_verify_request", options) + .then(reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseBody); + if (response.active === true) { + return response + } else { + throw new Error("Unauthorized: " + reply.responseBody) + } + } else { + throw new Error("Unauthorized: " + reply.responseBody) + } + }).then(verified_token => { + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + }) +} + +function requestToken(context){ + log(context, "Inside requestToken") + var options = { + "body" : "grant_type=client_credentials&client_id="+context.authn.user+"&client_secret="+context.authn.password + } + return context.request.subrequest("/jwt_request", options) + .then(reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseBody); + context.authn.token = response.access_token + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + } else if (reply.status === 400 || reply.status === 401){ + var options = { + "body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password + } + return context.request.subrequest("/jwt_request", options) + .then( reply=>{ + if (reply.status === 200) { + var response = JSON.parse(reply.responseBody); + context.authn.token = response.access_token + context.authn.verified_token = + JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) + return context + } else{ + throw new Error("Unauthorized " + reply.status) + } + }) + } else { + throw new Error("Unauthorized " + reply.status) + } + }) +} + +function pipExecutor(context){ + log(context, "Inside extra claims PIP") + context.authz.pip.forEach(extra =>{ + //call extra claim pip function + try{ + var operator = extra.operator + var result = wkf[operator](context, extra.args) + //ensure array and add to extra_claims + if(!(result instanceof Array)) result = [result] + if(!context.extra_claims) context.extra_claims = {}; + context.extra_claims[extra.claim] = result + } catch (error){ + log(context, "Skipping invalid extra claim " + njs.dump(error)) + } + }) + log(context, "Extra claims are " + njs.dump(context.extra_claims)) + return context +} + +function pdpExecutor(context){ + log(context, "Inside PDP") + return context.authz.pdp(context) +} + +function umaCall(context){ + log(context, "Inside UMA call") + var options = { "body" : computePermissionRequestBody(context) }; + return context.request.subrequest("/permission_request", options) + .then(reply =>{ + if(reply.status === 200){ + debug(context, "UMA call reply is " + reply.status) + return context + }else{ + throw new Error("Response for authorization request is not ok " + reply.status + " " + njs.dump(reply.responseBody)) + } + }) +} + +function pass(context){ + log(context, "Inside pass"); + if(typeof(context.backend) === "string") context.request.internalRedirect(context.backend); + else if (typeof(context.backend) === "function") context.request.internalRedirect(context.backend(context)) + return context; +} + +// ######## AUTHORIZATION PART ############### +function computePermissionRequestBody(context){ + + if(!context.authz.host || !context.authz.path ){ + throw new Error("Enforcemnt mode is always enforcing. Host or path not found...") + } + + var audience = computeAudience(context) + var grant = "grant_type=urn:ietf:params:oauth:grant-type:uma-ticket" + var mode = "response_mode=decision" + var permissions = computePermissions(context) + var extra = "" + if(context.extra_claims){ + extra = + "claim_token_format=urn:ietf:params:oauth:token-type:jwt&claim_token=" + + JSON.stringify(context.extra_claims).toString("base64url") + } + var body = audience + "&" + grant + "&" + permissions + "&" + mode + "&" + extra + context.request.error("Computed permission request body is " + body) + return body +} + +function computeAudience(context){ + var aud = context.request.headersIn.host + if(context.authz.host){ + aud = context.authz.host.audience||context.authz.host.host + } + return "audience=" + aud +} + +function computePermissions(context){ + var resource = context.request.uri + if(context.authz.path){ + resource = context.authz.path.name||context.authz.path.path + } + var scopes = [] + if(context.authz.method && context.authz.method.scopes){ + scopes = context.authz.method.scopes + } + if(scopes.length > 0){ + return scopes.map(s=>"permission=" + resource + "#" + s).join("&") + } + return "permission=" + resource +} + +function getPath(hostconfig, incomingpath, incomingmethod){ + var paths = hostconfig.paths || [] + var matchingpaths = paths + .filter(p => {return incomingpath.match(p.path) != null}) + .reduce((acc, p) => { + if (!p.methods || p.methods.length === 0) acc.weak.push({ path: p}); + else{ + var matchingmethods = p.methods.filter(m=>m.method.toUpperCase() === incomingmethod) + if(matchingmethods.length > 0) acc.strong.push({ method : matchingmethods[0], path: p}); + } + return acc; + }, { strong: [], weak: []}) + return matchingpaths.strong.concat(matchingpaths.weak)[0] +} + +function getHost(config, host){ + var matching = config.hosts.filter(h=>{ + //compare for both string and array of strings + return ((h.host.filter && h.host.indexOf(host) !== -1) || h.host === host) + }) + return matching.length > 0 ? matching[0] : null +} + +function computeProtection(context){ + debug(context, "Getting by host " + context.request.headersIn.host) + context.authz = {} + context.authz.host = getHost(context.config, context.request.headersIn.host) + if(context.authz.host !== null){ + log(context, "Host found:" + context.authz.host) + context.authz.pip = context.authz.host.pip ? context.authz.host.pip : []; + context.authz.pdp = context.authz.host.pdp ? context.authz.host.pdp : umaCall; + var pathandmethod = getPath(context.authz.host, context.request.uri, context.request.method); + if(pathandmethod){ + context.authz.path = pathandmethod.path; + context.authz.pip = context.authz.path.pip ? context.authz.pip.concat(context.authz.path.pip) : context.authz.pip; + context.authz.pdp = context.authz.path.pdp ? context.authz.path.pdp : context.authz.pdp; + context.authz.method = pathandmethod.method; + if(context.authz.method){ + context.authz.pip = context.authz.method.pip ? context.authz.pip.concat(context.authz.method.pip) : context.authz.pip; + context.authz.pdp = context.authz.method.pdp ? context.authz.method.pdp : context.authz.pdp; + } + } + } + debug(context, "Leaving protection computation: ") + return context +} +