diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d4f58be --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.9.18-slim-bullseye +RUN apt-get update && apt-get install -y nginx curl +RUN mkdir -p /var/www/html && chown -R www-data: /var/www +COPY config/*.sh /opt/ +RUN chmod 755 /opt/update-head.sh /opt/update-version.sh /opt/delete-version.sh +COPY config/default.conf /etc/nginx/conf.d/default.conf +COPY config/*.py /opt/ +COPY config/docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod 755 /docker-entrypoint.sh && chown -R www-data: /docker-entrypoint.sh + +RUN set -x \ +# create nginx user/group first, to be consistent throughout docker variants + && groupadd --system --gid 101 nginx \ + && useradd --system --gid nginx --no-create-home --home /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list + +# forward request and error logs to docker log collector +RUN ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log + +CMD ["/docker-entrypoint.sh"] diff --git a/README.md b/README.md deleted file mode 100644 index 2ce657e..0000000 --- a/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# README # - -### What is this repository for? ### - -* Overview - -This repository contains instructions to setup a base cdn service for the D4Science platform - -* Version - -0.0.1 - -### How do I get set up? ### - -* Summary of set up - -This repository contains all information to build a D4S cdn image. Most importantly it contains the ansible instructions to deploy a swarm containing ROUTER, CDN base service and PEP. - -* Deployment instructions - -There are 4 branches. - -__main__ branch contains configurations to start a local instance useful for development. It is configured to use the D4SCience dev IAM. - -__dev__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the DEV infrastructure. - -__pre__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the PRE infrastructure. - -__prod__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the PROD infrastructure. - -In order to run a local site the following commands need to be executed: - -``` -git clone https://code-repo.d4science.org/gCubeSystem/d4s-cdn-setup.git -ansible-playbook site.yaml --ask-vault-pass -``` -This will create a basic stack with an NGINX based router, a PEP and base CDN service. - -If you want to deploy on a specific D4S infrastructure please clone the corresponding branch. - -``` -git clone -b {infra} https://code-repo.d4science.org/gCubeSystem/d4s-cdn-setup.git -ansible-playbook site.yaml --ask-vault-pass -``` - -### Contribution guidelines ### - -* Adding a new CDN Island - -* Commit guidelines - -DO NOT commit the following files: - -conf/pep/config.js -conf/service/auth.js - -They will be generated by the ansible based deployment procedure in order to inject the secrets. - -### Who do I talk to? ### - -* Repo owner or admin - -Marco Lettere -marco.lettere@nubisware.com - diff --git a/build-docker-images-sh b/build-docker-images-sh deleted file mode 100755 index 7c7dded..0000000 --- a/build-docker-images-sh +++ /dev/null @@ -1 +0,0 @@ -docker build --rm --no-cache -t nubisware/d4s-cdn -f images/Dockerfile . diff --git a/build-images.sh b/build-images.sh new file mode 100755 index 0000000..290d895 --- /dev/null +++ b/build-images.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +if [ $# -eq 0 ] + then + tag='dev' + else + tag=$1 +fi + +docker build -t hub.dev.d4science.org/cdn/cdn:$tag . +docker push hub.dev.d4science.org/cdn/cdn:$tag \ No newline at end of file diff --git a/conf/pep/config.js.j2 b/conf/pep/config.js.j2 deleted file mode 100644 index cd3be8e..0000000 --- a/conf/pep/config.js.j2 +++ /dev/null @@ -1,67 +0,0 @@ -export default { config }; - -var config = { - "pep-credentials" : "{{pep_credentials}}", - "hosts" : [ - { - "host": "cdn-pep", - "audience" : "d4science-cdn", - "allow-basic-auth" : false, - "paths" : [ - { - "name" : "Default Resource", - "path" : "^/config/d4s-cdn/.+$", - "methods" : [ - { - "method" : "GET", - "scopes" : ["get"] - } - ] - }, - { - "name" : "Default Resource", - "path" : "^/visuals/d4s-cdn/.*$", - "methods" : [ - { - "method" : "GET" - } - ] - }, - { - "name" : "Default Resource", - "path" : "^/d4s-cdn/.+$", - "methods" : [ - { - "method" : "GET" - } - ] - }, - { - "name" : "Default Resource", - "path" : "^/services/d4s-cdn/.*$", - "methods" : [ - { - "method" : "OPTIONS" - }, - { - "method" : "POST" - }, - { - "method" : "HEAD" - }, - { - "method" : "PUT" - }, - { - "method" : "DELETE" - }, - { - "method" : "GET" - } - ] - } - ] - } - ] -} - diff --git a/conf/pep/default.conf b/conf/pep/default.conf deleted file mode 100644 index 4aa6dd1..0000000 --- a/conf/pep/default.conf +++ /dev/null @@ -1,90 +0,0 @@ -upstream d4s-cdn { - ip_hash; - server d4s-cdn:8984; -} - -js_var $auth_token; -js_var $pep_credentials; -js_var $user; -js_var $user_display; - -map $http_authorization $source_auth { - default ""; -} - -server { - - listen *:80; - listen [::]:80; - server_name d4s-cdn-pep; - - subrequest_output_buffer_size 200k; - - location /config/d4s-cdn/1 { - proxy_pass http://d4s-cdn; - } - - location / { - js_content pep.enforce; - } - - location /resources/ { - proxy_pass http://d4s-cdn; - } - - location @backend { - add_header Access-Control-Allow-Origin *; - add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - proxy_set_header Authorization "Bearer $auth_token"; - proxy_set_header X-User "$user"; - proxy_set_header X-User-display "$user_display"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Forwarded-Port $server_port; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Original-URI $request_uri; - proxy_pass http://d4s-cdn; - } - - location /jwt_verify_request { - internal; - gunzip on; - proxy_method POST; - proxy_http_version 1.1; - proxy_set_header Authorization $pep_credentials; - proxy_set_header Content-Type "application/x-www-form-urlencoded"; - proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token/introspect; - - proxy_cache token_responses; # Enable caching - proxy_cache_key $source_auth; # Cache for each source authentication - proxy_cache_lock on; # Duplicate tokens must wait - proxy_cache_valid 200 10s; # How long to use each response - proxy_ignore_headers Cache-Control Expires Set-Cookie; - } - - location /jwt_request { - internal; - gunzip on; - proxy_method POST; - proxy_http_version 1.1; - proxy_set_header Authorization $pep_credentials; - proxy_set_header Content-Type "application/x-www-form-urlencoded"; - proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token; - } - - location /permission_request { - internal; - gunzip on; - proxy_method POST; - proxy_http_version 1.1; - proxy_set_header Content-Type "application/x-www-form-urlencoded"; - proxy_set_header Authorization "Bearer $auth_token"; - proxy_pass https://accounts.dev.d4science.org/auth/realms/d4science/protocol/openid-connect/token; - } - -} - diff --git a/conf/pep/nginx.conf b/conf/pep/nginx.conf deleted file mode 100644 index 5796bf6..0000000 --- a/conf/pep/nginx.conf +++ /dev/null @@ -1,44 +0,0 @@ -# Added to load njs module -load_module modules/ngx_http_js_module.so; - -user nginx; -worker_processes auto; - -error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - - # added to import pep script - js_import pep.js; - - # added to bind enforce function - js_set $authorization pep.enforce; - - # added to create cache for tokens and auth calls - proxy_cache_path /var/cache/nginx/pep keys_zone=token_responses:1m max_size=2m; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - include /etc/nginx/conf.d/*.conf; -} diff --git a/conf/pep/pep.js b/conf/pep/pep.js deleted file mode 100644 index 184177d..0000000 --- a/conf/pep/pep.js +++ /dev/null @@ -1,323 +0,0 @@ -export default { enforce }; - -import defaultExport from './config.js'; - -function log(c, s){ - c.request.error(s) -} - -function enforce(r) { - - var context = { - request: r , - config : defaultExport["config"], - backend : (defaultExport.backend ? defaultExport.backend : "@backend") - } - - log(context, "Inside NJS enforce for " + r.method + " @ " + r.headersIn.host + "/" + r.uri) - - context = computeProtection(context) - - wkf.run(wkf.build(context), context) -} - -// ######## WORKFLOW FUNCTIONS ############### -var wkf = { - - build : (context)=>{ - var actions = [ - "export_pep_credentials", - "parse_authentication", - "check_authentication", - "export_authn_token", - "pip", - "pdp", - "export_backend_headers", - "pass" - ] - return actions - }, - - run : (actions, context) => { - context.request.error("Starting workflow with " + njs.dump(actions)) - var w = actions.reduce( - (acc, f) => { return acc.then(typeof(f) === "function" ? f : wkf[f]) }, - Promise.resolve().then(()=>context) - ) - w.catch(e => { context.request.error(njs.dump(e)); context.request.return(401)} ) - }, - - export_pep_credentials : exportPepCredentials, - export_authn_token : exportAuthToken, - export_backend_headers : exportBackendHeaders, - parse_authentication : parseAuthentication, - check_authentication : checkAuthentication, - verify_token : verifyToken, - request_token : requestToken, - pip : pipExecutor, - pdp : pdpExecutor, - pass : pass, - - //PIP utilities - "get-path-component" : (c, i) => c.request.uri.split("/")[i], - "get-token-field" : getTokenField, - "get-contexts" : (c) => { - var ra = c.authn.verified_token["resource_access"] - if(ra){ - var out = []; - for(var k in ra){ - if(ra[k].roles && ra[k].roles.length !== 0) out.push(k) - } - } - return out; - } -} - -function getTokenField(context, f){ - return context.authn.verified_token[f] -} - -function exportVariable(context, name, value){ - context.request.variables[name] = value - log(context, "Exported variables:" + njs.dump(context.request.variables)) - return context -} - -function exportBackendHeaders(context){ - exportVariable(context, "user", context.authn.verified_token.preferred_username) - exportVariable(context, "user_display", context.authn.verified_token.name) - return context -} - -function exportPepCredentials(context){ - if(!context.config["pep-credentials"]){ - throw new Error("Need PEP credentials") - } - return exportVariable(context, "pep_credentials", "Basic " + context.config["pep-credentials"]) -} - -function exportAuthToken(context){ - return exportVariable(context, "auth_token", context.authn.token) -} - -function checkAuthentication(context){ - return context.authn.type === "bearer" ? wkf.verify_token(context) : wkf.request_token(context) -} - -function parseAuthentication(context){ - context.request.log("Inside parseAuthentication") - var incomingauth = context.request.headersIn["Authorization"] - - if(!incomingauth) throw new Error("Authentication required"); - - var arr = incomingauth.trim().replace(/\s\s+/g, " ").split(" ") - if(arr.length != 2) throw new Error("Unknown authentication scheme"); - - var type = arr[0].toLowerCase() - if(type === "basic" && context.authz.host && context.authz.host["allow-basic-auth"]){ - var unamepass = Buffer.from(arr[1], 'base64').toString().split(":") - if(unamepass.length != 2) return null; - context.authn = { type : type, raw : arr[1], user : unamepass[0], password : unamepass[1]} - return context - }else if(type === "bearer"){ - context.authn = { type : type, raw : arr[1], token : arr[1]} - return context - } - throw new Error("Unknown authentication scheme"); -} - -function verifyToken(context){ - log(context, "Inside verifyToken") - log(context, "Token is " + context.authn.token) - var options = { - "body" : "token=" + context.authn.token + "&token_type_hint=access_token" - } - return context.request.subrequest("/jwt_verify_request", options) - .then(reply=>{ - if (reply.status === 200) { - var response = JSON.parse(reply.responseBody); - if (response.active === true) { - return response - } else { - throw new Error("Unauthorized: " + reply.responseBody) - } - } else { - throw new Error("Unauthorized: " + reply.responseBody) - } - }).then(verified_token => { - context.authn.verified_token = - JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) - return context - }) -} - -function requestToken(context){ - log(context, "Inside requestToken") - var options = { - "body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password - } - return context.request.subrequest("/jwt_request", options) - .then(reply=>{ - if (reply.status === 200) { - var response = JSON.parse(reply.responseBody); - context.authn.token = response.access_token - context.authn.verified_token = - JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) - return context - } else if (reply.status === 400 || reply.status === 401){ - var options = { - "body" : "grant_type=password&username="+context.authn.user+"&password="+context.authn.password - } - return context.request.subrequest("/jwt_request", options) - .then( reply=>{ - if (reply.status === 200) { - var response = JSON.parse(reply.responseBody); - context.authn.token = response.access_token - context.authn.verified_token = - JSON.parse(Buffer.from(context.authn.token.split('.')[1], 'base64url').toString()) - return context - } else{ - throw new Error("Unauthorized " + reply.status) - } - }) - } else { - throw new Error("Unauthorized " + reply.status) - } - }) -} - -function pipExecutor(context){ - log(context, "Inside extra claims PIP") - context.authz.pip.forEach(extra =>{ - //call extra claim pip function - try{ - var operator = extra.operator - var result = wkf[operator](context, extra.args) - //ensure array and add to extra_claims - if(!(result instanceof Array)) result = [result] - if(!context.extra_claims) context.extra_claims = {}; - context.extra_claims[extra.claim] = result - } catch (error){ - log(context, "Skipping invalid extra claim " + njs.dump(error)) - } - }) - log(context, "Extra claims are " + njs.dump(context.extra_claims)) - return context -} - -function pdpExecutor(context){ - log(context, "Inside PDP") - return context.authz.pdp(context) -} - -function umaCall(context){ - log(context, "Inside UMA call") - var options = { "body" : computePermissionRequestBody(context) }; - return context.request.subrequest("/permission_request", options) - .then(reply =>{ - if(reply.status === 200){ - return context - }else{ - throw new Error("Response for authorization request is not ok " + reply.status + " " + njs.dump(reply.responseBody)) - } - }) -} - -function pass(context){ - log(context, "Inside pass"); - if(typeof(context.backend) === "string") context.request.internalRedirect(context.backend); - else if (typeof(context.backend) === "function") context.request.internalRedirect(context.backend(context)) - return context; -} - -// ######## AUTHORIZATION PART ############### -function computePermissionRequestBody(context){ - - if(!context.authz.host || !context.authz.path ){ - throw new Error("Enforcemnt mode is always enforcing. Host or path not found...") - } - - var audience = computeAudience(context) - var grant = "grant_type=urn:ietf:params:oauth:grant-type:uma-ticket" - var mode = "response_mode=decision" - var permissions = computePermissions(context) - var extra = "" - if(context.extra_claims){ - extra = - "claim_token_format=urn:ietf:params:oauth:token-type:jwt&claim_token=" + - JSON.stringify(context.extra_claims).toString("base64url") - } - var body = audience + "&" + grant + "&" + permissions + "&" + mode + "&" + extra - context.request.error("Computed permission request body is " + body) - return body -} - -function computeAudience(context){ - var aud = context.request.headersIn.host - if(context.authz.host){ - aud = context.authz.host.audience||context.authz.host.host - } - return "audience=" + aud -} - -function computePermissions(context){ - var resource = context.request.uri - if(context.authz.path){ - resource = context.authz.path.name||context.authz.path.path - } - var scopes = [] - if(context.authz.method && context.authz.method.scopes){ - scopes = context.authz.method.scopes - } - if(scopes.length > 0){ - return scopes.map(s=>"permission=" + resource + "#" + s).join("&") - } - return "permission=" + resource -} - -function getPath(hostconfig, incomingpath, incomingmethod){ - var paths = hostconfig.paths || [] - var matchingpaths = paths - .filter(p => {return incomingpath.match(p.path) != null}) - .reduce((acc, p) => { - if (!p.methods || p.methods.length === 0) acc.weak.push({ path: p}); - else{ - var matchingmethods = p.methods.filter(m=>m.method.toUpperCase() === incomingmethod) - if(matchingmethods.length > 0) acc.strong.push({ method : matchingmethods[0], path: p}); - } - return acc; - }, { strong: [], weak: []}) - return matchingpaths.strong.concat(matchingpaths.weak)[0] -} - -function getHost(config, host){ - var matching = config.hosts.filter(h=>{ - return h.host === host - }) - return matching.length > 0 ? matching[0] : null -} - -function computeProtection(context){ - log(context, "Getting by host " + context.request.headersIn.host) - context.authz = {} - context.authz.host = getHost(context.config, context.request.headersIn.host) - if(context.authz.host !== null){ - log(context, "Host found:" + context.authz.host) - context.authz.pip = context.authz.host.pip ? context.authz.host.pip : []; - context.authz.pdp = context.authz.host.pdp ? context.authz.host.pdp : umaCall; - var pathandmethod = getPath(context.authz.host, context.request.uri, context.request.method); - if(pathandmethod){ - context.authz.path = pathandmethod.path; - context.authz.pip = context.authz.path.pip ? context.authz.pip.concat(context.authz.path.pip) : context.authz.pip; - context.authz.pdp = context.authz.path.pdp ? context.authz.path.pdp : context.authz.pdp; - context.authz.method = pathandmethod.method; - if(context.authz.method){ - context.authz.pip = context.authz.method.pip ? context.authz.pip.concat(context.authz.method.pip) : context.authz.pip; - context.authz.pdp = context.authz.method.pdp ? context.authz.method.pdp : context.authz.pdp; - } - } - } - log(context, "Leaving protection computation: ") - return context -} - diff --git a/conf/router/default.conf b/conf/router/default.conf deleted file mode 100644 index 5eb9ba8..0000000 --- a/conf/router/default.conf +++ /dev/null @@ -1,71 +0,0 @@ -server { - - listen *:80; - listen [::]:80; - - # this is the internal Docker DNS, cache only for 30s - resolver 127.0.0.11 valid=30s; - - server_name cdn.dev.d4science.org; - - location /health { - add_header Content-Length 0; - add_header Content-Type "text/plain"; - return 200; - } - - #add one such location for all new cdn islands - location ~ /.*/d4s-cdn/ { - if ($request_method = 'OPTIONS') { - add_header Access-Control-Allow-Origin '*'; - add_header Access-Control-Allow-Methods 'GET, POST, PUT, DELETE, OPTIONS'; - add_header Content-Type text/plain; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - add_header Content-Length 0; - return 204; - } - add_header Access-Control-Allow-Origin *; - add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - - # use dynamic setting of upstream so that router can be started also when not all backend cdn services are deployed - set $upstream cdn-pep; - proxy_pass http://$upstream; - } - - location ~* .*/d4s-vre-manager/ { - if ($request_method = 'OPTIONS') { - add_header Access-Control-Allow-Origin '*'; - add_header Access-Control-Allow-Methods 'GET, POST, PUT, DELETE, OPTIONS'; - add_header Content-Type text/plain; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - add_header Content-Length 0; - return 204; - } - add_header Access-Control-Allow-Origin *; - add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - - set $upstream d4s-vre-manager-pep; - proxy_pass http://$upstream; - } - - location ~* .*/d4s-navigation/ { - if ($request_method = 'OPTIONS') { - add_header Access-Control-Allow-Origin '*'; - add_header Access-Control-Allow-Methods 'GET, POST, PUT, DELETE, OPTIONS'; - add_header Content-Type text/plain; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - add_header Content-Length 0; - return 204; - } - add_header Access-Control-Allow-Origin *; - add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - - set $upstream d4s-navigation-pep; - proxy_pass http://$upstream; - } - - -} diff --git a/conf/secrets.yaml b/conf/secrets.yaml deleted file mode 100644 index 1221999..0000000 --- a/conf/secrets.yaml +++ /dev/null @@ -1,13 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -30623131396461633063383439663039623665613038313636303861336432366664303239616339 -3736366333333665396537636661383432633338663362340a616361393739343737336366333932 -30643639333333626634653561383936633933613130376461306363376433326634623234343035 -6431376663343165620a623664623236653264363232646238333331663433376533316361363936 -38303366633438623330306464316462646536633732623938306131363735316431646133333632 -63653532646566613032303138303335396366363063343666306632656465393035303563376461 -30343361373338663135343436333335303632333062323430333933343937333365363164366534 -39396163646432363036356339353833613431353034666237353265653966373631336364633533 -33633462366638373731623139363232646164356662333530393665396532383135353565363235 -35386531656636333933336333303030623232373062646165396530366263626336633462333164 -35616134636166353231303166393631653636623030353738316238343764656436346338393432 -36653563393836363131 diff --git a/conf/service/auth.json.j2 b/conf/service/auth.json.j2 deleted file mode 100644 index 90dd1a9..0000000 --- a/conf/service/auth.json.j2 +++ /dev/null @@ -1,10 +0,0 @@ -{ - - "keycloakurl" : "https://accounts.dev.d4science.org", - "keycloakrealm" : "d4science", - - "user-manager-client-id" : "orchestrator", - "user-manager-client-secret" : "{{ user_manager_client_secret }}" - -} - diff --git a/conf/service/d4s-cdn.json b/conf/service/d4s-cdn.json deleted file mode 100644 index 65302d3..0000000 --- a/conf/service/d4s-cdn.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "d4science_realm" : "d4science", - "d4science_iam_base" : "https://accounts.dev.d4science.org/auth", - "d4science_cdn_base" : "http://cdn.dev.d4science.org", - "gateways" : { - "/gcube/devsec" : ["d4science-example-wp", "dev4.d4science.org", "next.d4science.org"], - "/gcube/devNext" : ["d4science-example-wp", "dev4.d4science.org", "next.d4science.org"] - } -} diff --git a/config/api.py b/config/api.py new file mode 100644 index 0000000..06b00c3 --- /dev/null +++ b/config/api.py @@ -0,0 +1,94 @@ +# webapp.py + +from functools import cached_property +from http.cookies import SimpleCookie +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qsl, urlparse +import subprocess +import os +import json + +class WebRequestHandler(BaseHTTPRequestHandler): + @cached_property + def url(self): + return urlparse(self.path) + + @cached_property + def query_data(self): + return dict(parse_qsl(self.url.query)) + + @cached_property + def post_data(self): + content_length = int(self.headers.get("Content-Length", 0)) + return self.rfile.read(content_length) + + @cached_property + def json_data(self): + return json.loads(self.post_data.decode("utf-8")) + + @cached_property + def authorized(self): + return self.headers.get("Authorization") == os.environ["Authorization"] + + def do_GET(self): + print(self.url.path) + if self.url.path == "/health": + print("returning 200 ok") + self.send_response(200, "OK") + else: + print("returning 404") + self.send_error(404, "Not found") + + def do_POST(self): + if not self.authorized: + self.send_error(401, "Not authorized") + return + + data = self.json_data + ret = None + message = None + if (data.get("action") == "published" or data.get("action") == "updated") and data.get("release") and data.get("repository"): + if(data["repository"]["name"] == os.environ["GIT_REPOSITORY"]): + # Must be a new release of a specific version + ret = subprocess.run(["/opt/update-version.sh", data["release"]["tag_name"]]) + message = "Updating version " + data["release"]["tag_name"] + else: + message = "New release on unknown repository" + elif data.get("action") == "deleted" and data.get("release") and data.get("repository"): + if(data["repository"]["name"] == os.environ["GIT_REPOSITORY"]): + # Must be a deleteion of a release + ret = subprocess.run(["/opt/delete-version.sh", data["release"]["tag_name"]]) + message = "Deleting release" + else: + message = "Deleted release on unknown repository" + elif data.get("action") == "closed" and data.get("pull_request"): + pr = data.get("pull_request") + if pr["merged"] and data["repository"]["name"] == os.environ["GIT_REPOSITORY"] and pr["base"]["ref"] == os.environ["GIT_BRANCH"]: + # Must be a pull request onto the declared branch + ret = subprocess.run(["/opt/update-head.sh"]) + message = "Updating head after PR" + else: + message = "Closed PR on unknown repository" + elif data.get("commits") and data.get("pusher") and data.get("repository") and data.get("ref"): + if(data["repository"]["name"] == os.environ["GIT_REPOSITORY"] and data.get("ref") == "refs/heads/" + os.environ["GIT_BRANCH"]): + # Must be a push to the branch + ret = subprocess.run(["/opt/update-head.sh"]) + message = "Updating head after push" + else: + message = "Push onto unknown branch or repository" + else: + message = "Unknwown operation" + + if ret != None and ret.returncode == 0: + self.send_response(200, message) + else: + self.send_error(500, message) + + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(message.encode("UTF-8")) + self.wfile.flush() + +if __name__ == "__main__": + server = HTTPServer(("0.0.0.0", 8000), WebRequestHandler) + server.serve_forever() \ No newline at end of file diff --git a/config/default.conf b/config/default.conf new file mode 100644 index 0000000..24f9dd8 --- /dev/null +++ b/config/default.conf @@ -0,0 +1,40 @@ +server { + listen 80; + listen [::]:80; + server_name localhost cdn.dev.d4science.org cdn.cloud-dev.d4science.org cdn.cloud.d4science.org cdn.d4science.org; + + #error_log /var/log/nginx/localhost.error_log debug; + root /var/www/html; + + #access_log /var/log/nginx/host.access.log main; + + location / { + if ($request_method = 'OPTIONS') { + add_header Access-Control-Allow-Origin '*'; + add_header Access-Control-Allow-Methods 'GET, POST, PUT, DELETE, HEAD, OPTIONS'; + add_header Content-Type text/plain; + add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin, Location"; + add_header Content-Length 0; + add_header Access-Control-Allow-Private-Network true; + return 204; + } + try_files $uri /head$uri =404; + } + + location /health { + return 200; + } + + location /api/update/ { + proxy_pass http://localhost:8000; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/config/delete-version.sh b/config/delete-version.sh new file mode 100644 index 0000000..488f786 --- /dev/null +++ b/config/delete-version.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf /var/www/html/$1 \ No newline at end of file diff --git a/config/docker-entrypoint.sh b/config/docker-entrypoint.sh new file mode 100644 index 0000000..2f73906 --- /dev/null +++ b/config/docker-entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +python /opt/init.py && python /opt/api.py & +nginx -g 'daemon off;' \ No newline at end of file diff --git a/config/init.py b/config/init.py new file mode 100644 index 0000000..9468aa8 --- /dev/null +++ b/config/init.py @@ -0,0 +1,25 @@ +from functools import cached_property +from http.cookies import SimpleCookie +from urllib import request +import subprocess +import os +import json + +if __name__ == "__main__": + + # intiialize head of branch + subprocess.run(["/opt/update-head.sh"]) + + # pull and initialize all releases + api_url_parts = os.environ["GIT_SERVER"].split("/") + api_url_parts.insert(-1, "api/v1/repos") + api_url_parts.append(os.environ["GIT_REPOSITORY"] + "/releases") + api_url = "/".join(api_url_parts) + releases = [] + with request.urlopen(api_url) as data: + releases = json.loads(data.read()) + + if releases != None: + for r in releases: + if r["target_commitish"] == os.environ["GIT_BRANCH"]: + subprocess.run(["/opt/update-version.sh", r["tag_name"]]) \ No newline at end of file diff --git a/config/update-head.sh b/config/update-head.sh new file mode 100644 index 0000000..a4885b0 --- /dev/null +++ b/config/update-head.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# EXAMPLES +#GIT_BRANCH=master +#GIT_REPOSITORY=web-components + +curl $GIT_SERVER/$GIT_REPOSITORY/archive/$GIT_BRANCH.tar.gz -o /var/www/$GIT_BRANCH.tgz +tar xzf /var/www/$GIT_BRANCH.tgz -C /var/www +rm -rf /var/www/html/head +mv /var/www/$GIT_REPOSITORY /var/www/html/head +rm -rf /var/www/$GIT_BRANCH.tgz \ No newline at end of file diff --git a/config/update-version.sh b/config/update-version.sh new file mode 100644 index 0000000..c8db4be --- /dev/null +++ b/config/update-version.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +v=$1 +archive=$v.tar.gz + +# EXAMPLES +#GIT_SERVER=https://code-repo.d4science.org/gCubeSystem +#GIT_REPOSITORY=web-components + +curl -L $GIT_SERVER/$GIT_REPOSITORY/archive/$archive -o /var/www/$archive +tar xzvf /var/www/$archive -C /var/www +rm -f /var/wwww/html/$v +mv /var/www/$GIT_REPOSITORY /var/www/html/$v +rm -rf /var/www/$archive \ No newline at end of file diff --git a/images/Dockerfile b/images/Dockerfile deleted file mode 100644 index 6adba2f..0000000 --- a/images/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -### 1. Get muscle classic -FROM nubisware/muscle-classic:latest - -# 2. Install applicative fibers -ADD .muscle/ /root/.muscle/ -RUN git fiber install -p G3 -n d4s-cdn \ - && git service create -n cdn -w auth -w utils -w config -w error -w utils -w inspect -w resources -w d4s-cdn --debug \ - && rm -rf /root/.muscle - -WORKDIR cdn -EXPOSE 8984 -ENTRYPOINT ./start.sh diff --git a/site.yaml b/site.yaml deleted file mode 100644 index 3185d5d..0000000 --- a/site.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- hosts: localhost - vars_files: - - conf/secrets.yaml - vars: - infrastructure: "dev" - dry: false - tasks: - - name: Patch PEP config - template: - src: "conf/pep/config.js.j2" - dest: "conf/pep/config.js" - - - name: Patch service auth config - template: - src: "conf/service/auth.json.j2" - dest: "conf/service/auth.json" - - - name: Start swarm - docker_stack: - name: 'cdn-{{ infrastructure }}' - state: present - compose: - - "swarm.yaml" - when: dry is not defined or not dry|bool diff --git a/stacks/stack-dev.yaml b/stacks/stack-dev.yaml new file mode 100644 index 0000000..ff98778 --- /dev/null +++ b/stacks/stack-dev.yaml @@ -0,0 +1,35 @@ +version: '3.6' + +services: + cdn: + image: hub.dev.d4science.org/cdn/cdn:dev + networks: + - haproxy-public + healthcheck: + test: curl http://localhost/health + interval: 10s + timeout: 5s + retries: 12 + + deploy: + mode: replicated + replicas: 1 + endpoint_mode: dnsrr + restart_policy: + condition: any + delay: 5s + window: 120s + placement: + constraints: [node.role == worker] + + environment: + GIT_BRANCH: master + GIT_SERVER: https://code-repo.d4science.org/gCubeSystem + GIT_REPOSITORY: web-components + Authorization: ${cdn_credentials} + logging: + driver: "journald" + +networks: + haproxy-public: + external: true \ No newline at end of file diff --git a/stacks/stack-local.yaml b/stacks/stack-local.yaml new file mode 100644 index 0000000..d489508 --- /dev/null +++ b/stacks/stack-local.yaml @@ -0,0 +1,23 @@ +version: '3.6' + +services: + cdn: + image: hub.dev.d4science.org/cdn/cdn:dev + ports: + - "8080:80" + healthcheck: + test: curl http://localhost/health + interval: 5s + timeout: 5s + retries: 12 + volumes: + - /home/lettere/git/isti/d4s-cdn-setup/config/api.py:/opt/api.py + - /home/lettere/git/isti/d4s-cdn-setup/config/init.py:/opt/init.py + - /home/lettere/git/isti/d4s-cdn-setup/config/default.conf:/etc/nginx/conf.d/default.conf + environment: + GIT_BRANCH: master + GIT_SERVER: https://code-repo.d4science.org/gCubeSystem + GIT_REPOSITORY: web-components + Authorization: ${cd_vredentials} + logging: + driver: "journald" diff --git a/stacks/stack-prod.yaml b/stacks/stack-prod.yaml new file mode 100644 index 0000000..3ba9391 --- /dev/null +++ b/stacks/stack-prod.yaml @@ -0,0 +1,35 @@ +version: '3.6' + +services: + cdn: + image: hub.dev.d4science.org/cdn/cdn:prod + networks: + - haproxy-public + healthcheck: + test: curl http://localhost/health + interval: 10s + timeout: 5s + retries: 12 + + deploy: + mode: replicated + replicas: 1 + endpoint_mode: dnsrr + restart_policy: + condition: any + delay: 5s + window: 120s + placement: + constraints: [node.role == worker] + + environment: + GIT_BRANCH: prod + GIT_SERVER: https://code-repo.d4science.org/gCubeSystem + GIT_REPOSITORY: web-components + Authorization: ${cdn_credentials} + logging: + driver: "journald" + +networks: + haproxy-public: + external: true diff --git a/swarm.yaml b/swarm.yaml deleted file mode 100644 index 2ea283a..0000000 --- a/swarm.yaml +++ /dev/null @@ -1,83 +0,0 @@ -version: '3.6' - -services: - - cdn-router-dev: - image: nginx:stable-alpine - networks: - - cdn-network - - haproxy-public - deploy: - mode: replicated - endpoint_mode: dnsrr - replicas: 2 - restart_policy: - condition: on-failure - delay: 10s - window: 120s - configs: - - source: nginx_router_conf - target: /etc/nginx/templates/default.conf.template - - cdn-pep: - image: nginx:stable-alpine - networks: - - cdn-network - deploy: - mode: replicated - replicas: 2 - restart_policy: - condition: on-failure - delay: 10s - window: 120s - configs: - - source: cdn_pep_conf - target: /etc/nginx/templates/default.conf.template - - source: cdn_pep_baseconf - target: /etc/nginx/nginx.conf - - source: cdn_pep - target: /etc/nginx/pep.js - - source: cdn_pepconfig - target: /etc/nginx/config.js - - - d4s-cdn: - image: nubisware/d4s-cdn - networks: - - cdn-network - deploy: - mode: replicated - replicas: 2 - restart_policy: - condition: on-failure - delay: 10s - window: 200s - configs: - - source: cdn_conf - target: /opt/app/cdn/conf/d4s-cdn.json - - source: cdn_auth_conf - target: /opt/app/cdn/conf/auth.json - -networks: - cdn-network: - haproxy-public: - external: true - -configs: - nginx_router_conf: - file: ./conf/router/default.conf - - cdn_pep_conf: - file: ./conf/pep/default.conf - cdn_pep_baseconf: - file: ./conf/pep/nginx.conf - cdn_pep: - file: ./conf/pep/pep.js - cdn_pepconfig: - file: ./conf/pep/config.js - - cdn_conf: - file: ./conf/service/d4s-cdn.json - cdn_auth_conf: - file: ./conf/service/auth.json -