From 7ba5f20cd4a01b6707e2c4909939a22c4b5342be Mon Sep 17 00:00:00 2001 From: dcore94 Date: Wed, 5 Jan 2022 17:39:07 +0100 Subject: [PATCH] prepared for first share --- .gitignore | 47 ++-------------- README.md | 57 +++++++++++++++++--- build-docker-images-sh | 2 +- cdn-base-swarm.yaml | 62 --------------------- conf/cdn/auth.json | 24 --------- conf/cdn/d4s-cdn.json | 119 ----------------------------------------- conf/pep/default.conf | 22 +++----- conf/pep/pep.js | 1 + images/Dockerfile-cdn | 11 ---- 9 files changed, 61 insertions(+), 284 deletions(-) delete mode 100644 cdn-base-swarm.yaml delete mode 100644 conf/cdn/auth.json delete mode 100644 conf/cdn/d4s-cdn.json delete mode 100644 images/Dockerfile-cdn diff --git a/.gitignore b/.gitignore index b24d71e..a984a3c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,47 +4,6 @@ # https://www.atlassian.com/git/tutorials/saving-changes/gitignore # Node artifact files -node_modules/ -dist/ - -# Compiled Java class files -*.class - -# Compiled Python bytecode -*.py[cod] - -# Log files -*.log - -# Package files -*.jar - -# Maven -target/ -dist/ - -# JetBrains IDE -.idea/ - -# Unit test reports -TEST*.xml - -# Generated by MacOS -.DS_Store - -# Generated by Windows -Thumbs.db - -# Applications -*.app -*.exe -*.war - -# Large media files -*.mp4 -*.tiff -*.avi -*.flv -*.mov -*.wmv - +/conf/pep/config.js +/conf/service/auth.json +/.muscle diff --git a/README.md b/README.md index 39af52c..b658aef 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,70 @@ # README # -This README would normally document whatever steps are necessary to get your application up and running. - ### What is this repository for? ### -* Quick summary +* Overview + +This repository contains instructions to setup a base cdn service for the D4Science platform + * Version -* [Learn Markdown](https://bitbucket.org/tutorials/markdowndemo) + +0.0.1 ### How do I get set up? ### * Summary of set up + +This repository contains all information to build a D4S cdn image. Most importantly it contains the ansible instructions to deploy a swarm containing ROUTER, CDN base service and PEP. + * Configuration * Dependencies * Database configuration * How to run tests + * Deployment instructions +There are 4 branches. + +__main__ branch contains configurations to start a local instance useful for development. It is configured to use the D4SCience dev IAM. + +__dev__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the DEV infrastructure. + +__pre__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the PRE infrastructure. + +__prod__ branch contains the configuration to run on D4Science Docker swarm cluster acting on the PROD infrastructure. + +In order to run a local site the following commands need to be executed: + +``` +git clone https://code-repo.d4science.org/gCubeSystem/d4s-cdn-setup.git +ansible-playbook site.yaml --ask-vault-pass +``` +This will create a basic stack with an NGINX based router, a PEP and base CDN service. + +If you want to deploy on a specific D4S infrastructure please clone the corresponding branch. + +``` +git clone --b {infra} https://code-repo.d4science.org/gCubeSystem/d4s-cdn-setup.git +ansible-playbook site.yaml --ask-vault-pass +``` + ### Contribution guidelines ### -* Writing tests -* Code review -* Other guidelines +* Adding a new CDN Island + +* Commit guidelines + +DO NOT commit the following files: + +conf/pep/config.js +conf/service/auth.js + +They will be generated by the ansible based deployment procedure in order to inject the secrets. ### Who do I talk to? ### * Repo owner or admin -* Other community or team contact \ No newline at end of file + +Marco Lettere +marco.lettere@nubisware.com + diff --git a/build-docker-images-sh b/build-docker-images-sh index bd7949d..7c7dded 100755 --- a/build-docker-images-sh +++ b/build-docker-images-sh @@ -1 +1 @@ -docker build --rm --no-cache -t nubisware/d4s-cdn -f images/Dockerfile-cdn . +docker build --rm --no-cache -t nubisware/d4s-cdn -f images/Dockerfile . diff --git a/cdn-base-swarm.yaml b/cdn-base-swarm.yaml deleted file mode 100644 index 0e0370c..0000000 --- a/cdn-base-swarm.yaml +++ /dev/null @@ -1,62 +0,0 @@ -version: '3.6' - -services: - - pep: - image: nginx:stable-alpine - networks: - - cdn-network - ports: - - "80:80" - deploy: - mode: replicated - replicas: 2 - restart_policy: - condition: on-failure - delay: 10s - window: 120s - configs: - - source: nginxconf - target: /etc/nginx/templates/default.conf.template - - source: nginxbaseconf - target: /etc/nginx/nginx.conf - - source: pep - target: /etc/nginx/pep.js - - source: pepconfig - target: /etc/nginx/config.js - - cdn: - image: nubisware/d4s-cdn - networks: - - cdn-network - deploy: - mode: replicated - replicas: 2 - restart_policy: - condition: on-failure - delay: 10s - window: 200s - configs: - - source: authconf - target: /opt/app/cdn/conf/auth.json - - source: cdnconf - target: /opt/app/cdn/conf/d4s-cdn.json - -networks: - cdn-network: - -configs: - nginxconf: - file: ./conf/pep/default.conf - nginxbaseconf: - file: ./conf/pep/nginx.conf - pep: - file: ./conf/pep/pep.js - pepconfig: - file: ./conf/pep/config.js - - cdnconf: - file: ./conf/cdn/d4s-cdn.json - authconf: - file: ./conf/cdn/auth.json - diff --git a/conf/cdn/auth.json b/conf/cdn/auth.json deleted file mode 100644 index a05a0b8..0000000 --- a/conf/cdn/auth.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - - "principal-workflow" : "session-principal", - "login-workflow" : "keycloak", - - "principal-key" : "_PRINCIPAL_", - - "keycloakurl" : "https://accounts.dev.d4science.org", - "keycloakrealm" : "d4science", - - "clientid" : "d4science-example-wp", - "client_redirect_uri" : "http://d4science-cdn-public/d4s-cdn/auth/oidc-callback", - - "user-manager-client-id" : "orchestrator", - "user-manager-client-secret" : "######", - - "client-manager-client-id" : "orchestrator", - "client-manager-client-secret" : "######", - - "group-manager-client-id" : "orchestrator", - "group-manager-client-secret" : "######" - -} - diff --git a/conf/cdn/d4s-cdn.json b/conf/cdn/d4s-cdn.json deleted file mode 100644 index 6d824d4..0000000 --- a/conf/cdn/d4s-cdn.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "d4science_realm" : "d4science", - "d4science_iam_base" : "https://accounts.dev.d4science.org/auth", - "infrastructure" : { - "id" : "%2Fgcube", - "name" : "/gcube", - "type" : "root-vo", - "children" : [ - { - "id" : "%2Fgcube%2Fdevsec", - "name" : "/gcube/devsec", - "type" : "vo", - "gateways" : ["d4science-example-wp"], - "about" : "
devsec VO

This VO of the D4Science development infrastructure.

In this VO ...

", - "apps" : [ - { - "id" : "about", - "name" : "About", - "description" : "Welcome message" - }, - { - "id" : "vo-manage", - "name" : "Manage", - "type" : "category", - "apps" : [ - { - "id" : "user-manager", - "name" : "Users", - "description" : "Manage users of VO and roles" - }, - { - "id" : "ext-app-manager", - "name" : "External applications", - "description" : "Manage external applications and roles" - } - ], - "description" : "Manage Virtual Organization" - } - ], - "children" : [ - { - "id" : "%2Fgcube%2Fdevsec%2FdevVRE", - "name" : "/gcube/devsec/devVRE", - "type" : "vre", - "about" : "
/gcube/devsec/devVRE

on gcube dev infrastructure used for ...

", - "apps" : [ - { - "id" : "about", - "name" : "About", - "description" : "Welcome message" - }, - { - "id" : "vre-manage", - "type" : "category", - "name" : "Manage", - "description" : "Manage Virtual Research Environment", - "apps" : [ - { - "id" : "user-manager", - "name" : "Users", - "description" : "Manage users of VRE and roles" - }, - { - "id" : "ext-app-manager", - "name" : "External applications", - "description" : "Manage external applications and roles" - } - ] - }, - { - "id" : "analytics", - "type" : "category", - "name" : "Analytics 2", - "description" : "The new analytics framework", - "icon" : "images/batman.png", - "apps" : [ - { - "id" : "ccp_workflow-manager", - "name" : "Workflows", - "description" : "Manage workflows of analytics engine 2" - } - ] - } - ] - }, - { - "id" : "%2Fgcube%2Fdevsec%2FSAITEST", - "name" : "/gcube/devsec/SAITEST", - "type" : "vre", - "about" : "
/gcube/devsec/SAITEST

/gcube/devsec/SAITEST on gcube dev infrastructure used for ...

", - "apps" : [ - { - "id" : "about", - "name" : "About", - "description" : "Welcome message" - } - ] - }, - { - "id" : "%2Fgcube%2Fdevsec%2FanotherVRE4", - "name" : "/gcube/devsec/anotherVRE4", - "type" : "vre" - } - ] - }, - { - "id" : "%2Fgcube%2FdevNext", - "name" : "/gcube/devNext", - "children" : [ - { - "id" : "%2Fgcube%2FdevNext%2FNextNext", - "name" : "/gcube/devNext/NextNext", - "type" : "vre" - } - ] - } - ] - } -} diff --git a/conf/pep/default.conf b/conf/pep/default.conf index 6c4fe8a..4aa6dd1 100644 --- a/conf/pep/default.conf +++ b/conf/pep/default.conf @@ -1,6 +1,6 @@ -upstream d4science-cdn { +upstream d4s-cdn { ip_hash; - server cdn:8984; + server d4s-cdn:8984; } js_var $auth_token; @@ -16,28 +16,20 @@ server { listen *:80; listen [::]:80; - server_name d4science-cdn; + server_name d4s-cdn-pep; subrequest_output_buffer_size 200k; - location /d4s-cdn/config/ { - proxy_pass http://d4science-cdn; + location /config/d4s-cdn/1 { + proxy_pass http://d4s-cdn; } location / { - if ($request_method = OPTIONS) { - add_header Access-Control-Allow-Origin '*'; - add_header Access-Control-Allow-Methods 'GET, POST, PUT, DELETE, OPTIONS'; - add_header Content-Type text/plain; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, Accept, Origin"; - add_header Content-Length 0; - return 204; - } js_content pep.enforce; } location /resources/ { - proxy_pass http://d4science-cdn; + proxy_pass http://d4s-cdn; } location @backend { @@ -55,7 +47,7 @@ server { proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Original-URI $request_uri; - proxy_pass http://d4science-cdn; + proxy_pass http://d4s-cdn; } location /jwt_verify_request { diff --git a/conf/pep/pep.js b/conf/pep/pep.js index f698b4c..184177d 100644 --- a/conf/pep/pep.js +++ b/conf/pep/pep.js @@ -128,6 +128,7 @@ function parseAuthentication(context){ function verifyToken(context){ log(context, "Inside verifyToken") + log(context, "Token is " + context.authn.token) var options = { "body" : "token=" + context.authn.token + "&token_type_hint=access_token" } diff --git a/images/Dockerfile-cdn b/images/Dockerfile-cdn deleted file mode 100644 index b7c20cc..0000000 --- a/images/Dockerfile-cdn +++ /dev/null @@ -1,11 +0,0 @@ -### 1. Get muscle classic -FROM nubisware/muscle-classic:latest - -# 2. Install applicative fibers -ADD .muscle/ /root/.muscle/ -RUN git fiber install -p G3 -n d4s-cdn -RUN git fiber install -p G3 -n d4s-navigation -n d4s-vre-manager -RUN git service create -n cdn -w auth -w utils -w config -w error -w utils -w inspect -w resources -w d4s-cdn -w d4s-navigation -w d4s-vre-manager -WORKDIR cdn -EXPOSE 8984 -ENTRYPOINT ./start.sh