remove certificates & user and https configurations. Fix keycloak import realm names and urls

deployment/.env

@@ -11,7 +11,7 @@ DOCX_APP_TAG=0.0.8
 JSON_APP_TAG=0.0.6
 ZENODO_APP_TAG=2.0.4
 POSTGRES_TAG=16-alpine
-ELK_VERSION=8.13.0
+ELK_TAG=8.13.0
 KEYCLOAK_TAG=24.0.2
 RABBITMQ_TAG=3.13-management
 GOTENBERG_TAG=8.4.0

deployment/docker-compose.override.yml

@@ -1,7 +1,6 @@
 services:
 ############################## PROXY ########################################
   opendmp.proxy:
-    user: ${DEPLOY_USER}:${DEPLOY_GROUP}
     restart: unless-stopped
     cpus: 1
     mem_limit: 256m
@@ -11,18 +10,14 @@ services:
     env_file:
       - ./proxy/proxy.env
     volumes:
-      # - ./proxy/template-variables:/etc/nginx/templates/10-variables.conf.template:ro
       - ./proxy/nginx.conf:/etc/nginx/nginx.conf
       - ./proxy/ProxyNginx.conf:/etc/nginx/conf.d/default.conf
-      - ./proxy/nginx-selfsigned.crt:/certifcates/cert.crt
-      - ./proxy/nginx-selfsigned.key:/certifcates/key.key
       - ./logs/proxy:/tmp/logs
     networks:
       - opendmp-proxy-network
 
 ############################## OPENDMP APP #################################
 #   opendmp.backend:
-#     user: ${DEPLOY_USER}:${DEPLOY_GROUP}
 #     restart: unless-stopped
 #     cpus: 1
 #     mem_limit: 2048m
@@ -62,7 +57,6 @@ services:
 #       - opendmp-proxy-network
 
 #   opendmp.notification:
-#     user: ${DEPLOY_USER}:${DEPLOY_GROUP}
 #     cpus: 1
 #     mem_limit: 1024m
 #     restart: unless-stopped
@@ -84,7 +78,6 @@ services:
 #       - opendmp-postgres-shared-network
 
 #   opendmp.annotation:
-#     user: ${DEPLOY_USER}:${DEPLOY_GROUP}
 #     cpus: 1
 #     mem_limit: 1024m
 #     restart: unless-stopped
@@ -107,7 +100,6 @@ services:
 # ############################## FILE-TRANSFORMER #################################
 
 #   opendmp.file.transformer.docx:
-#     user: ${DEPLOY_USER}:${DEPLOY_GROUP}
 #     restart: unless-stopped
 #     cpus: 1
 #     mem_limit: 1024m
@@ -126,7 +118,6 @@ services:
 #       - opendmp-gotenberg-shared-network
 
 #   opendmp.file.transformer.rdajson:
-#     user: ${DEPLOY_USER}:${DEPLOY_GROUP}
 #     restart: unless-stopped
 #     cpus: 1
 #     mem_limit: 1024m
@@ -146,7 +137,6 @@ services:
 
 ############################## ZENODO #######################################
   # opendmp.zenodo:
-  #   user: ${DEPLOY_USER}:${DEPLOY_GROUP}
   #   restart: unless-stopped
   #   cpus: 1
   #   mem_limit: 1024m
@@ -165,7 +155,6 @@ services:
 ############################## POSTGRES 16 #################################
 
   opendmp.postgres:
-      user: ${DEPLOY_USER}:${DEPLOY_GROUP}
       restart: unless-stopped
       mem_limit: 2048M
       ports:
@@ -175,6 +164,8 @@ services:
         - ./postgres/postgres.env
       volumes:
         - ./storage/postgres/data:/var/lib/postgresql/data
+        - ./postgres/opendmp_init.sql:/docker-entrypoint-initdb.d/opendmp_init.sql
+        - ./postgres/user_init.sql:/docker-entrypoint-initdb.d/user_init.sql
       networks:
         - opendmp-postgres-shared-network
       healthcheck:
@@ -185,24 +176,49 @@ services:
 
 
 ################################# ELK #################################################
+  # elk.setup:
+  #   profiles:
+  #     - setup
+  #   build:
+  #     context: ./elk/setup/
+  #     args:
+  #       ELASTIC_VERSION: ${ELASTIC_VERSION}
+  #   init: true
+  #   env_file:
+  #     - elk/elk.env
+  #   volumes:
+  #     - ./setup/entrypoint.sh:/entrypoint.sh:ro,Z
+  #     - ./setup/lib.sh:/lib.sh:ro,Z
+  #     - ./setup/roles:/roles:ro,Z
+  #   environment:
+  #     ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+  #     LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
+  #     KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+  #     METRICBEAT_INTERNAL_PASSWORD: ${METRICBEAT_INTERNAL_PASSWORD:-}
+  #     FILEBEAT_INTERNAL_PASSWORD: ${FILEBEAT_INTERNAL_PASSWORD:-}
+  #     HEARTBEAT_INTERNAL_PASSWORD: ${HEARTBEAT_INTERNAL_PASSWORD:-}
+  #     MONITORING_INTERNAL_PASSWORD: ${MONITORING_INTERNAL_PASSWORD:-}
+  #     BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
+  #   networks:
+  #     - opendmp-elastic-network
+
   # opendmp.elasticsearch:
-  #   user: ${DEPLOY_USER}:${DEPLOY_GROUP}
-  #   group_add:
-  #     - 0
   #   restart: unless-stopped
   #   cpus: 2
   #   mem_limit: 1024m
+  #   init: true
   #   env_file:
-  #     - elk/config-elk/elasticsearch/elastic.env
+  #     - elk/elk.env
   #   environment:
-  #     - ES_JAVA_OPTS=-Xmx512m -Xms512m
+  #     ES_JAVA_OPTS: -Xmx512m -Xms512m
+  #     node.name: elasticsearch
+  #     ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
   #   ulimits:
   #     nproc: 65535
   #     memlock:
   #       soft: -1
   #       hard: -1
   #   volumes:
-  #     - ./elk/config-elk/elasticsearch/certificates:/usr/share/elasticsearch/config/certificates
   #     - ./elk/config-elk/elasticsearch/config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
   #     - ./elk/config-elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
   #     - ./elk/data-elk/elasticsearch-data:/usr/share/elasticsearch/data
@@ -219,39 +235,82 @@ services:
   #     timeout: 10s
   #     retries: 5
 
-#   # opendmp.logstash:
-#   #   volumes:
-#   #     - /elk/data-elk/logstash-log:/usr/share/logstash/logs
-
   # opendmp.kibana:
-  #   user: ${DEPLOY_USER}:${DEPLOY_GROUP}
   #   restart: unless-stopped
   #   cpus: 2
   #   mem_limit: 1024m
+  #   ulimits:
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
   #   environment:
   #     - xpack.license.self_generated.type=basic
   #     - xpack.security.enabled=true
   #   volumes:
-  #     - ./elk/config-elk/kibana/certificates:/usr/share/kibana/certificates
-  #     - ./elk/config-elk/kibana/certificates/ca:/usr/share/kibana/certificate_authorities
   #     - ./elk/config-elk/kibana/config:/usr/share/kibana/config:ro
   #   expose:
   #     - "5601"
   #   networks:
   #     - opendmp-elastic-network
 
-#   # opendmp.filebeat:
+  # logstash:
+  #   build:
+  #     context: logstash/
+  #     args:
+  #       ELASTIC_VERSION: ${ELASTIC_VERSION}
+  #   volumes:
+  #     - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z
+  #     - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
+  #   ports:
+  #     - 5044:5044
+  #     - 50000:50000/tcp
+  #     - 50000:50000/udp
+  #     - 9600:9600
+  #   environment:
+  #     LS_JAVA_OPTS: -Xms256m -Xmx256m
+  #     LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
+  #   networks:
+  #     - elk
+  #   depends_on:
+  #     - elasticsearch
+  #   restart: unless-stopped
+  # #     - /elk/data-elk/logstash-log:/usr/share/logstash/logs
+
+  # opendmp.filebeat:
+  #   restart: unless-stopped
+  #   cpus: 1
+  #   mem_limit: 1024m
+  #   ulimits:
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
+  #   volumes:
+  #     - /var/run/docker.sock:/host_docker/docker.sock
+  #     - /var/lib/docker:/host_docker/var/lib/docker
+  #     - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
+  #   command: ["--strict.perms=false"]
+  #   ulimits:
+  #     memlock:
+  #       soft: -1
+  #       hard: -1
+  #   stdin_open: true
+  #   tty: true
+  #   network_mode: bridge
+  #   deploy:
+  #     mode: global
+  #   logging:
+  #     driver: "json-file"
+  #     options:
+  #       max-size: "10m"
+  #       max-file: "50"
 
 
 ############################## KEYCLOAK ###############################################
   opendmp.keycloak:
     restart: unless-stopped
-    command: ["start", "--log=console,file", "--log-file=/tmp/logs/keycloak.log", "--import-realm"]
+    command: ["start-dev", "--log=console,file", "--log-file=/tmp/logs/keycloak.log", "--import-realm"]
-    # command: ["start", "--log=console,file", "--log-file=/tmp/logs/keycloak.log"]
     cpus: 1
     mem_limit: 1024M
-    security_opt:
-       - seccomp:unconfined
     env_file:
       - keycloak/keycloak.env
     environment:
@@ -259,16 +318,14 @@ services:
     volumes:
       - ./logs/keycloak:/tmp/logs
       - ./keycloak/imports/opendmp-realm.json:/opt/keycloak/data/import/opendmp-realm.json
-      - ./keycloak/certs/keycloak-selfsigned.crt:/tmp/keycloak-selfsigned.crt:ro
-      - ./keycloak/certs/keycloak-selfsigned.key:/tmp/keycloak-selfsigned.key:ro
     expose:
-      - "8443"
+      - "8080"
     networks:
       - opendmp-proxy-network
       - opendmp-postgres-shared-network
       - opendmp-keycloak-shared-network
 
-# ############################## RABBITMQ ###############################################
+############################## RABBITMQ ###############################################
   opendmp.rabbitmq:
     labels:
       NAME: "rabbitmq"
@@ -286,7 +343,7 @@ services:
       - opendmp-proxy-network
       - opendmp-rabbitmq-shared-network
 
-# ############################## GOTENBERG ##############################################
+############################## GOTENBERG ##############################################
   opendmp.gotenberg:
     mem_limit: 2048m
     restart: unless-stopped

deployment/docker-compose.yml

@@ -60,53 +60,35 @@ services:
   opendmp.postgres:
     container_name: opendmp.postgres
     image: postgres:${POSTGRES_TAG}
-    build:
-      context: ./postgres/
-      args:
-        POSTGRES_TAG: $POSTGRES_TAG
 
 ################################# ELK #################################################
-  # opendmp.elasticsearch:
+  # elk.setup:
-  #   container_name: opendmp.elasticsearch
+  #   container_name: elk.setup
-  #   image: elasticsearch
-  #   build:
-  #     context: ./elk/elasticsearch/
-  #     args:
-  #       ELK_VERSION: $ELK_VERSION
-  #       DEPLOY_USER : $DEPLOY_USER
-  #       DEPLOY_GROUP : $DEPLOY_GROUP
-
-#   # opendmp.logstash:
-#   #    container_name: opendmp.logstash
-#   #    image: logstash
-#   #    build:
-#   #     context: /elk/logstash/
-#   #     args:
-#   #       ELK_VERSION: $ELK_VERSION
-#   #    depends_on:
-#   #      - opendmp.elasticsearch
-
-  # opendmp.kibana:
-  #   container_name: opendmp.kibana
-  #   image: kibana
-  #   build:
-  #     context: ./elk/kibana/
-  #     args:
-  #       ELK_VERSION: $ELK_VERSION
-  #       DEPLOY_USER : $DEPLOY_USER
-  #       DEPLOY_GROUP : $DEPLOY_GROUP
   #   depends_on:
   #     - opendmp.elasticsearch
 
-#   # opendmp.filebeat:
+  # opendmp.elasticsearch:
-#   #   container_name: opendmp.filebeat
+  #   container_name: opendmp.elasticsearch
-#   #   image: filebeat
+  #   image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_TAG}
-#   #   build:
+
-#   #     context: /elk/filebeat/
+  # opendmp.kibana:
-#   #     args:
+  #   container_name: opendmp.kibana
-#   #       ELK_VERSION: $ELK_VERSION
+  #   image: docker.elastic.co/kibana/kibana:${ELK_TAG}
-#   #   depends_on:
+  #   depends_on:
-#   #      - opendmp.logstash
+  #     - opendmp.elasticsearch
+
+  # opendmp.logstash:
+  #    container_name: opendmp.logstash
+  #    image: docker.elastic.co/beats/filebeat:${ELK_TAG}
+  #    depends_on:
+  #     - opendmp.elasticsearch
+
+  # opendmp.filebeat:
+  #   container_name: opendmp.filebeat
+  #   image: docker.elastic.co/logstash/logstash:${ELK_TAG}
+  #   depends_on:
+  #     - opendmp.elasticsearch
+  #     - opendmp.logstash
 
 ############################## KEYCLOAK ###############################################
   opendmp.keycloak:
@@ -116,12 +98,12 @@ services:
       opendmp.postgres:
         condition: service_healthy
 
-# ############################## RABBITMQ ###############################################
+############################## RABBITMQ ###############################################
   opendmp.rabbitmq:
     container_name: opendmp.rabbitmq
     image: rabbitmq:${RABBITMQ_TAG}
 
-# ############################## GOTENBERG ##############################################
+############################## GOTENBERG ##############################################
   opendmp.gotenberg:
     image: gotenberg/gotenberg:${GOTENBERG_TAG}
     container_name: opendmp.gotenberg

deployment/elk/config-elk/elasticsearch/certificates/ca/ca-key.pem

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC3Ijv7bT86kta/
-1wx0nMbtZvXF5Hfmt1n3087NcOi/JdjnSmF7JfTCXjzKQtOKrv2tLGkaXfrzerL+
-yPBKuffTMk80vHSixmrH71IX0DriKNxnW6RNg1j3R3igVVsGTNdUbI26dc3RZpKR
-gb7u/HqR1GTLD6EfrcL655aitCmywvyzLQ/x1BWZ3WjaMONT1DdU7I1GXhqv96be
-Sbi2dQUdogNFGhiK9WwFrKJpeSERlOl6jPBqoYRZjBlgJ/DlmWhvlKwj91ilYeOd
-ifsB9e7F9lwUbADsSGTOKKqBIX29ZcFkXwJshtm6CIQXWnvz+jl46mu5ScEU+iwr
-mvILVUIJAgMBAAECggEAEgO3WEp9FYczwj/GfSMd62T9KLgKdmLTYg5PEcT5VDJG
-JaxarflEHCmWe8P6mLIRiKstdJdJlBFeEbOU9ZjZEMiqY3LlW0y+3MeFMQv9+tjP
-o4gvf6N7ySlZ65Wx5EsDRI4AHBcyBZb8NH2JmWszKGy29IWnUR0v6KwG1J752hhq
-vTO9aMaz3MTstKTal0cDJRaTjPctzXVSyJSTeClNpl8mFDYbCUR/PPklZbAx9CyY
-K6orDCUBGOH2wK85+l9uFaUWOcupKBhg99MKZTpX/6tIgqbCuBfN8FBk0LztJ/Uo
-SZAHf5QIt6eTmcBtarlbsTV0TeJj5llVUGynHTBvQQKBgQC+ZTbTkbfHIgbVqDeU
-YkBiKul4M8xzIOsogNtZVevL7R2KSco1TUmcY7SDq8flshtZJb6utXUXCUprNsZl
-OOM7QpXxfnYKTjv39NTM4eCCzvMcDpBRmBQmQkka+2NbAxMTy91T709EAiqgia+m
-tszU93IGIle9abv9Fo1giw/lqQKBgQD2PHhwtmVT3B/H/ywtadCmyfHm+kHi8IWR
-y//EvLjDgI+SzwIgM2ABLAkKqg1VXkgZ741AxaQkkcP+NgJ2saY0cJCKBr/SPyRe
-jTfbWWfH89Mf3EVl2fxkG3YL1EJu+boup3l9L1rGpK9japAIMNOXh8S4A5WCOZLr
-Hk6FuTF1YQKBgEr8K9qpcjrQMObm6HTdOUQwaGD57ZSOK295SGpnx4U6Lr8vDp9t
-gAdC0W5mMkVJnzG+BtpiBup6sz+EhCCLhhrpv4or5ytp4n5mg4TplPWPsfmj1rz7
-6zuiMY6Z4WiPzmymhtWu04YSYF13vKEpL4TUq6i0z99+jBZCUo3qVul5AoGAcYNG
-8o7i/1nGvOgBcZ4KNhl6jsRngzrmGGQ2sHdfpaCqjz8m97k3VNL8CBKEuwoPqwUn
-1OhH1yPrelFjqVwUBrCtsTOTUlURaxUm3tPEaAUbGuDsjRuEopGWRbXAOnCdR8yk
-0PT3oANjZy1E4MHBiWVpZnsgfTwVYpZCFJtfFYECgYBkyF06DC0DhZZ0AEZpJHxf
-xbP/1gq7KlBzR6WSSRzPxX/3VOdBuGs7qYP1orDEF9wG/0Jk35Ek+PcT97j6s0gE
-a4Zd8iYpSdgd36L+5uBxgRsavr/Xf4lQECRTQYfKUVhKhhCT1xjOUAAr52Vl+8V/
-5sIcUBUzbXDpZvyR/67pxQ==
------END PRIVATE KEY-----

deployment/elk/config-elk/elasticsearch/certificates/ca/ca.crt

@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDcTCCAlmgAwIBAgIUE5s/YTgomqob7mc88HmQUn/sHkswDQYJKoZIhvcNAQEL
-BQAwRzELMAkGA1UEBhMCR1IxDTALBgNVBAgMBENJVEUxDDAKBgNVBAcMA0FUSDEN
-MAsGA1UECgwEQ0lURTEMMAoGA1UEAwwDZWxrMCAXDTIyMDkwODA3Mzg0NFoYDzIx
-MjIwODE1MDczODQ0WjBHMQswCQYDVQQGEwJHUjENMAsGA1UECAwEQ0lURTEMMAoG
-A1UEBwwDQVRIMQ0wCwYDVQQKDARDSVRFMQwwCgYDVQQDDANlbGswggEiMA0GCSqG
-SIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3Ijv7bT86kta/1wx0nMbtZvXF5Hfmt1n3
-087NcOi/JdjnSmF7JfTCXjzKQtOKrv2tLGkaXfrzerL+yPBKuffTMk80vHSixmrH
-71IX0DriKNxnW6RNg1j3R3igVVsGTNdUbI26dc3RZpKRgb7u/HqR1GTLD6EfrcL6
-55aitCmywvyzLQ/x1BWZ3WjaMONT1DdU7I1GXhqv96beSbi2dQUdogNFGhiK9WwF
-rKJpeSERlOl6jPBqoYRZjBlgJ/DlmWhvlKwj91ilYeOdifsB9e7F9lwUbADsSGTO
-KKqBIX29ZcFkXwJshtm6CIQXWnvz+jl46mu5ScEU+iwrmvILVUIJAgMBAAGjUzBR
-MB0GA1UdDgQWBBQSAI1g3+gAsT5BHVfaWPlNFy9IgjAfBgNVHSMEGDAWgBQSAI1g
-3+gAsT5BHVfaWPlNFy9IgjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA
-A4IBAQCmR4cz47QtGX1xn2Rrl1NdLX2wiS2y7P4xRGzKeAYZIHLZWW/GaJDb+yw9
-Cz9qjhuBhGqfIeh8QryRgPotd64Oef0MscC+oFfprWxQA0svP83sITr9BazGb4A4
-LcIToVHZtIMnak119k1RsNYpzADDBxnaaODs3xCe21dfCVI/ea+wSPiUY3vvZZDn
-KejJclhRnQFV3yQ7hMdR9tq0BndWtqHrappa3oX2JU1yi/x3Ndi6dOMk+x7+kc4Q
-OAtzcXa29kowAyLUMHhGYwcsJp8ysa6Xlltqt/kkI+3CgbTl/egUU9igysMKDyMM
-0LQcef+IQwmeHfD1RAW2ksW2OOx5
------END CERTIFICATE-----

deployment/elk/config-elk/elasticsearch/certificates/elasticsearch/elasticsearch.crt

@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDXDCCAkSgAwIBAgIUK8FEbNAIdyPoRF/pTyqNuL3kP54wDQYJKoZIhvcNAQEL
-BQAwRzELMAkGA1UEBhMCR1IxDTALBgNVBAgMBENJVEUxDDAKBgNVBAcMA0FUSDEN
-MAsGA1UECgwEQ0lURTEMMAoGA1UEAwwDZWxrMCAXDTIyMDkwODA4MjgxM1oYDzIx
-MjIwODE1MDgyODEzWjAYMRYwFAYDVQQDDA1lbGFzdGljc2VhcmNoMIIBIjANBgkq
-hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoC6DoPC1kesTH0cKs1blVW8ddsQ3VmFO
-ROJiUorhDIHB3sXJhCSw0hxZFNZtqgG35CTa5w3XiQMT1fr6Ar/ztOQmARg9CMQa
-mOf8gR+tMTSwP7zr7WBR/1Q+GseeOnthFOfvfq7LLpRs8VNb/mhcSIjJsT9kMNXN
-5iHIyEuGhQSwPZDUYx+6Ag4belQLvic+QYDhwvujtPFWj8qLSG7kTpbBK5ahH/5E
-mvT5mpOYTR10f3LG4DKw7t6qG9tzh4WkwR0JYmlgxpAA/HBSa/QjS7CGxLrA4Sop
-gQF8KQPEP/0w98EbYpBUPS5jqbkBr6093M7Epksi86oRjtbcXAT0qwIDAQABo20w
-azApBgNVHREEIjAggg1lbGFzdGljc2VhcmNogglsb2NhbGhvc3SHBH8AAAEwHQYD
-VR0OBBYEFC/cKMOAVbx8bwyoKdg2Oiej9xoSMB8GA1UdIwQYMBaAFBIAjWDf6ACx
-PkEdV9pY+U0XL0iCMA0GCSqGSIb3DQEBCwUAA4IBAQBo42FOuxIMeIiMaKa347gc
-WsHpkazYOA6iHK5xXPsVUU1xSCLKp5HLCC04FU5P9njCDyZo1e/SR6rirQJJHEtT
-SAn7iabREE+vy0oN3JnyV+eJPmKWxlqeFr9Cs9uIXQbgjwyyj9rxT06eLr3M1MA1
-IsARV2eyxcgS5sCC8JBCEpKR4jLRrpAs0tGJOeIh1cmf/1id+NQaDa14sLFKHBH1
-3+6TfBPrhJoGqFz92jV2airr7dppyCXgmWymVc66iD00Nak6Bvchg6ARTkqJnfoZ
-2/Tz7asHV2V052ZLiow7Si34nS/9Hp8F8vUaj+FYXowvGwQUXLQIg/53KXh7piuW
------END CERTIFICATE-----

deployment/elk/config-elk/elasticsearch/certificates/elasticsearch/elasticsearch.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCgLoOg8LWR6xMf
-RwqzVuVVbx12xDdWYU5E4mJSiuEMgcHexcmEJLDSHFkU1m2qAbfkJNrnDdeJAxPV
-+voCv/O05CYBGD0IxBqY5/yBH60xNLA/vOvtYFH/VD4ax546e2EU5+9+rssulGzx
-U1v+aFxIiMmxP2Qw1c3mIcjIS4aFBLA9kNRjH7oCDht6VAu+Jz5BgOHC+6O08VaP
-yotIbuROlsErlqEf/kSa9Pmak5hNHXR/csbgMrDu3qob23OHhaTBHQliaWDGkAD8
-cFJr9CNLsIbEusDhKimBAXwpA8Q//TD3wRtikFQ9LmOpuQGvrT3czsSmSyLzqhGO
-1txcBPSrAgMBAAECggEAJdOmMPj3H9zgGKhJrh1Mrha94gCnQsZa2eiOKIj0aWQx
-GL8jfgm+Gfgoz0NuBVI/j2hmq1648fmgkw0gQkr7LdIc6XBEZZAN6eMK3aFR4Idd
-QcgG/PkclAvcWK2gP5ZIUEwPYh68C6VwbrLtTBBwDo8C9lEOg3vSElETHb28KCgZ
-sC0wLres01crLF0Azh+m3cx+p/6TBLfpBIrM0HQn1Lmn/dP6BKcRDoncE+GcKjE1
-JZcgypdC4Juq2WctMNbBvgt+7AjVB1n9ejrUN5rlK+JP3Xa7D4zvc79CDX+BP2C2
-X57ZT4Pff5mPF70zrlqGQNnBep09UxZTRnHDRMWzZQKBgQDfMT+G+x49TZYuaiQC
-gKguQ0k8F6nnrmz0rz2MNJiZ9oTYAtz5wRQ25KkbqTc7beKecSykp5izoluzgSJu
-dTFh11SO1i63kMzpFFyBui3rSoUjAq+sMzRY5ERyUsG90tsaAl2a9PT3M9b5a0XC
-8f1cDhKt+JQtaYRiZZJsC4Ru1wKBgQC3uirv92/dq1RcuWBf/yt2n6/JY9+9k1NI
-vDzQtVI0Q3OZfRX9Rn9/+h3fSXTG3w7p5FqfNguHYPbLNzO+6WxxeuDveAL3Nx4/
-HSURjbiK+ppYDwyeY4IgKgeq2mRrIZC4rSqEsrJMLnNiDRYaVTWZczqGLT5oZ5cT
-lBLDD6+STQKBgDhi68bBOIGKUW/GdvR+5n5Rl3XsEIusoHAsuaLrQsZa5nLgPk2G
-vwGjQSnw1ThZaZBXzUyH3uc7FGnELRu01dX/Hai8aa8MkQgtkbVggOtZt0sCCbm6
-cfYnLTeourOnSp1GjblxO1YcranztPssQbL5BzUWgPD8IGrveE99lWafAoGAG6q4
-PoynVt0vBguQXMRjOijP4ubcUYL2/rQCAHfdmisyJEH25r4QAyiaCP7Zy/zZFRWj
-I+iSkd9jKrT0YOJrxyb26njLEYlGT8DGzT7nNF6KkYoqn0ti1A8gOnVKu+tBDN5e
-0b7LJLe1/mT0GCEOwj3c6Um05Sn8USFyNdeN290CgYBSdmwqJYUGJXVGTCn2Ff4Z
-jdFtN/Q9kFDhCCYVV1XAJ5mdX4k77HIw5EAlDXM0EZnhQAec+RSKIO7Oc+9krmFq
-R1lCT/s7UDsitQBDmkQs+12PEILuk+Qbdan+CwTLwCik06vj+VzZhHylFoOMJLdm
-lf4Bnd2TNNykAsd2jy5cAg==
------END PRIVATE KEY-----

deployment/elk/config-elk/elasticsearch/certificates/elasticsearch/v3.ext

@@ -1,12 +0,0 @@
-[ req ]
-default_bits       = 2048
-distinguished_name = req_distinguished_name
-req_extensions     = req_ext
-[ req_distinguished_name ]
-commonName                 = elasticsearch
-[ req_ext ]
-subjectAltName = @alt_names
-[alt_names]
-DNS.1 = elasticsearch
-DNS.2 = localhost
-IP.1 = 127.0.0.1

deployment/elk/config-elk/elasticsearch/config/elasticsearch.yml

@@ -1,19 +0,0 @@
----
-## Default Elasticsearch configuration from elasticsearch-docker.
-## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml
-#
-network.host: 0.0.0.0
-
-# minimum_master_nodes need to be explicitly set when bound on a public IP
-# set to 1 to allow single node clusters
-# Details: https://github.com/elastic/elasticsearch/pull/17288
-# discovery.zen.minimum_master_nodes: 1
-
-## Use single node discovery in order to disable production mode and avoid bootstrap checks
-## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
-#
-discovery.type: single-node
-## Search Guard
-#
-
-

deployment/elk/config-elk/elasticsearch/elastic.env

@@ -1,16 +0,0 @@
-cluster.name=opendmp-cluster
-bootstrap.memory_lock=true
-xpack.license.self_generated.type=basic
-xpack.monitoring.collection.enabled=true
-xpack.ml.enabled=false
-xpack.security.enabled=true
-xpack.security.http.ssl.enabled=true
-xpack.security.http.ssl.verification_mode=certificate
-xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certificates/elasticsearch/elasticsearch.key
-xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
-xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certificates/elasticsearch/elasticsearch.crt
-xpack.security.transport.ssl.enabled=true
-xpack.security.transport.ssl.verification_mode=certificate
-xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/ca/ca.crt
-xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/config/certificates/elasticsearch/elasticsearch.crt
-xpack.security.transport.ssl.key=/usr/share/elasticsearch/config/certificates/elasticsearch/elasticsearch.key

deployment/elk/config-elk/kibana/certificates/ca/ca.crt

@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDSTCCAjGgAwIBAgIUXMpiJCPQnPeOHA1FjYo12FaHO1UwDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMTkwOTAzMTUyMDM1WhcNMjIwOTAyMTUyMDM1WjA0MTIwMAYD
-VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAI1ci/DoagopzxemkP21UmnP
-wv2Yoo267y6CR/okrT3a0lARDjPl28YaNsEQ2skAnPu3gNqqDWW9j1aWEtWwNuEA
-PudVCdc6irgFEbPlwU6Dh05LVB99FCw70UKM5G4CSH7gMQvzPcvjJT4ROKoDCh3W
-I+pWYqhqU9xEiMzwsPdC2uy2Om2I0bZ2A03WmMr8Ts58qmBqVOMBLIY008jFetj7
-ZH67WDT92pqfG9/xRKH9ELdZNlNw/2fSTb4KBek06MZIzPkHk0iMhw7bMLwEYyDy
-J14Rym4Up9akgr8J6XwyACek5oht1lQlJjYhUuf2ZSzVJ54LhYoTGg1ybYT9qx8C
-AwEAAaNTMFEwHQYDVR0OBBYEFAxdsx3VcEsMaPWe7GvbyHOEnftTMB8GA1UdIwQY
-MBaAFAxdsx3VcEsMaPWe7GvbyHOEnftTMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
-hvcNAQELBQADggEBADltg11WpSg0tYVXrAowySy68CkcK9t/XYioeRYRAvfSD5mB
-ONMFegqwJVqUzu6HbxkhpVBf/JykGqSkf0Cu5BRUYT7A+egpDNAPAIa1/SbSchjP
-mbFMbpLRXFfP60xqgVem0C5wKcMEFFg+0YRDkSf/232aCwb0sS63V52ssmnEDN6v
-k4Cn2k/MZjAi/seWNnphaTyU71Eu3ObftIpOGc4ZJ875KiUZQtCXrP36QICUdFAM
-ay+z2gEVQQE2zKbtaEeE0Sxyas9eRnGHXzbx/yoz706lME9QmzPmcvfVlHQH8N/o
-2nU+I07j6TDoHn/WRIgbWR0jrWv0hlTqzxOyCDM=
------END CERTIFICATE-----

deployment/elk/config-elk/kibana/certificates/kibana.crt

@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDUjCCAjqgAwIBAgIUKTnOSL0Rtnm8ZQkfSUvpQiBNGnMwDQYJKoZIhvcNAQEL
-BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
-cmF0ZWQgQ0EwHhcNMTkwOTAzMTUyMDM1WhcNMjIwOTAyMTUyMDM1WjAYMRYwFAYD
-VQQDEw1lbGFzdGljc2VhcmNoMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
-AQEAkB1OMRBRUDUxQ6fIv5gv0myKDZgVJFnEEjIjU6YjMpf/htTsvu8zdpBoWhg6
-7IsflSkUPynDG4geFjQ/WtxVeqxjnmtIB2fMDAHppX882as3tYjBlHj1sU0/OwwI
-Ga5/OtxOubGswrzCEEjIgZwTtSX2Yzx3tE2UzwjWsYwGCBM/ssN8Wc1vlqq20+Qr
-Lsggk+dXapN2wL9FABrxrJfV2SxXb2qKLKVd3EIfs+HVqIt9dVrpcFRV3Lwexg+Z
-wlJv58EPsynphczssBhMOhlmVqpRY8z88fqsbqDVdqHIF8hqn7czWFqeCRldnb7W
-LWaYaOG0Jd6SM7OpHnfNgBST4wIDAQABo3gwdjAdBgNVHQ4EFgQUCkDAcWSJ6H2G
-UFFh9dhk+mG0L08wHwYDVR0jBBgwFoAUDF2zHdVwSwxo9Z7sa9vIc4Sd+1MwKQYD
-VR0RBCIwIIIJbG9jYWxob3N0hwR/AAABgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQC
-MAAwDQYJKoZIhvcNAQELBQADggEBAAQpkdkGl2H0ylgbmmNiIlrQX+U2i4ag4sJ6
-xsVR5OWxuyB/aMWhuOHkgueMh2wElufn60jK0Mh25b2U7oO/0Nq+28rhhP9HURLz
-7/TwCbLcglTAgHQPWItwn5r5WKDFNCPNpZXFU/oG5H6hUJqTvuaTN6G/PQ6V9Yp3
-J00NbPuFq8tjNAc/kQnhC7zdC/7YQ/fanHBPkvQnkGbac5+VAF/se/JYbxRpSz23
-5a+v6BDb/kjs82QgV8dzsyFmntO+Neesu9tTJurBbQD5T3xMgoGSWLgnTCq3/drl
-PMBLgUQHik629dU+7o8ePCdyULruGMR6CIBqO7ZKQASulhkxdUo=
------END CERTIFICATE-----

deployment/elk/config-elk/kibana/certificates/kibana.key

@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEAkB1OMRBRUDUxQ6fIv5gv0myKDZgVJFnEEjIjU6YjMpf/htTs
-vu8zdpBoWhg67IsflSkUPynDG4geFjQ/WtxVeqxjnmtIB2fMDAHppX882as3tYjB
-lHj1sU0/OwwIGa5/OtxOubGswrzCEEjIgZwTtSX2Yzx3tE2UzwjWsYwGCBM/ssN8
-Wc1vlqq20+QrLsggk+dXapN2wL9FABrxrJfV2SxXb2qKLKVd3EIfs+HVqIt9dVrp
-cFRV3Lwexg+ZwlJv58EPsynphczssBhMOhlmVqpRY8z88fqsbqDVdqHIF8hqn7cz
-WFqeCRldnb7WLWaYaOG0Jd6SM7OpHnfNgBST4wIDAQABAoIBACRGPBg9czotKWlO
-IkmXlPHyQA2L6kZsEd5CoIG9n75sY/UcQzsMGngNgTasQqinnBt/a4k6idG0QV51
-aD0GmL14BtspRcgXaFTdZmIx1K81WaBn+9HTYIRwXSoPrTpJody+91HmVwXtXSuc
-Jlv5XTyLgakY30iS/pHeN6wZAqulS7p6DkCH+W3c15BvNYnsjDX5vYZLgmktNl62
-LoHymTt54rLhxheZiwFeiHePsl9IanvnDEjCl2jBFnFB402NeapTex84ZnOgPYEV
-5w81MUYMeLy1aaziWEICPP1TU62T2oYKUFC1kQKUgCYp3t1UUkLsF6KfybJ+baMs
-+Nm5IQECgYEAzy4kQ9wo2x+3teQgMR+QLIN2cgfDCgI1BXUzhIr607kpo6F12Jf0
-zT5sC+9OzSPh3dPkyvnWtg2K3ld3L+ATqGvpWC42OgSI0HK6rnfL8Q3cYld8KLwn
-C8lbSQuDFo+hMJoGU7V+QTXM8j1e+2KoxgwGfceeiwql5GpqqcHFSJkCgYEAshLP
-QYOBaimhOlY1m1B9YgXrgNKoj66njhHJLTD9rD4BvMsntGo8UaBrZxpcQKaDO778
-UtuRPM8Pfrg2Q2fR6SHAydoQpiGQ0XrkUXNmh3v9YOBlFzdg1PhKSCRZRv75KjnC
-Z4jyL8GVYMhB+vWl/bQJO5o5YYHR7OdJCs8p4dsCgYEAnTQl0Ff9qEco3pt60cth
-WmVmOqApHi8OjFWiAbBzIrQdJEwfX3nuBs6WrXeoE4BmoitmL53jjcA43vz/MxST
-1fgL1x05iExog8KKZFgCJ6ac0wIO89nQxuDCo2pYzYWuiXJV2q6cXl60ZBqtN5Jk
-/eGiC06svlQWDHuy85xUVgECgYEAm8jk1FWtxCiDSjvdTfEOn9C6BMtGd9y97QYW
-T5jKdAMTFg1MwQMnnemPzHU9O7nwmTYuHHbGCsLCtYOGxVVwSFolBPHnGs/Gl9zB
-HZitm92W/0eQaM0jw20r3w0dpYSuiohZUKZ36dubST4oqtp4ywjUAvabOHiQJIb6
-WaB+7X0CgYEAptNxensUEn+hSKfMVCxS36U1QL2njRfGiCfqVHVOKeu6+oLB4N/N
-0mZngesMGV1HxzYivwkXW07U0drgfqv+iHBIF5HYRM6PkFNpop6PJAlVpFaATx0s
-tDvtrcmgz3hunhHURvr/VlXcGuYo4mpySPhHDTeF6Ad/9Ml16vO7uW0=
------END RSA PRIVATE KEY-----

deployment/elk/config-elk/kibana/config/kibana.yml

@@ -1,15 +0,0 @@
----
-## Default Kibana configuration from kibana-docker.
-## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml
-#
-server.name: opendmp.kibana
-server.host: "0.0.0.0"
-## Custom configuration
-elasticsearch.hosts: [ "https://opendmp.elasticsearch:9200" ]
-elasticsearch.ssl.certificateAuthorities: [ "/usr/share/kibana/certificate_authorities/ca.crt" ]
-
-elasticsearch.username: "kibana"
-elasticsearch.password: "2VzNck1n3uCed9d27wHn"
-server.ssl.enabled: false
-server.ssl.key: "/usr/share/kibana/certificates/kibana.key"
-server.ssl.certificate: "/usr/share/kibana/certificates/kibana.crt"

deployment/elk/elasticsearch/Dockerfile

@@ -1,23 +0,0 @@
-ARG ELK_VERSION
-ARG DEPLOY_USER
-ARG DEPLOY_GROUP
-# https://github.com/elastic/elasticsearch-docker
-FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
-
-ARG DEPLOY_USER
-ARG DEPLOY_GROUP
-ENV DEPLOY_USER $DEPLOY_USER
-ENV DEPLOY_GROUP $DEPLOY_GROUP
-
-RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-icu && \ 
-    /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-phonetic
-USER root
-RUN groupmod -g ${DEPLOY_GROUP} elasticsearch
-RUN usermod -u ${DEPLOY_USER} -g ${DEPLOY_GROUP} elasticsearch
-RUN chown -R elasticsearch /usr/share/elasticsearch
-RUN sed -i -e 's/--userspec=1000/--userspec=1000/g' \
-           -e 's/UID 1000/UID 1000/' \
-           -e 's/chown -R 1000/chown -R 1000/' /usr/local/bin/docker-entrypoint.sh
-RUN chown elasticsearch /usr/local/bin/docker-entrypoint.sh
-
-ENV JAVA_HOME /usr/share/elasticsearch/jdk

deployment/elk/elasticsearch/elasticsearch.yml

@@ -0,0 +1,5 @@
+---
+cluster.name: opendmp-cluster
+network.host: 0.0.0.0
+
+discovery.type: single-node

deployment/elk/elk.env

@@ -0,0 +1,8 @@
+ELASTIC_PASSWORD= elastic
+KIBANA_SYSTEM_PASSWORD= kibana
+LOGSTASH_INTERNAL_PASSWORD= logstash
+FILEBEAT_INTERNAL_PASSWORD= filebeat
+METRICBEAT_INTERNAL_PASSWORD=''
+HEARTBEAT_INTERNAL_PASSWORD=''
+MONITORING_INTERNAL_PASSWORD=''
+BEATS_SYSTEM_PASSWORD=''

deployment/elk/filebeat/Dockerfile

@@ -1,14 +0,0 @@
-ARG ELK_VERSION
-
-FROM docker.elastic.co/beats/filebeat:${ELK_VERSION}
-
-USER root
-RUN groupmod -g 1008 filebeat
-RUN usermod -u 1008 -g 1008 filebeat
-RUN chown -R filebeat /usr/share/filebeat
-RUN sed -i -e 's/--userspec=1000/--userspec=1008/g' \
-           -e 's/UID 1000/UID 1008/' \
-           -e 's/chown -R 1000/chown -R 1008/' /usr/local/bin/docker-entrypoint
-RUN chown filebeat /usr/local/bin/docker-entrypoint
-
-USER 1008:1008

deployment/elk/kibana/Dockerfile

@@ -1,21 +0,0 @@
-ARG ELK_VERSION
-ARG DEPLOY_USER
-ARG DEPLOY_GROUP
-
-# https://github.com/elastic/kibana-docker
-FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
-
-ARG DEPLOY_USER
-ARG DEPLOY_GROUP
-ENV DEPLOY_USER $DEPLOY_USER
-ENV DEPLOY_GROUP $DEPLOY_GROUP
-
-USER root
-RUN groupmod -g ${DEPLOY_GROUP} kibana
-RUN usermod -u ${DEPLOY_USER} -g ${DEPLOY_GROUP} kibana
-RUN chown -R kibana /usr/share/kibana
-
-USER ${DEPLOY_USER}:${DEPLOY_GROUP}
-
-# Add your kibana plugins setup here
-# Example: RUN kibana-plugin install <name|url>

deployment/elk/kibana/kibana.yml

@@ -0,0 +1,8 @@
+---
+server.name: opendmp.kibana
+server.host: 0.0.0.0
+
+elasticsearch.hosts: [ "http://opendmp.elasticsearch:9200" ]
+
+elasticsearch.username: kibana
+elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}

deployment/elk/logstash/Dockerfile

@@ -1,19 +0,0 @@
-ARG ELK_VERSION
-
-# https://github.com/elastic/logstash-docker
-FROM docker.elastic.co/logstash/logstash:${ELK_VERSION}
-
-USER root
-RUN groupmod -g 1008 logstash
-RUN usermod -u 1008 -g 1008 logstash
-RUN chown -R logstash /usr/share/logstash
-RUN sed -i -e 's/--userspec=1000/--userspec=1008/g' \
-           -e 's/UID 1000/UID 1008/' \
-           -e 's/chown -R 1000/chown -R 1008/' /usr/local/bin/docker-entrypoint
-RUN chown logstash /usr/local/bin/docker-entrypoint
-
-USER 1008:1008
-
-# Add your logstash plugins setup here
-# Example: RUN logstash-plugin install logstash-filter-json
-RUN logstash-plugin update logstash-input-beats

deployment/elk/logstash/logstash.yml

@@ -0,0 +1,4 @@
+---
+http.host: 0.0.0.0
+
+node.name: logstash

deployment/elk/setup/entrypoint.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+source "${BASH_SOURCE[0]%/*}"/lib.sh
+
+
+# --------------------------------------------------------
+# Users declarations
+
+declare -A users_passwords
+users_passwords=(
+	[logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}"
+	[kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}"
+	[metricbeat_internal]="${METRICBEAT_INTERNAL_PASSWORD:-}"
+	[filebeat_internal]="${FILEBEAT_INTERNAL_PASSWORD:-}"
+	[heartbeat_internal]="${HEARTBEAT_INTERNAL_PASSWORD:-}"
+	[monitoring_internal]="${MONITORING_INTERNAL_PASSWORD:-}"
+	[beats_system]="${BEATS_SYSTEM_PASSWORD=:-}"
+)
+
+declare -A users_roles
+users_roles=(
+	[logstash_internal]='logstash_writer'
+	[metricbeat_internal]='metricbeat_writer'
+	[filebeat_internal]='filebeat_writer'
+	[heartbeat_internal]='heartbeat_writer'
+	[monitoring_internal]='remote_monitoring_collector'
+)
+
+# --------------------------------------------------------
+# Roles declarations
+
+declare -A roles_files
+roles_files=(
+	[logstash_writer]='logstash_writer.json'
+	[metricbeat_writer]='metricbeat_writer.json'
+	[filebeat_writer]='filebeat_writer.json'
+	[heartbeat_writer]='heartbeat_writer.json'
+)
+
+# --------------------------------------------------------
+
+
+log 'Waiting for availability of Elasticsearch. This can take several minutes.'
+
+declare -i exit_code=0
+wait_for_elasticsearch || exit_code=$?
+
+if ((exit_code)); then
+	case $exit_code in
+		6)
+			suberr 'Could not resolve host. Is Elasticsearch running?'
+			;;
+		7)
+			suberr 'Failed to connect to host. Is Elasticsearch healthy?'
+			;;
+		28)
+			suberr 'Timeout connecting to host. Is Elasticsearch healthy?'
+			;;
+		*)
+			suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}"
+			;;
+	esac
+
+	exit $exit_code
+fi
+
+sublog 'Elasticsearch is running'
+
+log 'Waiting for initialization of built-in users'
+
+wait_for_builtin_users || exit_code=$?
+
+if ((exit_code)); then
+	suberr 'Timed out waiting for condition'
+	exit $exit_code
+fi
+
+sublog 'Built-in users were initialized'
+
+for role in "${!roles_files[@]}"; do
+	log "Role '$role'"
+
+	declare body_file
+	body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
+	if [[ ! -f "${body_file:-}" ]]; then
+		sublog "No role body found at '${body_file}', skipping"
+		continue
+	fi
+
+	sublog 'Creating/updating'
+	ensure_role "$role" "$(<"${body_file}")"
+done
+
+for user in "${!users_passwords[@]}"; do
+	log "User '$user'"
+	if [[ -z "${users_passwords[$user]:-}" ]]; then
+		sublog 'No password defined, skipping'
+		continue
+	fi
+
+	declare -i user_exists=0
+	user_exists="$(check_user_exists "$user")"
+
+	if ((user_exists)); then
+		sublog 'User exists, setting password'
+		set_user_password "$user" "${users_passwords[$user]}"
+	else
+		if [[ -z "${users_roles[$user]:-}" ]]; then
+			suberr '  No role defined, skipping creation'
+			continue
+		fi
+
+		sublog 'User does not exist, creating'
+		create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}"
+	fi
+done

deployment/elk/setup/lib.sh

@@ -0,0 +1,240 @@
+#!/usr/bin/env bash
+
+# Log a message.
+function log {
+	echo "[+] $1"
+}
+
+# Log a message at a sub-level.
+function sublog {
+	echo "   ⠿ $1"
+}
+
+# Log an error.
+function err {
+	echo "[x] $1" >&2
+}
+
+# Log an error at a sub-level.
+function suberr {
+	echo "   ⠍ $1" >&2
+}
+
+# Poll the 'elasticsearch' service until it responds with HTTP code 200.
+function wait_for_elasticsearch {
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" )
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	# retry for max 300s (60*5s)
+	for _ in $(seq 1 60); do
+		local -i exit_code=0
+		output="$(curl "${args[@]}")" || exit_code=$?
+
+		if ((exit_code)); then
+			result=$exit_code
+		fi
+
+		if [[ "${output: -3}" -eq 200 ]]; then
+			result=0
+			break
+		fi
+
+		sleep 5
+	done
+
+	if ((result)) && [[ "${output: -3}" -ne 000 ]]; then
+		echo -e "\n${output::-3}"
+	fi
+
+	return $result
+}
+
+# Poll the Elasticsearch users API until it returns users.
+function wait_for_builtin_users {
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" )
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+
+	local line
+	local -i exit_code
+	local -i num_users
+
+	# retry for max 30s (30*1s)
+	for _ in $(seq 1 30); do
+		num_users=0
+
+		# read exits with a non-zero code if the last read input doesn't end
+		# with a newline character. The printf without newline that follows the
+		# curl command ensures that the final input not only contains curl's
+		# exit code, but causes read to fail so we can capture the return value.
+		# Ref. https://unix.stackexchange.com/a/176703/152409
+		while IFS= read -r line || ! exit_code="$line"; do
+			if [[ "$line" =~ _reserved.+true ]]; then
+				(( num_users++ ))
+			fi
+		done < <(curl "${args[@]}"; printf '%s' "$?")
+
+		if ((exit_code)); then
+			result=$exit_code
+		fi
+
+		# we expect more than just the 'elastic' user in the result
+		if (( num_users > 1 )); then
+			result=0
+			break
+		fi
+
+		sleep 1
+	done
+
+	return $result
+}
+
+# Verify that the given Elasticsearch user exists.
+function check_user_exists {
+	local username=$1
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/user/${username}"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local -i exists=0
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then
+		result=0
+	fi
+	if [[ "${output: -3}" -eq 200 ]]; then
+		exists=1
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}"
+	else
+		echo "$exists"
+	fi
+
+	return $result
+}
+
+# Set password of a given Elasticsearch user.
+function set_user_password {
+	local username=$1
+	local password=$2
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/user/${username}/_password"
+		'-X' 'POST'
+		'-H' 'Content-Type: application/json'
+		'-d' "{\"password\" : \"${password}\"}"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 ]]; then
+		result=0
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}\n"
+	fi
+
+	return $result
+}
+
+# Create the given Elasticsearch user.
+function create_user {
+	local username=$1
+	local password=$2
+	local role=$3
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/user/${username}"
+		'-X' 'POST'
+		'-H' 'Content-Type: application/json'
+		'-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 ]]; then
+		result=0
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}\n"
+	fi
+
+	return $result
+}
+
+# Ensure that the given Elasticsearch role is up-to-date, create it if required.
+function ensure_role {
+	local name=$1
+	local body=$2
+
+	local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+	local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+		"http://${elasticsearch_host}:9200/_security/role/${name}"
+		'-X' 'POST'
+		'-H' 'Content-Type: application/json'
+		'-d' "$body"
+		)
+
+	if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+		args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+	fi
+
+	local -i result=1
+	local output
+
+	output="$(curl "${args[@]}")"
+	if [[ "${output: -3}" -eq 200 ]]; then
+		result=0
+	fi
+
+	if ((result)); then
+		echo -e "\n${output::-3}\n"
+	fi
+
+	return $result
+}

deployment/elk/setup/roles/filebeat_writer.json

@@ -0,0 +1,20 @@
+{
+    "cluster": [
+      "manage_ilm",
+      "manage_index_templates",
+      "manage_ingest_pipelines",
+      "monitor",
+      "read_pipeline"
+    ],
+    "indices": [
+      {
+        "names": [
+          "filebeat-*"
+        ],
+        "privileges": [
+          "create_doc",
+          "manage"
+        ]
+      }
+    ]
+  }

deployment/elk/setup/roles/heartbeat_writer.json

@@ -0,0 +1,18 @@
+{
+    "cluster": [
+      "manage_ilm",
+      "manage_index_templates",
+      "monitor"
+    ],
+    "indices": [
+      {
+        "names": [
+          "heartbeat-*"
+        ],
+        "privileges": [
+          "create_doc",
+          "manage"
+        ]
+      }
+    ]
+  }

deployment/elk/setup/roles/logstash_writer.json

@@ -0,0 +1,33 @@
+{
+    "cluster": [
+      "manage_index_templates",
+      "monitor",
+      "manage_ilm"
+    ],
+    "indices": [
+      {
+        "names": [
+          "logs-generic-default",
+          "logstash-*",
+          "ecs-logstash-*"
+        ],
+        "privileges": [
+          "write",
+          "create",
+          "create_index",
+          "manage",
+          "manage_ilm"
+        ]
+      },
+      {
+        "names": [
+          "logstash",
+          "ecs-logstash"
+        ],
+        "privileges": [
+          "write",
+          "manage"
+        ]
+      }
+    ]
+  }

deployment/elk/setup/roles/metricbeat_writer.json

@@ -0,0 +1,19 @@
+{
+    "cluster": [
+      "manage_ilm",
+      "manage_index_templates",
+      "monitor"
+    ],
+    "indices": [
+      {
+        "names": [
+          ".monitoring-*-mb",
+          "metricbeat-*"
+        ],
+        "privileges": [
+          "create_doc",
+          "manage"
+        ]
+      }
+    ]
+  }

deployment/keycloak/imports/opendmp-realm.json

@@ -1414,18 +1414,18 @@
       "clientId": "dmp_webapp",
       "name": "dmp_webapp",
       "description": "",
-      "rootUrl": "https://test.opendmp.eu/home",
+      "rootUrl": "http://localhost:8081/home",
       "adminUrl": "",
-      "baseUrl": "https://test.opendmp.eu/home",
+      "baseUrl": "http://localhost:8081/home",
       "surrogateAuthRequired": false,
       "enabled": true,
       "alwaysDisplayInConsole": false,
       "clientAuthenticatorType": "client-secret",
       "redirectUris": [
-        "https://test.opendmp.eu/*"
+        "http://localhost:8081/*"
       ],
       "webOrigins": [
-        "https://test.opendmp.eu"
+        "http://localhost:8081"
       ],
       "notBefore": 0,
       "bearerOnly": false,

deployment/keycloak/imports/opendmp-realm.jsonZone.Identifier

@@ -0,0 +1,3 @@
+[ZoneTransfer]
+ZoneId=3
+HostUrl=http://localhost:8081/

deployment/keycloak/keycloak.env

@@ -11,14 +11,12 @@ KC_DB_PASSWORD=keycloak-admin
 #Keycloak related configuration
 KEYCLOAK_ADMIN=keycloak-admin
 KEYCLOAK_ADMIN_PASSWORD=admin
-KC_HOSTNAME_URL=https://localhost:8082/keycloak
+KC_HOSTNAME_URL=http://localhost:8082/keycloak
-KC_HOSTNAME_ADMIN_URL=https://localhost:8082/keycloak
+KC_HOSTNAME_ADMIN_URL=http://localhost:8082/keycloak
 KC_HTTP_RELATIVE_PATH=/keycloak
 KC_PROXY_HEADERS=xforwarded
-KC_HOSTNAME_STRICT_HTTPS=true
+KC_HOSTNAME_STRICT_HTTPS=false
 KC_HOSTNAME_STRICT_BACKCHANNEL=true
 KC_TRANSACTION_XA_ENABLED=false
 KC_HEALTH_ENABLED=true
-KC_METRICS_ENABLED=true
+KC_METRICS_ENABLED=true
-KC_HTTPS_CERTIFICATE_FILE=/tmp/keycloak-selfsigned.crt
-KC_HTTPS_CERTIFICATE_KEY_FILE=/tmp/keycloak-selfsigned.key

deployment/postgres/Dockerfile

@@ -1,9 +0,0 @@
-ARG POSTGRES_TAG
-ARG DEPLOY_USER
-ARG DEPLOY_GROUP
-FROM postgres:${POSTGRES_TAG}
-COPY ./opendmp_init.sql /docker-entrypoint-initdb.d/
-COPY ./user_init.sql /docker-entrypoint-initdb.d/
-ENTRYPOINT ["docker-entrypoint.sh"]
-EXPOSE 5432
-CMD ["postgres"]

deployment/proxy/ProxyNginx.conf

@@ -18,9 +18,7 @@ server {
 # server {
 #     set $app_host $APP_HOST;
 #     set $app_port $APP_PORT;
-#     listen 8081 ssl;
+#     listen 8081;
-#     ssl_certificate /certifcates/cert.crt;
-#     ssl_certificate_key /certifcates/key.key;
 #     server_name  ${APP_HOST}${APP_PORT};
 #     proxy_pass_header Server;
     
@@ -97,18 +95,16 @@ server {
 server {
     set $ms_host $MS_HOST;
     set $ms_port $MS_PORT;
-    listen 8082 ssl;
+    listen 8082;
-    ssl_certificate /certifcates/cert.crt;
-    ssl_certificate_key /certifcates/key.key;
     server_name  ${MS_HOST};
     proxy_pass_header Server;
     
-    add_header X-XSS-Protection "1; mode=block" always;
+    # add_header X-XSS-Protection "1; mode=block" always;
-    add_header X-Content-Type-Options nosniff;
+    # add_header X-Content-Type-Options nosniff;
-    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
+    # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
-    add_header Referrer-Policy 'strict-origin' always;
+    # add_header Referrer-Policy 'strict-origin' always;
-    add_header Feature-Policy "usb 'none'; xr-spatial-tracking 'none'" always;
+    # add_header Feature-Policy "usb 'none'; xr-spatial-tracking 'none'" always;
-    add_header Permissions-Policy "geolocation=(),sync-xhr=(),microphone=(),camera=(),magnetometer=(),gyroscope=()" always;
+    # add_header Permissions-Policy "geolocation=(),sync-xhr=(),microphone=(),camera=(),magnetometer=(),gyroscope=()" always;
 
     location /keycloak/ {
 
@@ -121,26 +117,26 @@ server {
         proxy_set_header        X-Forwarded-Server $host;
 
         # Fix the “It appears that your reverse proxy set up is broken" error.
-        proxy_pass              https://opendmp.keycloak:8443;
+        proxy_pass              http://opendmp.keycloak:8080;
         proxy_read_timeout      90;
 
-        proxy_redirect          http://opendmp.keycloak:8443 https://${MS_HOST}${MS_PORT}/keycloak;
+        proxy_redirect          http://opendmp.keycloak:8080 http://${MS_HOST}${MS_PORT}/keycloak;
     }
 
-    location /elastic/ {
+    # location /elastic/ {
 
-        proxy_set_header        Host $host;
+    #     proxy_set_header        Host $host;
-        proxy_set_header        X-Real-IP $remote_addr;
+    #     proxy_set_header        X-Real-IP $remote_addr;
-        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    #     proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header        X-Forwarded-Proto $scheme;
+    #     proxy_set_header        X-Forwarded-Proto $scheme;
-        proxy_set_header        X-Forwarded-Port   $server_port;
+    #     proxy_set_header        X-Forwarded-Port   $server_port;
-        proxy_set_header        X-Forwarded-Host   $host;
+    #     proxy_set_header        X-Forwarded-Host   $host;
-        proxy_set_header        X-Forwarded-Server $host;
+    #     proxy_set_header        X-Forwarded-Server $host;
 
-        # Fix the “It appears that your reverse proxy set up is broken" error.
+    #     # Fix the “It appears that your reverse proxy set up is broken" error.
-        proxy_pass              https://opendmp.kibana:5601;
+    #     proxy_pass              http://opendmp.kibana:5601;
-        proxy_read_timeout      90;
+    #     proxy_read_timeout      90;
 
-        proxy_redirect          http://opendmp.kibana:5601 https://${MS_HOST}${MS_PORT}/elastic;
+    #     proxy_redirect          http://opendmp.kibana:5601 http://${MS_HOST}${MS_PORT}/elastic;
-    }
+    # }
 }

deployment/proxy/nginx-selfsigned.crt

@@ -1,23 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDvzCCAqegAwIBAgIUL9YHiVgQxrFPSGq4nMe6KbMznaowDQYJKoZIhvcNAQEL
-BQAwbzELMAkGA1UEBhMCR1IxDzANBgNVBAgMBkF0dGljYTEPMA0GA1UEBwwGQXRo
-ZW5zMQwwCgYDVQQKDANOTEcxDjAMBgNVBAsMBU9TRFlFMSAwHgYDVQQDDBdubGct
-b3NkeWUubG9jYWwuY2l0ZS5ncjAeFw0yMDExMjcxODMzNTJaFw0yMTExMjcxODMz
-NTJaMG8xCzAJBgNVBAYTAkdSMQ8wDQYDVQQIDAZBdHRpY2ExDzANBgNVBAcMBkF0
-aGVuczEMMAoGA1UECgwDTkxHMQ4wDAYDVQQLDAVPU0RZRTEgMB4GA1UEAwwXbmxn
-LW9zZHllLmxvY2FsLmNpdGUuZ3IwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
-AoIBAQCYsoKFVSg67/NckladOuqFDeJWMYOYa1MhMTCpLL5UksoYM+BiKbWvXmRH
-AGZAjIO2sGf6vTFO7SN6LdjCRRqJfS9zQVKHZKUdvfxkGnPaAIqWAwQkM4fFUlZF
-e5jsGd7owCoEEFAirJpz53z27Xa0En5CLSI5eLGBqLNz31zi1Rdh5BPLAtgLGEoS
-kOhlDXwgGtTmBAS82sXRr51J6DQr5stKsN68+DCwJiY6GU3Kun2Kyl+bepkEtHLM
-SOxe5Du3F93kC4TwFckG+JSLrq8neXNbyoBOLYNxPzAGLg9vmD7nX02hFfONdX5p
-zeGMD+0oyujL0HtH8nKkv9OtvYP1AgMBAAGjUzBRMB0GA1UdDgQWBBTrvPpWYBX6
-spVt2gpkpDW6yzrBZzAfBgNVHSMEGDAWgBTrvPpWYBX6spVt2gpkpDW6yzrBZzAP
-BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqfOJiorxiszfKKeD2
-eVda5pBU+qDnUOewRKizAktPm7V+cjM7bTEkmEHQ/oKlsT0FX5nsa8YfZCYgdotd
-cDWC9KqYy9itXPESoJIZ/gMJ57v1BaVDly8tHedXEltob9ywrUyf7OF55eP0fWwb
-AK90PfnRsxaurVYU5nfI9U/2jpi/LdsEYlJ7zUj7KM/Z2MwPA+be4EqjNcYLT/NB
-bavhjLgZNoTkI7wYOJug+ouPn6xJJcj06RS1Q4FxtfrsnAuT+L33HemUludEUE00
-TJwYWJN9hOgbyzTf5EsHxxME1gIhcYLYPMeBr9VIyJxdAEuuDJbn87oOB094sCA+
-nvpp
------END CERTIFICATE-----

deployment/proxy/nginx-selfsigned.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCYsoKFVSg67/Nc
-kladOuqFDeJWMYOYa1MhMTCpLL5UksoYM+BiKbWvXmRHAGZAjIO2sGf6vTFO7SN6
-LdjCRRqJfS9zQVKHZKUdvfxkGnPaAIqWAwQkM4fFUlZFe5jsGd7owCoEEFAirJpz
-53z27Xa0En5CLSI5eLGBqLNz31zi1Rdh5BPLAtgLGEoSkOhlDXwgGtTmBAS82sXR
-r51J6DQr5stKsN68+DCwJiY6GU3Kun2Kyl+bepkEtHLMSOxe5Du3F93kC4TwFckG
-+JSLrq8neXNbyoBOLYNxPzAGLg9vmD7nX02hFfONdX5pzeGMD+0oyujL0HtH8nKk
-v9OtvYP1AgMBAAECggEAajxoCpPAtaCT1GgL0sBWwdNzETzJrZWd9I3gqRL0KKsn
-58bP6fvS5/voEG36thYM3WHGNfDDCYJ7GFolYKPrXpS2Gp3r6T7gkdzIaGzvBVEz
-GkNm8jjX0TUDyLvBHSKyr4RitwkSd81WeCUqEDIOUCI9rZTxJsMN3IOetpNEcJt1
-xR2kVuTkQiIs5evQCQ2arqTf/VQFb7FuVTtmrOggsTn33FnrUDujTAAsbEoglgvw
-w1A2AYtMdGcrFIsUMJFdECkQGPVeqzKHddi1k1hv3DmOx5Rf4xJCdTL9ZYzbU1lw
-ydSAM88UA7MTZWNYCGb4HjqEeDRnMUN/Qbi4f80PgQKBgQDKWLxsCBBgFRoH6nlh
-TmBwYOmdQkBE09txpcpAFVZVv3eq/syZnT5+pcyj7EUexhu/p78UHPPkXDfnIKKe
-C+7VOEmSM9cPMZU6cqB1x4+YZkyiU1rPD2SG46ZxBBTKsPWauJNvtuhW3K6kwMCT
-ECiwG1f9EAmf9q7YKqdXGgUL3QKBgQDBL5WDIRtqHBdIPgHMue/teM+fP8I0/GKV
-D3oJjBLE+I7JNp0lpeVhDvqfAL0AgZ5023hjlPobUNtpWyuLufzY6S2Pv1scyM3A
-xW/LVXtC7QzdPmhrPxZkHEmRFA1zXdYo0xH4O+KDXVmYuzpIPfgrQkzt1EvP5jxv
-tbjY935C+QKBgHZhr+rsVNhBwDb9YQIi3p0gtcyBZCRgZjR5MHiJgzcri5GI/J6q
-tlNWIQGOS2oTsUxRkaLsxWvG4BXirAEXLiWkhrZ4icuj0JCfW21M0Z/xycf2SFx3
-vvKD4W6hWqCzIx3f+rITKp8XAT32XzQq8gMGHFY7ucXShryFR93XpTgpAoGALaAF
-WaDaDqdvwDoUxrsrNRSRRHUUctsglT/AfLy+OhLR9ieV2axijhexjRfpi1MRj1u+
-BRbMMuNXznwfvrJASyJXBloVNKkgHuUCUC1yHQ5LOX1hv+J4cVBU95Sa0KJaz+15
-kvzhtFC5tl6Rlzo7gEv6SzkWZpVjtKZgb62T/lECgYBr6PyDcGVGc8rbjxugnUnm
-rShh7nMRUiTMLpWrucf7Mfr4cwFgejCoEMXPgxJF5Q1acppL0dKQzfmLVqazNX58
-0XM0+fNDIeGyYKAbqtnqfmyI7O/Lb1jXPFuCNujzDxfeJX0yxoo38US4ZCD2iUrW
-ZK/FFkm5ncXTenBhpHOANQ==
------END PRIVATE KEY-----