refactored database roles and added support for remote hosts

pull/1/head
dcore94 2 years ago
parent b874434f2d
commit 116fbc95c2
  1. 5
      README.md
  2. 2
      hosts
  3. 6
      roles/conductor/tasks/main.yaml
  4. 6
      roles/conductor/templates/conductor-db-init-mysql.sql.j2
  5. 20
      roles/conductor/templates/conductor-swarm-config.properties.j2
  6. 20
      roles/databases/defaults/main.yaml
  7. 17
      roles/databases/tasks/main.yaml
  8. 31
      roles/databases/templates/elasticsearch-swarm.yaml.j2
  9. 30
      roles/databases/templates/mysql-swarm.yaml.j2
  10. 31
      roles/databases/templates/postgres-swarm.yaml.j2
  11. 2
      roles/mysql/defaults/main.yml
  12. 6
      roles/mysql/templates/mysql-swarm.yaml.j2
  13. 2
      roles/postgres/defaults/main.yml
  14. 6
      roles/postgres/templates/postgres-swarm.yaml.j2
  15. 1
      roles/workers/tasks/main.yaml
  16. 21
      site.yaml

@ -41,6 +41,11 @@ Other setting can be fine tuned by checking the variables in the proper roles wh
- *postgres*: defaults, templates and tasks for starting in the swarm a single instance of postgres
- *workers*: defaults and task for starting in the swarm a replicated instance of the workers for executing HTTP, Shell, Eval operations.
## Examples
The following example runs as user username on the remote hosts listed in hosts a swarm with 2 replicas of conductor server and ui, 1 postgres, 1 elasticsearch, 2 replicas of simple PyExec, an HAProxy that acts as load balancer.
`ansible-playbook -u username -i hosts site.yaml -e target_path=/tmp/conductor -e cluster_replacement=true`
## Change log
See [CHANGELOG.md](CHANGELOG.md).

@ -0,0 +1,2 @@
[nw-cluster]
nubis1.int.d4science.net

@ -5,7 +5,7 @@
dest: "{{ target_path }}/conductor-swarm.yaml"
- name: Generate conductor config from dynomite seeds
when: not use_jdbc
when: conductor_db is defined and conductor_db == 'dynomite'
vars:
seeds: "{{ lookup('file', '{{ target_path}}/seeds.list').splitlines() }}"
template:
@ -13,13 +13,13 @@
dest: "{{ target_path }}/{{ conductor_config }}"
- name: Generate conductor config for JDBC DB
when: use_jdbc
when: conductor_db is not defined or conductor_db != 'dynomite'
template:
src: "templates/{{ conductor_config_template }}"
dest: "{{ target_path }}/{{ conductor_config }}"
- name: Copy conductor SQL schema init for JDBC DB
when: use_jdbc and init_db
when: (conductor_db is not defined or conductor_db != 'dynomite') and init_db
template:
src: "templates/conductor-db-init-{{ conductor_db }}.sql.j2"
dest: "{{ target_path }}/conductor-db-init.sql"

@ -1,7 +1,7 @@
CREATE DATABASE IF NOT EXISTS {{ jdbc_db }};
GRANT ALL PRIVILEGES ON {{ jdbc_db }}.* TO {{ jdbc_user}};
CREATE DATABASE IF NOT EXISTS {{ mysql_jdbc_db }};
GRANT ALL PRIVILEGES ON {{ jdbc_db }}.* TO {{ mysql_jdbc_user}};
FLUSH PRIVILEGES;
USE {{ jdbc_db }};
USE {{ mysql_jdbc_db }};
-- V1__initial_schema.sql --
-- --------------------------------------------------------------------------------------------------------------

@ -8,15 +8,25 @@ conductor.grpc.server.enabled=false
# memory : The data is stored in memory and lost when the server dies. Useful for testing or demo
# redis : non-Dynomite based redis instance
# dynomite : Dynomite cluster. Use this for HA configuration.
{% if use_jdbc is defined and use_jdbc %}
db={{ conductor_db }}
jdbc.url={{ jdbc_url }}
jdbc.username={{ jdbc_user }}
jdbc.password={{ jdbc_pass }}
{% if conductor_db is not defined or conductor_db == 'postgres' %}
db=postgres
jdbc.url={{ postgres_jdbc_url }}
jdbc.username={{ postgres_jdbc_user }}
jdbc.password={{ postgres_jdbc_pass }}
conductor.{{ conductor_db }}.connection.pool.size.max=10
conductor.{{ conductor_db }}.connection.pool.idle.min=2
flyway.enabled=false
{% elif conductor_db is defined and conductor_db == 'mysql' %}
db=mysql
jdbc.url={{ mysql_jdbc_url }}
jdbc.username={{ mysql_jdbc_user }}
jdbc.password={{ mysql_jdbc_pass }}
conductor.{{ conductor_db }}.connection.pool.size.max=10
conductor.{{ conductor_db }}.connection.pool.idle.min=2
flyway.enabled=false
{% else %}
db=dynomite

@ -0,0 +1,20 @@
---
elasticsearch_replicas: 1
postgres_service_name: 'postgresdb'
postgres_replicas: 1
postgres_conductor_db: postgres
postgres_jdbc_user: conductor
postgres_jdbc_pass: password
postgres_jdbc_db: conductor
postgres_jdbc_url: jdbc:postgresql://{{ postgres_service_name }}:5432/{{ mysql_jdbc_db }}
mysql_image_name: 'mariadb'
mysql_service_name: 'mysqldb'
mysql_replicas: 1
mysql_conductor_db: mysql
mysql_jdbc_user: conductor
mysql_jdbc_pass: password
mysql_jdbc_db: conductor
mysql_jdbc_url: jdbc:mysql://{{ mysql_service_name }}:3306/{{ mysql_jdbc_db }}?useSSL=false&allowPublicKeyRetrieval=true

@ -0,0 +1,17 @@
---
- name: Generate elasticsearch swarm
template:
src: templates/elasticsearch-swarm.yaml.j2
dest: "{{ target_path }}/elasticsearch-swarm.yaml"
- name: Generate postgres swarm
template:
src: templates/postgres-swarm.yaml.j2
dest: "{{ target_path }}/postgres-swarm.yaml"
when: conductor_db is not defined or conductor_db == 'postgres'
- name: "Generate mysql swarm, image used: {{ mysql_image_name }}"
template:
src: templates/mysql-swarm.yaml.j2
dest: "{{ target_path }}/mysql-swarm.yaml"
when: conductor_di is defined and conductor_db == 'mysql'

@ -0,0 +1,31 @@
version: '3.6'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:5.6.8
environment:
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- transport.host=0.0.0.0
- discovery.type=single-node
- xpack.security.enabled=false
networks:
{{ conductor_network }}:
aliases:
- es
logging:
driver: "journald"
deploy:
mode: replicated
replicas: {{ elasticsearch_replicas }}
#endpoint_mode: dnsrr
placement:
constraints: [node.role == worker]
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
networks:
{{ conductor_network }}:

@ -0,0 +1,30 @@
version: '3.6'
services:
{{ mysql_service_name }}:
image: {{ mysql_image_name }}
environment:
MYSQL_USER: {{ mysql_jdbc_user }}
MYSQL_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_ROOT_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_DB: {{ mysql_jdbc_db }}
{% if init_db %}
configs:
- source: db-init
target: "/docker-entrypoint-initdb.d/db-init.sql"
{% endif %}
networks:
- {{ conductor_network }}
deploy:
replicas: {{ mysql_replicas }}
placement:
constraints: [node.role == worker]
networks:
{{ conductor_network }}:
{% if init_db %}
configs:
db-init:
file: {{ target_path }}/conductor-db-init.sql
{% endif %}

@ -0,0 +1,31 @@
version: '3.6'
services:
{{ postgres_service_name }}:
image: postgres
ports:
- "5432:5432"
environment:
POSTGRES_USER: "{{ postgres_jdbc_user }}"
POSTGRES_PASSWORD: "{{ postgres_jdbc_pass }}"
POSTGRES_DB: "{{ postgres_jdbc_db }}"
{% if init_db %}
configs:
- source: db-init
target: "/docker-entrypoint-initdb.d/db-init.sql"
{% endif %}
networks:
- {{ conductor_network }}
deploy:
replicas: {{ postgres_replicas }}
placement:
constraints: [node.role == worker]
networks:
{{ conductor_network }}:
{% if init_db %}
configs:
db-init:
file: {{ target_path }}/conductor-db-init.sql
{% endif %}

@ -7,4 +7,4 @@ conductor_db: mysql
jdbc_user: conductor
jdbc_pass: password
jdbc_db: conductor
jdbc_url: jdbc:mysql://{{ mysql_service_name }}:3306/{{ jdbc_db }}?useSSL=false&allowPublicKeyRetrieval=true
jdbc_url: jdbc:mysql://{{ mysql_service_name }}:3306/{{ mysql_jdbc_db }}?useSSL=false&allowPublicKeyRetrieval=true

@ -5,9 +5,9 @@ services:
{{ mysql_service_name }}:
image: {{ mysql_image_name }}
environment:
MYSQL_USER: {{ jdbc_user }}
MYSQL_PASSWORD: {{ jdbc_pass }}
MYSQL_ROOT_PASSWORD: {{ jdbc_pass }}
MYSQL_USER: {{ mysql_jdbc_user }}
MYSQL_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_ROOT_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_DB: {{ jdbc_db }}
{% if init_db %}
configs:

@ -6,4 +6,4 @@ conductor_db: postgres
jdbc_user: conductor
jdbc_pass: password
jdbc_db: conductor
jdbc_url: jdbc:postgresql://{{ postgres_service_name }}:5432/{{ jdbc_db }}
jdbc_url: jdbc:postgresql://{{ postgres_service_name }}:5432/{{ postgres_jdbc_db }}

@ -7,9 +7,9 @@ services:
ports:
- "5432:5432"
environment:
POSTGRES_USER: "{{ jdbc_user }}"
POSTGRES_PASSWORD: "{{ jdbc_pass }}"
POSTGRES_DB: "{{ jdbc_db }}"
POSTGRES_USER: "{{ postgres_jdbc_user }}"
POSTGRES_PASSWORD: "{{ postgres_jdbc_pass }}"
POSTGRES_DB: "{{ postgres_jdbc_db }}"
{% if init_db %}
configs:
- source: db-init

@ -8,4 +8,3 @@
src: templates/config.cfg.j2
dest: "{{ target_path }}/{{ item.service }}-config.cfg"
loop: "{{ conductor_workers }}"

@ -1,16 +1,15 @@
---
- hosts: localhost
- hosts: [nw-cluster]
roles:
- common
- role: cluster-replacement
when:
- cluster_replacement is defined and cluster_replacement|bool
- role: postgres
when: db is not defined or db == 'postgres'
- role: mysql
when: db is defined and db == 'mysql'
- elasticsearch
- role: databases
- conductor
- role: workers
when:
- no_workers is not defined or not no_workers|bool
tasks:
- name: Start {{ db|default('postgres', true) }} and es
docker_stack:
@ -46,10 +45,12 @@
- cluster_replacement|bool
- name: Start workers
include_role:
name: workers
docker_stack:
name: conductor
state: present
compose:
- "{{ target_path }}/conductor-workers-swarm.yaml"
when:
- dry is not defined or dry|bool
- workers is defined
- workers|bool
- no_workers is not defined or not no_workers|bool

Loading…
Cancel
Save