refactored database roles and added support for remote hosts

pull/1/head
dcore94 3 years ago
parent b874434f2d
commit 116fbc95c2

@ -41,6 +41,11 @@ Other setting can be fine tuned by checking the variables in the proper roles wh
- *postgres*: defaults, templates and tasks for starting in the swarm a single instance of postgres - *postgres*: defaults, templates and tasks for starting in the swarm a single instance of postgres
- *workers*: defaults and task for starting in the swarm a replicated instance of the workers for executing HTTP, Shell, Eval operations. - *workers*: defaults and task for starting in the swarm a replicated instance of the workers for executing HTTP, Shell, Eval operations.
## Examples
The following example runs as user username on the remote hosts listed in hosts a swarm with 2 replicas of conductor server and ui, 1 postgres, 1 elasticsearch, 2 replicas of simple PyExec, an HAProxy that acts as load balancer.
`ansible-playbook -u username -i hosts site.yaml -e target_path=/tmp/conductor -e cluster_replacement=true`
## Change log ## Change log
See [CHANGELOG.md](CHANGELOG.md). See [CHANGELOG.md](CHANGELOG.md).

@ -0,0 +1,2 @@
[nw-cluster]
nubis1.int.d4science.net

@ -5,7 +5,7 @@
dest: "{{ target_path }}/conductor-swarm.yaml" dest: "{{ target_path }}/conductor-swarm.yaml"
- name: Generate conductor config from dynomite seeds - name: Generate conductor config from dynomite seeds
when: not use_jdbc when: conductor_db is defined and conductor_db == 'dynomite'
vars: vars:
seeds: "{{ lookup('file', '{{ target_path}}/seeds.list').splitlines() }}" seeds: "{{ lookup('file', '{{ target_path}}/seeds.list').splitlines() }}"
template: template:
@ -13,13 +13,13 @@
dest: "{{ target_path }}/{{ conductor_config }}" dest: "{{ target_path }}/{{ conductor_config }}"
- name: Generate conductor config for JDBC DB - name: Generate conductor config for JDBC DB
when: use_jdbc when: conductor_db is not defined or conductor_db != 'dynomite'
template: template:
src: "templates/{{ conductor_config_template }}" src: "templates/{{ conductor_config_template }}"
dest: "{{ target_path }}/{{ conductor_config }}" dest: "{{ target_path }}/{{ conductor_config }}"
- name: Copy conductor SQL schema init for JDBC DB - name: Copy conductor SQL schema init for JDBC DB
when: use_jdbc and init_db when: (conductor_db is not defined or conductor_db != 'dynomite') and init_db
template: template:
src: "templates/conductor-db-init-{{ conductor_db }}.sql.j2" src: "templates/conductor-db-init-{{ conductor_db }}.sql.j2"
dest: "{{ target_path }}/conductor-db-init.sql" dest: "{{ target_path }}/conductor-db-init.sql"

@ -1,7 +1,7 @@
CREATE DATABASE IF NOT EXISTS {{ jdbc_db }}; CREATE DATABASE IF NOT EXISTS {{ mysql_jdbc_db }};
GRANT ALL PRIVILEGES ON {{ jdbc_db }}.* TO {{ jdbc_user}}; GRANT ALL PRIVILEGES ON {{ jdbc_db }}.* TO {{ mysql_jdbc_user}};
FLUSH PRIVILEGES; FLUSH PRIVILEGES;
USE {{ jdbc_db }}; USE {{ mysql_jdbc_db }};
-- V1__initial_schema.sql -- -- V1__initial_schema.sql --
-- -------------------------------------------------------------------------------------------------------------- -- --------------------------------------------------------------------------------------------------------------

@ -8,15 +8,25 @@ conductor.grpc.server.enabled=false
# memory : The data is stored in memory and lost when the server dies. Useful for testing or demo # memory : The data is stored in memory and lost when the server dies. Useful for testing or demo
# redis : non-Dynomite based redis instance # redis : non-Dynomite based redis instance
# dynomite : Dynomite cluster. Use this for HA configuration. # dynomite : Dynomite cluster. Use this for HA configuration.
{% if use_jdbc is defined and use_jdbc %} {% if conductor_db is not defined or conductor_db == 'postgres' %}
db={{ conductor_db }} db=postgres
jdbc.url={{ jdbc_url }} jdbc.url={{ postgres_jdbc_url }}
jdbc.username={{ jdbc_user }} jdbc.username={{ postgres_jdbc_user }}
jdbc.password={{ jdbc_pass }} jdbc.password={{ postgres_jdbc_pass }}
conductor.{{ conductor_db }}.connection.pool.size.max=10 conductor.{{ conductor_db }}.connection.pool.size.max=10
conductor.{{ conductor_db }}.connection.pool.idle.min=2 conductor.{{ conductor_db }}.connection.pool.idle.min=2
flyway.enabled=false flyway.enabled=false
{% elif conductor_db is defined and conductor_db == 'mysql' %}
db=mysql
jdbc.url={{ mysql_jdbc_url }}
jdbc.username={{ mysql_jdbc_user }}
jdbc.password={{ mysql_jdbc_pass }}
conductor.{{ conductor_db }}.connection.pool.size.max=10
conductor.{{ conductor_db }}.connection.pool.idle.min=2
flyway.enabled=false
{% else %} {% else %}
db=dynomite db=dynomite

@ -0,0 +1,20 @@
---
elasticsearch_replicas: 1
postgres_service_name: 'postgresdb'
postgres_replicas: 1
postgres_conductor_db: postgres
postgres_jdbc_user: conductor
postgres_jdbc_pass: password
postgres_jdbc_db: conductor
postgres_jdbc_url: jdbc:postgresql://{{ postgres_service_name }}:5432/{{ mysql_jdbc_db }}
mysql_image_name: 'mariadb'
mysql_service_name: 'mysqldb'
mysql_replicas: 1
mysql_conductor_db: mysql
mysql_jdbc_user: conductor
mysql_jdbc_pass: password
mysql_jdbc_db: conductor
mysql_jdbc_url: jdbc:mysql://{{ mysql_service_name }}:3306/{{ mysql_jdbc_db }}?useSSL=false&allowPublicKeyRetrieval=true

@ -0,0 +1,17 @@
---
- name: Generate elasticsearch swarm
template:
src: templates/elasticsearch-swarm.yaml.j2
dest: "{{ target_path }}/elasticsearch-swarm.yaml"
- name: Generate postgres swarm
template:
src: templates/postgres-swarm.yaml.j2
dest: "{{ target_path }}/postgres-swarm.yaml"
when: conductor_db is not defined or conductor_db == 'postgres'
- name: "Generate mysql swarm, image used: {{ mysql_image_name }}"
template:
src: templates/mysql-swarm.yaml.j2
dest: "{{ target_path }}/mysql-swarm.yaml"
when: conductor_di is defined and conductor_db == 'mysql'

@ -0,0 +1,31 @@
version: '3.6'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:5.6.8
environment:
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- transport.host=0.0.0.0
- discovery.type=single-node
- xpack.security.enabled=false
networks:
{{ conductor_network }}:
aliases:
- es
logging:
driver: "journald"
deploy:
mode: replicated
replicas: {{ elasticsearch_replicas }}
#endpoint_mode: dnsrr
placement:
constraints: [node.role == worker]
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
networks:
{{ conductor_network }}:

@ -0,0 +1,30 @@
version: '3.6'
services:
{{ mysql_service_name }}:
image: {{ mysql_image_name }}
environment:
MYSQL_USER: {{ mysql_jdbc_user }}
MYSQL_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_ROOT_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_DB: {{ mysql_jdbc_db }}
{% if init_db %}
configs:
- source: db-init
target: "/docker-entrypoint-initdb.d/db-init.sql"
{% endif %}
networks:
- {{ conductor_network }}
deploy:
replicas: {{ mysql_replicas }}
placement:
constraints: [node.role == worker]
networks:
{{ conductor_network }}:
{% if init_db %}
configs:
db-init:
file: {{ target_path }}/conductor-db-init.sql
{% endif %}

@ -0,0 +1,31 @@
version: '3.6'
services:
{{ postgres_service_name }}:
image: postgres
ports:
- "5432:5432"
environment:
POSTGRES_USER: "{{ postgres_jdbc_user }}"
POSTGRES_PASSWORD: "{{ postgres_jdbc_pass }}"
POSTGRES_DB: "{{ postgres_jdbc_db }}"
{% if init_db %}
configs:
- source: db-init
target: "/docker-entrypoint-initdb.d/db-init.sql"
{% endif %}
networks:
- {{ conductor_network }}
deploy:
replicas: {{ postgres_replicas }}
placement:
constraints: [node.role == worker]
networks:
{{ conductor_network }}:
{% if init_db %}
configs:
db-init:
file: {{ target_path }}/conductor-db-init.sql
{% endif %}

@ -7,4 +7,4 @@ conductor_db: mysql
jdbc_user: conductor jdbc_user: conductor
jdbc_pass: password jdbc_pass: password
jdbc_db: conductor jdbc_db: conductor
jdbc_url: jdbc:mysql://{{ mysql_service_name }}:3306/{{ jdbc_db }}?useSSL=false&allowPublicKeyRetrieval=true jdbc_url: jdbc:mysql://{{ mysql_service_name }}:3306/{{ mysql_jdbc_db }}?useSSL=false&allowPublicKeyRetrieval=true

@ -5,9 +5,9 @@ services:
{{ mysql_service_name }}: {{ mysql_service_name }}:
image: {{ mysql_image_name }} image: {{ mysql_image_name }}
environment: environment:
MYSQL_USER: {{ jdbc_user }} MYSQL_USER: {{ mysql_jdbc_user }}
MYSQL_PASSWORD: {{ jdbc_pass }} MYSQL_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_ROOT_PASSWORD: {{ jdbc_pass }} MYSQL_ROOT_PASSWORD: {{ mysql_jdbc_pass }}
MYSQL_DB: {{ jdbc_db }} MYSQL_DB: {{ jdbc_db }}
{% if init_db %} {% if init_db %}
configs: configs:

@ -6,4 +6,4 @@ conductor_db: postgres
jdbc_user: conductor jdbc_user: conductor
jdbc_pass: password jdbc_pass: password
jdbc_db: conductor jdbc_db: conductor
jdbc_url: jdbc:postgresql://{{ postgres_service_name }}:5432/{{ jdbc_db }} jdbc_url: jdbc:postgresql://{{ postgres_service_name }}:5432/{{ postgres_jdbc_db }}

@ -7,9 +7,9 @@ services:
ports: ports:
- "5432:5432" - "5432:5432"
environment: environment:
POSTGRES_USER: "{{ jdbc_user }}" POSTGRES_USER: "{{ postgres_jdbc_user }}"
POSTGRES_PASSWORD: "{{ jdbc_pass }}" POSTGRES_PASSWORD: "{{ postgres_jdbc_pass }}"
POSTGRES_DB: "{{ jdbc_db }}" POSTGRES_DB: "{{ postgres_jdbc_db }}"
{% if init_db %} {% if init_db %}
configs: configs:
- source: db-init - source: db-init

@ -8,4 +8,3 @@
src: templates/config.cfg.j2 src: templates/config.cfg.j2
dest: "{{ target_path }}/{{ item.service }}-config.cfg" dest: "{{ target_path }}/{{ item.service }}-config.cfg"
loop: "{{ conductor_workers }}" loop: "{{ conductor_workers }}"

@ -1,16 +1,15 @@
--- ---
- hosts: localhost - hosts: [nw-cluster]
roles: roles:
- common - common
- role: cluster-replacement - role: cluster-replacement
when: when:
- cluster_replacement is defined and cluster_replacement|bool - cluster_replacement is defined and cluster_replacement|bool
- role: postgres - role: databases
when: db is not defined or db == 'postgres'
- role: mysql
when: db is defined and db == 'mysql'
- elasticsearch
- conductor - conductor
- role: workers
when:
- no_workers is not defined or not no_workers|bool
tasks: tasks:
- name: Start {{ db|default('postgres', true) }} and es - name: Start {{ db|default('postgres', true) }} and es
docker_stack: docker_stack:
@ -46,10 +45,12 @@
- cluster_replacement|bool - cluster_replacement|bool
- name: Start workers - name: Start workers
include_role: docker_stack:
name: workers name: conductor
state: present
compose:
- "{{ target_path }}/conductor-workers-swarm.yaml"
when: when:
- dry is not defined or dry|bool - dry is not defined or dry|bool
- workers is defined - no_workers is not defined or not no_workers|bool
- workers|bool

Loading…
Cancel
Save