From d576994dce205134bc878d9c826e4970ce1b9a19 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Tue, 17 May 2022 13:35:13 +0200 Subject: [PATCH] minio baremetal installation. --- defaults/main.yml | 81 ++++++++----- handlers/main.yml | 10 +- tasks/main.yml | 178 ++++++++++++++++++++++------ templates/minio-letsencrypt-hook.j2 | 50 ++++++++ templates/minio.conf.upstart.j2 | 21 ++++ templates/minio.default.j2 | 59 +++++++++ templates/minio.service.j2 | 33 ++++++ 7 files changed, 364 insertions(+), 68 deletions(-) create mode 100644 templates/minio-letsencrypt-hook.j2 create mode 100644 templates/minio.conf.upstart.j2 create mode 100644 templates/minio.default.j2 create mode 100644 templates/minio.service.j2 diff --git a/defaults/main.yml b/defaults/main.yml index a34b2e2..2f16152 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,37 +1,54 @@ --- -minio_compose_dir: '/srv/minio_stack' -minio_docker_stack_name: 'minio' +minio_baremetal: true +minio_enabled: true +minio_binary: 'minio' +minio_binary_download: 'https://dl.min.io/server/minio/release/linux-amd64/{{ minio_binary }}' +minio_work_dir: /usr/local +minio_install_dir: '{{ minio_work_dir }}/bin' +minio_executable: '{{ minio_install_dir }}/{{ minio_binary }}' +minio_username: 'minio-user' +minio_user_home: '/srv/{{ minio_username }}' minio_access_key: 'use a vault' minio_secret_key: 'use a vault' minio_secrets: - - { name: minio_access_key, data: '{{ minio_access_key }}' } - - { name: minio_secret_key, data: '{{ minio_secret_key }}' } -minio_docker_service_server_name: 'minio' -#minio_docker_server_image: 'quay.io/minio/minio:RELEASE.2021-10-23T03-28-24Z' -minio_docker_server_image: 'quay.io/minio/minio' -minio_docker_network: 'distributed' -minio_docker_swarm_dnsrr: True -minio_server_instances: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - + - {name: minio_access_key, data: '{{ minio_access_key }}'} + - {name: minio_secret_key, data: '{{ minio_secret_key }}'} +minio_server_instances_num: 4 +minio_server_name_prefix: 'minio' +minio_server_domain_name: 'example.org' +minio_disk_volumes: 4 # -minio_data_prefix: /minio -minio_volume_prefix: /min_io -minio_disk_volumes: - - 3 - - 4 -minio_behind_haproxy: True -minio_haproxy_public_net: 'haproxy-public' -# -minio_keylocak_auth_url: http://localhost:8080/auth/ -#minio_keycloak_client_secret: 'use a vault' -minio_keycloak_realm: 'realm' -minio_keycloak_client_name: 'minio_client_name' -minio_keycloak_client_id: 'minio_client_id' +minio_data_prefix: /storage +minio_volume_prefix: 'minio' +minio_volume_subdir: 'data' +minio_port: 9000 +minio_volumes: 'https://{{ minio_server_name_prefix }}{1...{{ minio_server_instances_num }}}.{{ minio_server_domain_name }}:{{ minio_port }}/{{ minio_data_prefix }}/{{ minio_volume_prefix }}{1...{{ minio_disk_volumes }}}/{{ minio_volume_subdir }}' + +minio_dedicated_console: true +minio_console_port: 9001 +minio_behind_haproxy: true +minio_server_url: 'https://minio-reverse-proxy.example.org' +minio_ui_url: 'https://minio-ui-reverse-proxy.example.org' +minio_over_tls: true +minio_letsencrypt_certs: true +minio_tls_certs_dir: /etc/pki/minio +minio_tls_cert_file: '{{ minio_tls_certs_dir }}/fullchain' +minio_tls_key_file: '{{ minio_tls_certs_dir }}/privkey' +minio_root_user: minio_adm +# minio_root_password: 'Use a vault' +minio_storage_class_standard: 4 +minio_storage_class_rrs: 2 +# +minio_prometheus_url: '{{ minio_ui_url }}' +minio_prometheus_jobid: 'minio-job' +minio_prometheus_auth_type: public +minio_external_oidc: false +minio_openid_config_url: http://localhost:8080/auth/ +minio_openid_realm: 'realm' +minio_openid_client_id: 'minio_client_id' +# minio_openid_client_secret: 'use a vault' +minio_openid_client_name: 'minio_client_name' +minio_openid_set_claim_prefix: false +minio_openid_claim_prefix: 'minio_' +minio_openid_scopes: '' +minio_openid_redirect_uri: '' diff --git a/handlers/main.yml b/handlers/main.yml index 27474e0..efab34a 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,2 +1,10 @@ --- -# handlers file for ansible-role-template \ No newline at end of file +- name: Restart minio + service: + name: minio + state: restarted + +- name: reload minio + service: + name: minio + state: reloaded diff --git a/tasks/main.yml b/tasks/main.yml index 03a712e..8f5d118 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,50 +1,158 @@ --- -- name: Manage the installation of the MinIO configuration of the swarm service +- name: Manage the installation of a baremetal distributed MinIO block: - - name: Add the label that will be used as a constraint for the minio instances - docker_node: - hostname: '{{ item.1 }}' - labels: - minio: 'minio{{ item.0 }}' - labels_state: 'merge' - loop: "{{ minio_server_instances|zip(groups['minio_docker_swarm_nodes'])|list }}" - ignore_errors: True + - name: Create the minio-user username + user: + name: '{{ minio_username }}' + home: '{{ minio_user_home }}' + createhome: true + shell: /usr/sbin/nologin + system: true - - name: Create the min.io compose directory in the docker swarm manager + - name: Ensure that /etc/default exists file: - dest: '{{ minio_compose_dir }}' + dest: /etc/default + state: directory + + - name: Install the configuration file + template: + src: minio.default.j2 + dest: /etc/default/minio + owner: '{{ minio_username }}' + group: '{{ minio_username }}' + mode: 0440 + + - name: Change the ownership of the minio data disks + file: + dest: '{{ minio_data_prefix }}/{{ minio_volume_prefix }}_{{ item }}' + state: directory + owner: '{{ minio_username }}' + group: '{{ minio_username }}' + mode: 0700 + loop: '{{ minio_disk_volumes }}' + + - name: Download the minio binary + get_url: + url: '{{ minio_binary_download }}' + dest: '{{ minio_executable }}' + owner: root + group: root + mode: 0755 + + tags: ['minio', 'minio_baremetal'] + +- name: TLS certificates management with Letsencrypt + block: + - name: Create the acme hooks directory if it does not yet exist + file: + dest: '{{ letsencrypt_acme_services_scripts_dir }}' state: directory owner: root group: root - mode: 0400 - tags: [ 'minio', 'minio_docker', minio_docker_stack ] - - name: Install the min.io docker compose file + - name: Create the minio certificate directory + file: + dest: '{{ minio_tls_certs_dir }}' + state: directory + owner: root + group: '{{ minio_username }}' + mode: 0750 + + - name: Copy the key file where minio expects it + copy: + src: '{{ letsencrypt_acme_sh_certificates_install_path }}/privkey' + dest: '{{ minio_tls_key_file }}' + owner: root + group: '{{ minio_username }}' + mode: 0640 + remote_src: true + notify: reload minio + + - name: Copy the certificate file where minio expects it + copy: + src: '{{ letsencrypt_acme_sh_certificates_install_path }}/fullchain' + dest: '{{ minio_tls_cert_file }}' + owner: root + group: '{{ minio_username }}' + mode: 0640 + remote_src: true + notify: reload minio + + - name: Copy the CA trust file on deb systems + copy: + src: '/etc/ssl/certs/ca-certificates.crt' + dest: '{{ minio_tls_certs_dir }}/ca-certificates.crt' + owner: root + group: '{{ minio_username }}' + mode: 0640 + remote_src: true + notify: reload minio + when: ansible_distribution_file_variety == "Debian" + + - name: Copy the CA trust file on EL systems + copy: + src: '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem' + dest: '{{ minio_tls_certs_dir }}/ca-certificates.crt' + owner: root + group: '{{ minio_username }}' + mode: 0640 + remote_src: true + notify: reload minio + when: ansible_distribution_file_variety == "RedHat" + + - name: Install a script that updates the certificates upon renewal template: - src: minio-docker-compose.yml.j2 - dest: '{{ minio_compose_dir }}/docker-stack-minio.yml' + src: minio-letsencrypt-hook.j2 + dest: '{{ letsencrypt_acme_services_scripts_dir }}/minio' owner: root group: root - mode: 0400 - tags: [ 'minio', 'minio_docker', minio_docker_stack ] + mode: 4555 - - name: Create the secrets - ansible.builtin.docker_secret: - name: '{{ item.name }}' - data: '{{ item.data }}' - state: present - loop: '{{ minio_secrets }}' - tags: [ 'minio', 'minio_docker', minio_docker_stack ] + when: + - minio_letsencrypt_certs + - letsencrypt_acme_install + tags: ['minio', 'minio_baremetal', 'minio_letsencrypt'] - - name: Start the min.io stack - docker_stack: - name: '{{ minio_docker_stack_name }}' - state: present - compose: - - '{{ minio_compose_dir }}/docker-stack-minio.yml' - tags: [ 'minio', 'minio_docker', minio_docker_stack ] +- name: minio service + block: + - name: Install the minio systemd unit + template: + src: minio.conf.upstart.j2 + dest: /etc/init/minio.conf + owner: root + group: root + mode: 0644 + when: ansible_service_mgr != 'systemd' + notify: Restart minio + - name: Install the minio systemd unit + template: + src: minio.service.j2 + dest: /etc/systemd/system/minio.service + owner: root + group: root + mode: 0644 + register: minio_unit_update + when: ansible_service_mgr == 'systemd' + notify: Restart minio - run_once: True - when: docker_swarm_manager_main_node is defined and docker_swarm_manager_main_node | bool - tags: [ 'minio', 'minio_docker' ] + - name: Reload systemd + systemd: + daemon_reload: true + when: minio_unit_update is changed + + - name: Ensure that minio is running and enabled + service: + name: minio + state: started + enabled: true + when: minio_enabled + + - name: Ensure that minio is stopped and disabled + service: + name: minio + state: stopped + enabled: false + when: not minio_enabled + + tags: ['minio', 'minio_baremetal', 'minio_letsencrypt'] diff --git a/templates/minio-letsencrypt-hook.j2 b/templates/minio-letsencrypt-hook.j2 new file mode 100644 index 0000000..4b132b7 --- /dev/null +++ b/templates/minio-letsencrypt-hook.j2 @@ -0,0 +1,50 @@ +#!/bin/bash + +LE_CERTS_DIR="{{ letsencrypt_acme_sh_certificates_install_path }}" +LE_LOG_DIR=/var/log/letsencrypt +LE_LOGFILE="$LE_LOG_DIR/minio.log" +MINIO_CERTS_DIR="{{ minio_tls_certs_dir }}" +MINIO_KEYFILE="{{ minio_tls_key_file }}" +MINIO_CERTFILE="{{ minio_tls_cert_file }}" +DATE=$( date ) +RETVAL= + +[ ! -d $LE_LOG_DIR ] && mkdir $LE_LOG_DIR +echo "$DATE" >> "$LE_LOGFILE" + +logger "acme-minio-hook: Check if the certificate has been renewed" +cmp ${LE_CERTS_DIR}/privkey ${MINIO_KEYFILE} +RETVAL=$? +if [ $RETVAL -eq 0 ] ; then + logger "acme-minio-hook: No new cerficate." + echo "acme-minio-hook: No new cerficate." >> $LE_LOGFILE + exit 0 +else + logger "acme-minio-hook: Copying the key file" + echo "Copy the certificate files" >> $LE_LOGFILE + /bin/cp -f ${LE_CERTS_DIR}/privkey ${MINIO_KEYFILE} + /bin/cp -f ${LE_CERTS_DIR}/fullchain ${MINIO_CERTFILE} +{% if ansible_distribution_file_variety == "Debian" %} + if [ -f /etc/ssl/certs/ca-certificates.crt ] ; then + cp -pf /etc/ssl/certs/ca-certificates.crt ${MINIO_CERTS_DIR}/ca-certificates.crt + fi +{% endif %} +{% if ansible_distribution_file_variety == "RedHat" %} + if [ -f /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem ] ; then + cp -pf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem ${MINIO_CERTS_DIR}/ca-certificates.crt + fi +{% endif %} +fi + +chmod 440 ${MINIO_KEYFILE} ${MINIO_CERTFILE} +chown root ${MINIO_KEYFILE} ${MINIO_CERTFILE} +chgrp minio ${MINIO_KEYFILE} ${MINIO_CERTFILE} + +logger "acme-minio-hook: Restart the minio service after a certificate renewal" +systemctl restart minio >> $LE_LOGFILE 2>&1 +echo "acme-minio-hook: Restart the minio service" >> $LE_LOGFILE + +logger "acme-minio-hook: Done" +echo "acme-minio-hook: Done." >> $LE_LOGFILE + +exit 0 diff --git a/templates/minio.conf.upstart.j2 b/templates/minio.conf.upstart.j2 new file mode 100644 index 0000000..bf17da7 --- /dev/null +++ b/templates/minio.conf.upstart.j2 @@ -0,0 +1,21 @@ +description "MinIO object storage server" + +start on runlevel [2345] +stop on runlevel [016] + +console log + +setuid {{ minio_username }} +setgid {{ minio_username }} + +respawn +respawn limit 10 5 + +env MINIOVOLUMES="{{ minio_volumes }}" +env MINIO_OPTS="{% if minio_dedicated_console }}--console-address :{{ minio_console_port }}{% endif %} --certs-dir {{ minio_tls_certs_dir }}" + +limit nofile 1048576 1048576 + +script + exec {{ minio_executable }} server $MINIO_OPTS $MINIOVOLUMES +end script diff --git a/templates/minio.default.j2 b/templates/minio.default.j2 new file mode 100644 index 0000000..ec9a08f --- /dev/null +++ b/templates/minio.default.j2 @@ -0,0 +1,59 @@ +# See https://docs.min.io/minio/baremetal/reference/minio-server/minio-server.html#environment-variables +# Set the hosts and volumes MinIO uses at startup +# The command uses MinIO expansion notation {x...y} to denote a +# sequential series. +# +# The following example covers four MinIO hosts +# with 4 drives each at the specified hostname and drive locations. +# The command includes the port that each MinIO server listens on +# (default 9000) + +MINIO_VOLUMES="{{ minio_volumes }}" + +# Set all MinIO server options +# +# The following explicitly sets the MinIO Console listen address to +# port 9001 on all network interfaces. The default behavior is dynamic +# port selection. + +MINIO_OPTS="{% if minio_dedicated_console }}--console-address :{{ minio_console_port }}{% endif %} --certs-dir {{ minio_tls_certs_dir }}" + +# Set the root username. This user has unrestricted permissions to +# perform S3 and administrative API operations on any resource in the +# deployment. +# +# Defer to your organizations requirements for superadmin user name. + +MINIO_ROOT_USER={{ minio_root_user }} + +# Set the root password +# +# Use a long, random, unique string that meets your organizations +# requirements for passwords. + +MINIO_ROOT_PASSWORD={{ minio_root_password }} + +# Set to the URL of the load balancer for the MinIO deployment +# This value *must* match across all MinIO servers. If you do +# not have a load balancer, set this value to to any *one* of the +# MinIO hosts in the deployment as a temporary measure. +MINIO_SERVER_URL="{{ minio_server_url }}" +# +MINIO_PROMETHEUS_URL="{{ minio_prometheus_url }}" +MINIO_PROMETHEUS_JOB_ID="{{ minio_prometheus_jobid }}" +MINIO_BROWSER_REDIRECT_URL="{{ minio_ui_url }}" +MINIO_STORAGE_CLASS_STANDARD={{ minio_storage_class_standard }} +MINIO_STORAGE_CLASS_RRS={{ minio_storage_class_rrs }} +# jwt, public +MINIO_PROMETHEUS_AUTH_TYPE={{ minio_prometheus_auth_type }} +{% if minio_external_oidc %} +MINIO_IDENTITY_OPENID_CONFIG_URL={{ minio_openid_config_url }} +MINIO_IDENTITY_OPENID_CLIENT_ID={{ minio_openid_client_id }} +MINIO_IDENTITY_OPENID_CLIENT_SECRET={{ minio_openid_client_secret }} +MINIO_IDENTITY_OPENID_CLAIM_NAME={{ minio_openid_client_name }} +{% if minio_openid_set_claim_prefix %} +MINIO_IDENTITY_OPENID_CLAIM_PREFIX={{ minio_openid_claim_prefix }} +{% endif %} +MINIO_IDENTITY_OPENID_SCOPES={{ minio_openid_scopes }} +MINIO_IDENTITY_OPENID_REDIRECT_URI={{ minio_openid_redirect_uri }} +{% endif %} diff --git a/templates/minio.service.j2 b/templates/minio.service.j2 new file mode 100644 index 0000000..bcddf39 --- /dev/null +++ b/templates/minio.service.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=MinIO +Documentation=https://docs.min.io +Wants=network-online.target +After=network-online.target +AssertFileIsExecutable={{ minio_executable }} + +[Service] +WorkingDirectory={{ minio_work_dir }} + +User={{ minio_username }} +Group={{ minio_username }} +ProtectProc=invisible + +EnvironmentFile=-/etc/default/minio +ExecStartPre=/bin/bash -c "if [ -z \"${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi" +ExecStart={{ minio_executable }} server $MINIO_OPTS $MINIO_VOLUMES + +# Let systemd restart this service always +Restart=always + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=1048576 + +# Specifies the maximum number of threads this process can create +TasksMax=infinity + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=infinity +SendSIGKILL=no + +[Install] +WantedBy=multi-user.target