From 23413d4e0f74a8f94f8c5e1e5fceb61c7d7d8327 Mon Sep 17 00:00:00 2001 From: Matus Jenca Date: Mon, 15 Apr 2024 17:01:54 +0200 Subject: [PATCH] Add backend TLS between MariaDB and ProxySQL This commit adds TLS connection between ProxySQL and MariaDB. Frontend TLS ( between services and ProxySQL) will be added in another commit. Parialy Implements: mariadb-ssl-support Change-Id: I154cbb096469c5515c9d8156c2c1c5dd07b95849 Signed-off-by: Matus Jenca --- ansible/group_vars/all.yml | 1 + .../certificates/tasks/generate-backend.yml | 14 +++++ ansible/roles/loadbalancer/tasks/config.yml | 2 - .../roles/loadbalancer/tasks/copy-certs.yml | 10 ++++ .../templates/proxysql/proxysql.json.j2 | 19 +++++++ .../templates/proxysql/proxysql.yaml.j2 | 13 +++++ ansible/roles/mariadb/handlers/main.yml | 6 +-- .../roles/mariadb/tasks/check-containers.yml | 2 +- ansible/roles/mariadb/tasks/config.yml | 6 ++- ansible/roles/mariadb/tasks/copy-certs.yml | 7 +++ ansible/roles/mariadb/templates/galera.cnf.j2 | 6 ++- .../roles/mariadb/templates/mariadb.json.j2 | 19 +++++++ etc/kolla/globals.yml | 2 +- ...proxysql-ssl-support-dd0b264212655424.yaml | 6 +++ requirements-core.yml | 3 ++ tests/j2lint.py | 5 ++ tests/run.yml | 10 ++++ tests/test-proxysql.sh | 53 +++++++++++++++++++ zuul.d/base.yaml | 2 + 19 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 ansible/roles/mariadb/tasks/copy-certs.yml create mode 100644 releasenotes/notes/mariadb-proxysql-ssl-support-dd0b264212655424.yaml create mode 100755 tests/test-proxysql.sh diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index a212005d83..9c90466cfd 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -85,6 +85,7 @@ database_user: "root" database_port: "3306" database_connection_recycle_time: 10 database_max_pool_size: 1 +database_enable_tls_backend: "{{ 'yes' if ((kolla_enable_tls_backend | bool ) and ( enable_proxysql | bool)) else 'no' }}" #################### # Container engine options diff --git a/ansible/roles/certificates/tasks/generate-backend.yml b/ansible/roles/certificates/tasks/generate-backend.yml index edb7789134..8b9b0600cf 100644 --- a/ansible/roles/certificates/tasks/generate-backend.yml +++ b/ansible/roles/certificates/tasks/generate-backend.yml @@ -77,3 +77,17 @@ dest: "{{ kolla_certificates_dir }}/rabbitmq-key.pem" when: - rabbitmq_enable_tls | bool + +- name: Copy backend TLS certificate and key for Mariadb + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0660" + remote_src: true + with_items: + - src: "{{ kolla_tls_backend_cert }}" + dest: "{{ kolla_certificates_dir }}/mariadb-cert.pem" + - src: "{{ kolla_tls_backend_key }}" + dest: "{{ kolla_certificates_dir }}/mariadb-key.pem" + when: + - database_enable_tls_backend | bool diff --git a/ansible/roles/loadbalancer/tasks/config.yml b/ansible/roles/loadbalancer/tasks/config.yml index d2ca4fea10..b5488b306c 100644 --- a/ansible/roles/loadbalancer/tasks/config.yml +++ b/ansible/roles/loadbalancer/tasks/config.yml @@ -221,8 +221,6 @@ - Restart haproxy container - include_tasks: copy-certs.yml - when: - - kolla_copy_ca_into_containers | bool - name: Copying over haproxy start script vars: diff --git a/ansible/roles/loadbalancer/tasks/copy-certs.yml b/ansible/roles/loadbalancer/tasks/copy-certs.yml index 7e26c26482..3c628dfa62 100644 --- a/ansible/roles/loadbalancer/tasks/copy-certs.yml +++ b/ansible/roles/loadbalancer/tasks/copy-certs.yml @@ -4,3 +4,13 @@ role: service-cert-copy vars: project_services: "{{ loadbalancer_services }}" + when: + - kolla_copy_ca_into_containers | bool + +- name: "Copy certificates and keys for MariaDB " + import_role: + role: service-cert-copy + vars: + project_services: "{{ loadbalancer_services }}" + project_name: mariadb + when: database_enable_tls_backend | bool diff --git a/ansible/roles/loadbalancer/templates/proxysql/proxysql.json.j2 b/ansible/roles/loadbalancer/templates/proxysql/proxysql.json.j2 index 047692b25d..8ad11470d3 100644 --- a/ansible/roles/loadbalancer/templates/proxysql/proxysql.json.j2 +++ b/ansible/roles/loadbalancer/templates/proxysql/proxysql.json.j2 @@ -25,5 +25,24 @@ "owner": "proxysql", "perm": "0700" } + {% if database_enable_tls_backend | bool %}, + { + "source": "{{ container_config_directory }}/ca-certificates/root.crt", + "dest": "/etc/proxysql/certs/root.crt", + "owner": "proxysql", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/mariadb-cert.pem", + "dest": "/etc/proxysql/certs/mariadb-cert.pem", + "owner": "proxysql", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/mariadb-key.pem", + "dest": "/etc/proxysql/certs/mariadb-key.pem", + "owner": "proxysql", + "perm": "0600" + }{% endif %} ] } diff --git a/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 b/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 index ac632c009f..28d3fbb7d7 100644 --- a/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 +++ b/ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2 @@ -32,6 +32,16 @@ mysql_variables: monitor_ping_interval: "{{ mariadb_monitor_ping_interval }}" monitor_ping_timeout: "{{ mariadb_monitor_ping_timeout }}" monitor_ping_max_failures: "{{ mariadb_monitor_ping_max_failures }}" + monitor_connect_timeout: 6000 + connect_timeout_client: 100000 + connect_timeout_server: 30000 + connect_timeout_server_max: 100000 +{% if database_enable_tls_backend | bool %} + ssl_p2s_ca: "/etc/proxysql/certs/root.crt" + ssl_p2s_cert: "/etc/proxysql/certs/mariadb-cert.pem" + ssl_p2s_key: "/etc/proxysql/certs/mariadb-key.pem" + have_ssl: true +{% endif %} mysql_servers: {% for shard_id, shard in mariadb_shards_info.shards.items() %} @@ -49,6 +59,9 @@ mysql_servers: max_replication_lag: {{ proxysql_backend_max_replication_lag }} weight : {{ WEIGHT }} comment : "Writer {{ host }}" +{% if database_enable_tls_backend | bool %} + use_ssl: 1 +{% endif %} {% endfor %} {% endfor %} diff --git a/ansible/roles/mariadb/handlers/main.yml b/ansible/roles/mariadb/handlers/main.yml index 68422deed3..d73fa754c0 100644 --- a/ansible/roles/mariadb/handlers/main.yml +++ b/ansible/roles/mariadb/handlers/main.yml @@ -60,7 +60,7 @@ - groups[mariadb_shard_group + '_port_alive_True'] is defined - inventory_hostname in groups[mariadb_shard_group + '_port_alive_True'] - kolla_action != "config" - listen: restart mariadb + listen: Restart mariadb container - name: Start MariaDB on new nodes group_by: @@ -70,7 +70,7 @@ - groups[mariadb_shard_group + '_port_alive_False'] is defined - inventory_hostname in groups[mariadb_shard_group + '_port_alive_False'] - kolla_action != "config" - listen: restart mariadb + listen: Restart mariadb container - name: Restart mariadb-clustercheck container vars: @@ -86,7 +86,7 @@ dimensions: "{{ service.dimensions }}" environment: "{{ service.environment }}" listen: - - restart mariadb-clustercheck + - Restart mariadb-clustercheck container when: - kolla_action != "config" - service | service_enabled_and_mapped_to_host diff --git a/ansible/roles/mariadb/tasks/check-containers.yml b/ansible/roles/mariadb/tasks/check-containers.yml index e5ff27d4e9..f4f7a603bd 100644 --- a/ansible/roles/mariadb/tasks/check-containers.yml +++ b/ansible/roles/mariadb/tasks/check-containers.yml @@ -12,4 +12,4 @@ healthcheck: "{{ item.value.healthcheck | default(omit) }}" with_dict: "{{ mariadb_services | select_services_enabled_and_mapped_to_host }}" notify: - - "restart {{ item.key }}" + - "Restart {{ item.key }} container" diff --git a/ansible/roles/mariadb/tasks/config.yml b/ansible/roles/mariadb/tasks/config.yml index b0691deef9..ad0e892103 100644 --- a/ansible/roles/mariadb/tasks/config.yml +++ b/ansible/roles/mariadb/tasks/config.yml @@ -44,7 +44,7 @@ become: true with_dict: "{{ mariadb_services | select_services_enabled_and_mapped_to_host }}" notify: - - "restart {{ item.key }}" + - "Restart {{ item.key }} container" - name: Copying over config.json files for mariabackup vars: @@ -72,4 +72,6 @@ become: true when: service | service_enabled_and_mapped_to_host notify: - - restart mariadb + - Restart mariadb container + +- include_tasks: copy-certs.yml diff --git a/ansible/roles/mariadb/tasks/copy-certs.yml b/ansible/roles/mariadb/tasks/copy-certs.yml new file mode 100644 index 0000000000..e1f6cd766a --- /dev/null +++ b/ansible/roles/mariadb/tasks/copy-certs.yml @@ -0,0 +1,7 @@ +--- +- name: "Copy certificates and keys for {{ project_name }}" + import_role: + role: service-cert-copy + vars: + project_services: "{{ mariadb_services }}" + when: database_enable_tls_backend | bool diff --git a/ansible/roles/mariadb/templates/galera.cnf.j2 b/ansible/roles/mariadb/templates/galera.cnf.j2 index ddf6a360f5..c7e5916fd5 100644 --- a/ansible/roles/mariadb/templates/galera.cnf.j2 +++ b/ansible/roles/mariadb/templates/galera.cnf.j2 @@ -11,7 +11,11 @@ default-character-set=utf8 basedir=/usr bind-address={{ api_interface_address }} port={{ mariadb_port }} - +{% if database_enable_tls_backend | bool %} +ssl_ca=/etc/mariadb/certs/root.crt +ssl_cert=/etc/mariadb/certs/mariadb-cert.pem +ssl_key=/etc/mariadb/certs/mariadb-key.pem +{% endif %} log_error=/var/log/kolla/mariadb/mariadb.log log_bin=mysql-bin diff --git a/ansible/roles/mariadb/templates/mariadb.json.j2 b/ansible/roles/mariadb/templates/mariadb.json.j2 index ac1b5bf27d..249cce4528 100644 --- a/ansible/roles/mariadb/templates/mariadb.json.j2 +++ b/ansible/roles/mariadb/templates/mariadb.json.j2 @@ -8,6 +8,25 @@ "owner": "mysql", "perm": "0600" } + {% if database_enable_tls_backend | bool %}, + { + "source": "{{ container_config_directory }}/ca-certificates/root.crt", + "dest": "/etc/mariadb/certs/root.crt", + "owner": "mysql", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/mariadb-cert.pem", + "dest": "/etc/mariadb/certs/mariadb-cert.pem", + "owner": "mysql", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/mariadb-key.pem", + "dest": "/etc/mariadb/certs/mariadb-key.pem", + "owner": "mysql", + "perm": "0600" + }{% endif %} ], "permissions": [ { diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index da7574eff6..4d0dd0e88e 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -245,7 +245,7 @@ workaround_ansible_issue_8743: yes #kolla_copy_ca_into_containers: "no" #haproxy_backend_cacert: "{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}" #haproxy_backend_cacert_dir: "/etc/ssl/certs" - +#database_enable_tls_backend: "{{ 'yes' if kolla_enable_tls_backend | bool and enable_proxysql | bool else 'no' }}" ################## # Backend options ################## diff --git a/releasenotes/notes/mariadb-proxysql-ssl-support-dd0b264212655424.yaml b/releasenotes/notes/mariadb-proxysql-ssl-support-dd0b264212655424.yaml new file mode 100644 index 0000000000..7415d495d1 --- /dev/null +++ b/releasenotes/notes/mariadb-proxysql-ssl-support-dd0b264212655424.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Implements SSL between Proxysql and MariaDB + ProxySQL must be enabled in order for encryption to work + `Partial Blueprint mariadb-ssl-support `__ diff --git a/requirements-core.yml b/requirements-core.yml index d5016a62f8..189b719f27 100644 --- a/requirements-core.yml +++ b/requirements-core.yml @@ -6,6 +6,9 @@ collections: - name: ansible.posix source: https://galaxy.ansible.com version: <2 + - name: ansible.utils + source: https://galaxy.ansible.com + version: <5 - name: community.crypto source: https://galaxy.ansible.com version: <3 diff --git a/tests/j2lint.py b/tests/j2lint.py index d9cc6e8bf9..b3981af249 100755 --- a/tests/j2lint.py +++ b/tests/j2lint.py @@ -33,6 +33,10 @@ from jinja2 import TemplateNotFound from kolla_ansible import kolla_address from kolla_ansible import put_address_in_context import os.path +try: + from ansible_collections.ansible.utils.plugins.filter import ipwrap +except ImportError: + from ansible_collections.ansible.netcommon.plugins.filter import ipwrap class AbsolutePathLoader(BaseLoader): @@ -57,6 +61,7 @@ def check(template, out, err, env=Environment(loader=AbsolutePathLoader(), env.filters['password_hash'] = get_encrypted_password env.filters['kolla_address'] = kolla_address env.filters['put_address_in_context'] = put_address_in_context + env.filters['ipwrap'] = ipwrap env.get_template(template) out.write("%s: Syntax OK\n" % template) return 0 diff --git a/tests/run.yml b/tests/run.yml index a32cec0c4b..a93917f53b 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -555,6 +555,16 @@ KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}" CONTAINER_ENGINE: "{{ container_engine }}" + - name: Run test-proxysql.sh script + script: + cmd: test-proxysql.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + when: scenario == "cells" + environment: + VIP: "{{ kolla_internal_vip_address }}" + TLS_ENABLED: "{{ tls_enabled }}" + - name: Run test-prometheus-opensearch.sh script script: cmd: test-prometheus-opensearch.sh diff --git a/tests/test-proxysql.sh b/tests/test-proxysql.sh new file mode 100755 index 0000000000..122048f9da --- /dev/null +++ b/tests/test-proxysql.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -o xtrace +set -o pipefail + + +function test_proxysql_connection_logged { + mariadb -h $VIP -P$DATABASE_PORT -u$DATABASE_USER -p$DATABASE_PASSWORD -e 'SHOW TABLES' +} + +function test_proxysql { + test_proxysql_connection_logged > /tmp/logs/ansible/test-proxysql 2>&1 + result=$? + echo $result + if [[ $result != 0 ]]; then + echo "Testing ProxySQL failed. See ansible/test-proxysql for details" + else + echo "Successfully tested ProxySQL. See ansible/test-proxysql for details" + fi + return $result +} +function test_proxysql_ssl_connection { + query="SELECT SUBSTRING_INDEX(variable_value, ',', -1) AS '' FROM information_schema.session_status WHERE variable_name = 'Ssl_cipher' LIMIT 1;" + result=$(mariadb -h $VIP -P$DATABASE_PORT -u$DATABASE_USER -p$DATABASE_PASSWORD -e "$query" --silent) + echo $result + if [[ "$result" =~ ^[[:space:]]*$ || -z "${result}" ]]; then + echo "ERROR: SSL is not utilized in ProxySQL" + return 1 + else + echo "SSL connection is working properly in proxysql" + return 0 + fi + +} + +DATABASE_PORT="${DATABASE_PORT:-3306}" +DATABASE_USER="${DATABASE_USER:-root_shard_0}" +TLS_ENABLED="${TLS_ENABLED:-false}" +if [[ -z "${VIP}" ]]; then + echo "VIP not set" + exit 1 +fi + +if [[ -z "${DATABASE_PASSWORD}" ]]; then + DATABASE_PASSWORD=$(grep ^database_password /etc/kolla/passwords.yml | cut -d" " -f2) +fi + +test_proxysql +if [ "$TLS_ENABLED" = true ]; then + test_proxysql_ssl_connection +fi + + + diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 461d322579..3b7e726954 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -241,7 +241,9 @@ - ^requirements-core.yml - ^ansible/roles/nova/ - ^tests/templates/(inventory|globals-default.j2) + - ^ansible/roles/loadbalancer/ - ^tests/test-core-openstack.sh + - ^tests/test-proxysql.sh vars: scenario: cells