Add backend TLS between MariaDB and ProxySQL

This commit adds TLS connection between ProxySQL and MariaDB.
Frontend TLS ( between services and ProxySQL) will be
added in another commit.

Parialy Implements: mariadb-ssl-support

Change-Id: I154cbb096469c5515c9d8156c2c1c5dd07b95849
Signed-off-by: Matus Jenca <matus.jenca@dnation.cloud>

ansible/group_vars/all.yml

@@ -85,6 +85,7 @@ database_user: "root"
 database_port: "3306"
 database_connection_recycle_time: 10
 database_max_pool_size: 1
+database_enable_tls_backend: "{{ 'yes' if ((kolla_enable_tls_backend | bool ) and ( enable_proxysql | bool)) else 'no' }}"
 
 ####################
 # Container engine options

ansible/roles/certificates/tasks/generate-backend.yml

@@ -77,3 +77,17 @@
       dest: "{{ kolla_certificates_dir }}/rabbitmq-key.pem"
   when:
     - rabbitmq_enable_tls | bool
+
+- name: Copy backend TLS certificate and key for Mariadb
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: "0660"
+    remote_src: true
+  with_items:
+    - src: "{{ kolla_tls_backend_cert }}"
+      dest: "{{ kolla_certificates_dir }}/mariadb-cert.pem"
+    - src: "{{ kolla_tls_backend_key }}"
+      dest: "{{ kolla_certificates_dir }}/mariadb-key.pem"
+  when:
+    - database_enable_tls_backend | bool

ansible/roles/loadbalancer/tasks/config.yml

@@ -221,8 +221,6 @@
     - Restart haproxy container
 
 - include_tasks: copy-certs.yml
-  when:
-    - kolla_copy_ca_into_containers | bool
 
 - name: Copying over haproxy start script
   vars:

ansible/roles/loadbalancer/tasks/copy-certs.yml

@@ -4,3 +4,13 @@
     role: service-cert-copy
   vars:
     project_services: "{{ loadbalancer_services }}"
+  when:
+    - kolla_copy_ca_into_containers | bool
+
+- name: "Copy certificates and keys for MariaDB "
+  import_role:
+    role: service-cert-copy
+  vars:
+    project_services: "{{ loadbalancer_services }}"
+    project_name: mariadb
+  when: database_enable_tls_backend | bool

ansible/roles/loadbalancer/templates/proxysql/proxysql.json.j2

@@ -25,5 +25,24 @@
             "owner": "proxysql",
             "perm": "0700"
         }
+        {% if database_enable_tls_backend | bool %},
+        {
+            "source": "{{ container_config_directory }}/ca-certificates/root.crt",
+            "dest": "/etc/proxysql/certs/root.crt",
+            "owner": "proxysql",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/mariadb-cert.pem",
+            "dest": "/etc/proxysql/certs/mariadb-cert.pem",
+            "owner": "proxysql",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/mariadb-key.pem",
+            "dest": "/etc/proxysql/certs/mariadb-key.pem",
+            "owner": "proxysql",
+            "perm": "0600"
+        }{% endif %}
     ]
 }

ansible/roles/loadbalancer/templates/proxysql/proxysql.yaml.j2

@@ -32,6 +32,16 @@ mysql_variables:
    monitor_ping_interval: "{{ mariadb_monitor_ping_interval }}"
    monitor_ping_timeout: "{{ mariadb_monitor_ping_timeout }}"
    monitor_ping_max_failures: "{{ mariadb_monitor_ping_max_failures }}"
+   monitor_connect_timeout: 6000
+   connect_timeout_client: 100000
+   connect_timeout_server: 30000
+   connect_timeout_server_max: 100000
+{% if database_enable_tls_backend | bool %}
+   ssl_p2s_ca: "/etc/proxysql/certs/root.crt"
+   ssl_p2s_cert: "/etc/proxysql/certs/mariadb-cert.pem"
+   ssl_p2s_key: "/etc/proxysql/certs/mariadb-key.pem"
+   have_ssl: true
+{% endif %}
 
 mysql_servers:
 {% for shard_id, shard in mariadb_shards_info.shards.items() %}
@@ -49,6 +59,9 @@ mysql_servers:
     max_replication_lag: {{ proxysql_backend_max_replication_lag }}
     weight :  {{ WEIGHT }}
     comment :  "Writer {{ host }}"
+{% if database_enable_tls_backend | bool %}
+    use_ssl: 1
+{% endif %}
 {% endfor %}
 {% endfor %}
 

ansible/roles/mariadb/handlers/main.yml

@@ -60,7 +60,7 @@
     - groups[mariadb_shard_group + '_port_alive_True'] is defined
     - inventory_hostname in groups[mariadb_shard_group + '_port_alive_True']
     - kolla_action != "config"
-  listen: restart mariadb
+  listen: Restart mariadb container
 
 - name: Start MariaDB on new nodes
   group_by:
@@ -70,7 +70,7 @@
     - groups[mariadb_shard_group + '_port_alive_False'] is defined
     - inventory_hostname in groups[mariadb_shard_group + '_port_alive_False']
     - kolla_action != "config"
-  listen: restart mariadb
+  listen: Restart mariadb container
 
 - name: Restart mariadb-clustercheck container
   vars:
@@ -86,7 +86,7 @@
     dimensions: "{{ service.dimensions }}"
     environment: "{{ service.environment }}"
   listen:
-    - restart mariadb-clustercheck
+    - Restart mariadb-clustercheck container
   when:
     - kolla_action != "config"
     - service | service_enabled_and_mapped_to_host

ansible/roles/mariadb/tasks/check-containers.yml

@@ -12,4 +12,4 @@
     healthcheck: "{{ item.value.healthcheck | default(omit) }}"
   with_dict: "{{ mariadb_services | select_services_enabled_and_mapped_to_host }}"
   notify:
-    - "restart {{ item.key }}"
+    - "Restart {{ item.key }} container"

ansible/roles/mariadb/tasks/config.yml

@@ -44,7 +44,7 @@
   become: true
   with_dict: "{{ mariadb_services | select_services_enabled_and_mapped_to_host }}"
   notify:
-    - "restart {{ item.key }}"
+    - "Restart {{ item.key }} container"
 
 - name: Copying over config.json files for mariabackup
   vars:
@@ -72,4 +72,6 @@
   become: true
   when: service | service_enabled_and_mapped_to_host
   notify:
-    - restart mariadb
+    - Restart mariadb container
+
+- include_tasks: copy-certs.yml

ansible/roles/mariadb/tasks/copy-certs.yml

@@ -0,0 +1,7 @@
+---
+- name: "Copy certificates and keys for {{ project_name }}"
+  import_role:
+    role: service-cert-copy
+  vars:
+    project_services: "{{ mariadb_services }}"
+  when: database_enable_tls_backend | bool

ansible/roles/mariadb/templates/galera.cnf.j2

@@ -11,7 +11,11 @@ default-character-set=utf8
 basedir=/usr
 bind-address={{ api_interface_address }}
 port={{ mariadb_port }}
-
+{% if database_enable_tls_backend | bool %}
+ssl_ca=/etc/mariadb/certs/root.crt
+ssl_cert=/etc/mariadb/certs/mariadb-cert.pem
+ssl_key=/etc/mariadb/certs/mariadb-key.pem
+{% endif %}
 log_error=/var/log/kolla/mariadb/mariadb.log
 
 log_bin=mysql-bin

ansible/roles/mariadb/templates/mariadb.json.j2

@@ -8,6 +8,25 @@
             "owner": "mysql",
             "perm": "0600"
         }
+        {% if database_enable_tls_backend | bool %},
+        {
+            "source": "{{ container_config_directory }}/ca-certificates/root.crt",
+            "dest": "/etc/mariadb/certs/root.crt",
+            "owner": "mysql",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/mariadb-cert.pem",
+            "dest": "/etc/mariadb/certs/mariadb-cert.pem",
+            "owner": "mysql",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/mariadb-key.pem",
+            "dest": "/etc/mariadb/certs/mariadb-key.pem",
+            "owner": "mysql",
+            "perm": "0600"
+        }{% endif %}
     ],
     "permissions": [
         {

etc/kolla/globals.yml

@@ -245,7 +245,7 @@ workaround_ansible_issue_8743: yes
 #kolla_copy_ca_into_containers: "no"
 #haproxy_backend_cacert: "{{ 'ca-certificates.crt' if kolla_base_distro in ['debian', 'ubuntu'] else 'ca-bundle.trust.crt' }}"
 #haproxy_backend_cacert_dir: "/etc/ssl/certs"
-
+#database_enable_tls_backend: "{{ 'yes' if kolla_enable_tls_backend | bool and enable_proxysql | bool else 'no' }}"
 ##################
 # Backend options
 ##################

releasenotes/notes/mariadb-proxysql-ssl-support-dd0b264212655424.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Implements SSL between Proxysql and MariaDB
+    ProxySQL must be enabled in order for encryption to work
+    `Partial Blueprint  mariadb-ssl-support  <https://blueprints.launchpad.net/kolla-ansible/+spec/mariadb-ssl-support>`__

requirements-core.yml

@@ -6,6 +6,9 @@ collections:
   - name: ansible.posix
     source: https://galaxy.ansible.com
     version: <2
+  - name: ansible.utils
+    source: https://galaxy.ansible.com
+    version: <5
   - name: community.crypto
     source: https://galaxy.ansible.com
     version: <3

tests/j2lint.py

@@ -33,6 +33,10 @@ from jinja2 import TemplateNotFound
 from kolla_ansible import kolla_address
 from kolla_ansible import put_address_in_context
 import os.path
+try:
+    from ansible_collections.ansible.utils.plugins.filter import ipwrap
+except ImportError:
+    from ansible_collections.ansible.netcommon.plugins.filter import ipwrap
 
 
 class AbsolutePathLoader(BaseLoader):
@@ -57,6 +61,7 @@ def check(template, out, err, env=Environment(loader=AbsolutePathLoader(),
         env.filters['password_hash'] = get_encrypted_password
         env.filters['kolla_address'] = kolla_address
         env.filters['put_address_in_context'] = put_address_in_context
+        env.filters['ipwrap'] = ipwrap
         env.get_template(template)
         out.write("%s: Syntax OK\n" % template)
         return 0

tests/run.yml

@@ -555,6 +555,16 @@
             KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
             CONTAINER_ENGINE: "{{ container_engine }}"
 
+        - name: Run test-proxysql.sh script
+          script:
+            cmd: test-proxysql.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: scenario == "cells"
+          environment:
+            VIP: "{{ kolla_internal_vip_address }}"
+            TLS_ENABLED: "{{ tls_enabled }}"
+
         - name: Run test-prometheus-opensearch.sh script
           script:
             cmd: test-prometheus-opensearch.sh

tests/test-proxysql.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -o xtrace
+set -o pipefail
+
+
+function test_proxysql_connection_logged {
+    mariadb -h $VIP -P$DATABASE_PORT -u$DATABASE_USER -p$DATABASE_PASSWORD -e 'SHOW TABLES'
+}
+
+function test_proxysql {
+    test_proxysql_connection_logged > /tmp/logs/ansible/test-proxysql 2>&1
+    result=$?
+    echo $result
+    if [[ $result != 0 ]]; then
+        echo "Testing ProxySQL failed. See ansible/test-proxysql for details"
+    else
+        echo "Successfully tested ProxySQL. See ansible/test-proxysql for details"
+    fi
+    return $result
+}
+function test_proxysql_ssl_connection {
+    query="SELECT SUBSTRING_INDEX(variable_value, ',', -1) AS '' FROM information_schema.session_status WHERE variable_name = 'Ssl_cipher' LIMIT 1;"
+    result=$(mariadb -h $VIP -P$DATABASE_PORT -u$DATABASE_USER -p$DATABASE_PASSWORD -e "$query" --silent)
+    echo $result
+    if [[ "$result" =~ ^[[:space:]]*$ || -z "${result}" ]]; then
+        echo "ERROR: SSL is not utilized in ProxySQL"
+        return 1
+    else
+        echo "SSL connection is working properly in proxysql"
+        return 0
+    fi
+
+}
+
+DATABASE_PORT="${DATABASE_PORT:-3306}"
+DATABASE_USER="${DATABASE_USER:-root_shard_0}"
+TLS_ENABLED="${TLS_ENABLED:-false}"
+if [[ -z "${VIP}"  ]]; then
+    echo "VIP  not set"
+    exit 1
+fi
+
+if [[ -z "${DATABASE_PASSWORD}"  ]]; then
+    DATABASE_PASSWORD=$(grep ^database_password /etc/kolla/passwords.yml | cut -d" " -f2)
+fi
+
+test_proxysql
+if [ "$TLS_ENABLED" = true ]; then
+    test_proxysql_ssl_connection
+fi
+
+
+

zuul.d/base.yaml

@@ -241,7 +241,9 @@
       - ^requirements-core.yml
       - ^ansible/roles/nova/
       - ^tests/templates/(inventory|globals-default.j2)
+      - ^ansible/roles/loadbalancer/
       - ^tests/test-core-openstack.sh
+      - ^tests/test-proxysql.sh
     vars:
       scenario: cells