From c86526cfbcff33523f2b2f2ef94c317aa48a7ace Mon Sep 17 00:00:00 2001
From: Gage Hugo <gagehugo@gmail.com>
Date: Thu, 25 Jun 2020 18:13:13 -0500
Subject: [PATCH] feat(tls): add tls to mariadb chart

This patch set provides capability to enable TLS termination for the
MariaDB chart. This will be used by the follow on patches in OSH
services patches.

Co-authored-by: Tin Lam <tin@irrational.io>
Co-authored-by: sgupta <sg774j@att.com>
Change-Id: I5ebc8db58c0aa7b4e9eb0b5c671b280250d3cd1f
---
 .../manifests/_job-db-drop-mysql.tpl          | 18 +++++++
 .../manifests/_job-db-init-mysql.tpl          | 17 ++++++
 .../templates/manifests/_job-db-sync.tpl      |  7 +++
 .../templates/scripts/_db-drop.py.tpl         | 23 +++++++-
 .../templates/scripts/_db-init.py.tpl         | 27 ++++++++--
 mariadb/templates/bin/_readiness.sh.tpl       | 27 ++++++----
 mariadb/templates/bin/_start.py.tpl           | 54 ++++++++++---------
 mariadb/templates/certificates.yaml           | 17 ++++++
 mariadb/templates/statefulset.yaml            |  6 +++
 mariadb/values.yaml                           | 25 ++++++++-
 mariadb/values_overrides/tls.yaml             | 23 ++++++++
 11 files changed, 204 insertions(+), 40 deletions(-)
 create mode 100644 mariadb/templates/certificates.yaml
 create mode 100644 mariadb/values_overrides/tls.yaml

diff --git a/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
index 1b639f03c..265a4ba9c 100644
--- a/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
+++ b/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl
@@ -34,6 +34,9 @@ limitations under the License.
 {{- $backoffLimit := index . "backoffLimit" | default "1000" -}}
 {{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}}
 {{- $serviceNamePretty := $serviceName | replace "_" "-" -}}
+{{- $tlsPath := index . "tlsPath" | default (printf "/etc/%s/certs" $serviceNamePretty ) -}}
+{{- $tlsSecret := index . "tlsSecret" | default "" -}}
+{{- $dbAdminTlsSecret := index . "dbAdminTlsSecret" | default "" -}}
 
 {{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "db-drop" }}
 {{ tuple $envAll "db_drop" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -82,6 +85,12 @@ spec:
             - name: OPENSTACK_CONFIG_DB_KEY
               value: {{ $dbToDrop.configDbKey | quote }}
 {{- end }}
+{{- if $envAll.Values.manifests.certificates }}
+            - name: MARIADB_X509
+              value: "REQUIRE X509"
+            - name: USER_CERT_PATH
+              value: {{ $tlsPath | quote }}
+{{- end }}
 {{- if eq $dbToDropType "secret" }}
             - name: DB_CONNECTION
               valueFrom:
@@ -98,6 +107,7 @@ spec:
               mountPath: /tmp/db-drop.py
               subPath: db-drop.py
               readOnly: true
+
 {{- if eq $dbToDropType "oslo" }}
             - name: etc-service
               mountPath: {{ dir $dbToDrop.configFile | quote }}
@@ -110,6 +120,10 @@ spec:
               subPath: {{ base $dbToDrop.logConfigFile | quote }}
               readOnly: true
 {{- end }}
+{{- if $envAll.Values.manifests.certificates }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret "path" $tlsPath | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- end }}
 {{- end }}
       volumes:
         - name: pod-tmp
@@ -124,6 +138,10 @@ spec:
             name: {{ $configMapBin | quote }}
             defaultMode: 0555
 {{- end }}
+{{- if $envAll.Values.manifests.certificates }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- end }}
 {{- $local := dict "configMapBinFirst" true -}}
 {{- range $key1, $dbToDrop := $dbsToDrop }}
 {{- $dbToDropType := default "oslo" $dbToDrop.inputType }}
diff --git a/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
index 73ac04d26..3f72f3335 100644
--- a/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
+++ b/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl
@@ -34,6 +34,9 @@ limitations under the License.
 {{- $backoffLimit := index . "backoffLimit" | default "1000" -}}
 {{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}}
 {{- $serviceNamePretty := $serviceName | replace "_" "-" -}}
+{{- $tlsPath := index . "tlsPath" | default (printf "/etc/%s/certs" $serviceNamePretty ) -}}
+{{- $tlsSecret := index . "tlsSecret" | default "" -}}
+{{- $dbAdminTlsSecret := index . "dbAdminTlsSecret" | default "" -}}
 
 {{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "db-init" }}
 {{ tuple $envAll "db_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -87,6 +90,12 @@ spec:
                 secretKeyRef:
                   name: {{ $dbToInit.userSecret | quote }}
                   key: DB_CONNECTION
+{{- end }}
+{{- if $envAll.Values.manifests.certificates }}
+            - name: MARIADB_X509
+              value: "REQUIRE X509"
+            - name: USER_CERT_PATH
+              value: {{ $tlsPath | quote }}
 {{- end }}
           command:
             - /tmp/db-init.py
@@ -109,6 +118,10 @@ spec:
               subPath: {{ base $dbToInit.logConfigFile | quote }}
               readOnly: true
 {{- end }}
+{{- if $envAll.Values.manifests.certificates }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret "path" $tlsPath | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- end }}
 {{- end }}
       volumes:
         - name: pod-tmp
@@ -123,6 +136,10 @@ spec:
             name: {{ $configMapBin | quote }}
             defaultMode: 0555
 {{- end }}
+{{- if $envAll.Values.manifests.certificates }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- end }}
 {{- $local := dict "configMapBinFirst" true -}}
 {{- range $key1, $dbToInit := $dbsToInit }}
 {{- $dbToInitType := default "oslo" $dbToInit.inputType }}
diff --git a/helm-toolkit/templates/manifests/_job-db-sync.tpl b/helm-toolkit/templates/manifests/_job-db-sync.tpl
index 0e4e3ad83..1352293b5 100644
--- a/helm-toolkit/templates/manifests/_job-db-sync.tpl
+++ b/helm-toolkit/templates/manifests/_job-db-sync.tpl
@@ -31,6 +31,9 @@ limitations under the License.
 {{- $backoffLimit := index . "backoffLimit" | default "1000" -}}
 {{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}}
 {{- $serviceNamePretty := $serviceName | replace "_" "-" -}}
+{{- $tlsPath := index . "tlsPath" | default (printf "/etc/%s/certs" $serviceNamePretty ) -}}
+{{- $tlsSecret := index . "tlsSecret" | default "" -}}
+{{- $dbAdminTlsSecret := index . "dbAdminTlsSecret" | default "" -}}
 
 {{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "db-sync" }}
 {{ tuple $envAll "db_sync" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -87,6 +90,8 @@ spec:
               mountPath: {{ $dbToSync.logConfigFile | quote }}
               subPath: {{ base $dbToSync.logConfigFile | quote }}
               readOnly: true
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret "path" $tlsPath | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{- if $podVolMounts }}
 {{ $podVolMounts | toYaml | indent 12 }}
 {{- end }}
@@ -109,6 +114,8 @@ spec:
           secret:
             secretName: {{ $configMapEtc | quote }}
             defaultMode: 0444
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{- if $podVols }}
 {{ $podVols | toYaml | indent 8 }}
 {{- end }}
diff --git a/helm-toolkit/templates/scripts/_db-drop.py.tpl b/helm-toolkit/templates/scripts/_db-drop.py.tpl
index 814475467..322932eb1 100644
--- a/helm-toolkit/templates/scripts/_db-drop.py.tpl
+++ b/helm-toolkit/templates/scripts/_db-drop.py.tpl
@@ -54,6 +54,13 @@ else:
     logger.critical('environment variable ROOT_DB_CONNECTION not set')
     sys.exit(1)
 
+mysql_x509 = os.getenv('MARIADB_X509', "")
+if mysql_x509:
+    user_tls_cert_path = os.getenv('USER_CERT_PATH', "")
+    if not user_tls_cert_path:
+        logger.critical('environment variable USER_CERT_PATH not set')
+        sys.exit(1)
+
 # Get the connection string for the service db
 if "OPENSTACK_CONFIG_FILE" in os.environ:
     os_conf = os.environ['OPENSTACK_CONFIG_FILE']
@@ -94,7 +101,13 @@ try:
     host = root_engine_full.url.host
     port = root_engine_full.url.port
     root_engine_url = ''.join([drivername, '://', root_user, ':', root_password, '@', host, ':', str (port)])
-    root_engine = create_engine(root_engine_url)
+    if mysql_x509:
+        ssl_args = {'ssl': {'ca': '/etc/mysql/certs/ca.crt',
+                            'key': '/etc/mysql/certs/tls.key',
+                            'cert': '/etc/mysql/certs/tls.crt'}}
+        root_engine = create_engine(root_engine_url, connect_args=ssl_args)
+    else:
+        root_engine = create_engine(root_engine_url)
     connection = root_engine.connect()
     connection.close()
     logger.info("Tested connection to DB @ {0}:{1} as {2}".format(
@@ -105,7 +118,13 @@ except:
 
 # User DB engine
 try:
-    user_engine = create_engine(user_db_conn)
+    if mysql_x509:
+        ssl_args = {'ssl': {'ca': '{0}/ca.crt'.format(user_tls_cert_path),
+                            'key': '{0}/tls.key'.format(user_tls_cert_path),
+                            'cert': '{0}/tls.crt'.format(user_tls_cert_path)}}
+        user_engine = create_engine(user_db_conn, connect_args=ssl_args)
+    else:
+        user_engine = create_engine(user_db_conn)
     # Get our user data out of the user_engine
     database = user_engine.url.database
     user = user_engine.url.username
diff --git a/helm-toolkit/templates/scripts/_db-init.py.tpl b/helm-toolkit/templates/scripts/_db-init.py.tpl
index c620a8d27..d0bda49a6 100644
--- a/helm-toolkit/templates/scripts/_db-init.py.tpl
+++ b/helm-toolkit/templates/scripts/_db-init.py.tpl
@@ -54,6 +54,13 @@ else:
     logger.critical('environment variable ROOT_DB_CONNECTION not set')
     sys.exit(1)
 
+mysql_x509 = os.getenv('MARIADB_X509', "")
+if mysql_x509:
+    user_tls_cert_path = os.getenv('USER_CERT_PATH', "")
+    if not user_tls_cert_path:
+        logger.critical('environment variable USER_CERT_PATH not set')
+        sys.exit(1)
+
 # Get the connection string for the service db
 if "OPENSTACK_CONFIG_FILE" in os.environ:
     os_conf = os.environ['OPENSTACK_CONFIG_FILE']
@@ -94,7 +101,13 @@ try:
     host = root_engine_full.url.host
     port = root_engine_full.url.port
     root_engine_url = ''.join([drivername, '://', root_user, ':', root_password, '@', host, ':', str (port)])
-    root_engine = create_engine(root_engine_url)
+    if mysql_x509:
+        ssl_args = {'ssl': {'ca': '/etc/mysql/certs/ca.crt',
+                    'key': '/etc/mysql/certs/tls.key',
+                    'cert': '/etc/mysql/certs/tls.crt'}}
+        root_engine = create_engine(root_engine_url, connect_args=ssl_args)
+    else:
+        root_engine = create_engine(root_engine_url)
     connection = root_engine.connect()
     connection.close()
     logger.info("Tested connection to DB @ {0}:{1} as {2}".format(
@@ -105,7 +118,13 @@ except:
 
 # User DB engine
 try:
-    user_engine = create_engine(user_db_conn)
+    if mysql_x509:
+        ssl_args = {'ssl': {'ca': '{0}/ca.crt'.format(user_tls_cert_path),
+                            'key': '{0}/tls.key'.format(user_tls_cert_path),
+                            'cert': '{0}/tls.crt'.format(user_tls_cert_path)}}
+        user_engine = create_engine(user_db_conn, connect_args=ssl_args)
+    else:
+        user_engine = create_engine(user_db_conn)
     # Get our user data out of the user_engine
     database = user_engine.url.database
     user = user_engine.url.username
@@ -126,8 +145,8 @@ except:
 # Create DB User
 try:
     root_engine.execute(
-        "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\' IDENTIFIED BY \'{2}\'".format(
-            database, user, password))
+        "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\' IDENTIFIED BY \'{2}\' {3}".format(
+            database, user, password, mysql_x509))
     logger.info("Created user {0} for {1}".format(user, database))
 except:
     logger.critical("Could not create user {0} for {1}".format(user, database))
diff --git a/mariadb/templates/bin/_readiness.sh.tpl b/mariadb/templates/bin/_readiness.sh.tpl
index b8fac01ec..fae9172c6 100644
--- a/mariadb/templates/bin/_readiness.sh.tpl
+++ b/mariadb/templates/bin/_readiness.sh.tpl
@@ -19,6 +19,12 @@ set -e
 MYSQL="mysql \
   --defaults-file=/etc/mysql/admin_user.cnf \
   --host=localhost \
+{{- if .Values.manifests.certificates }}
+  --ssl-verify-server-cert=false \
+  --ssl-ca=/etc/mysql/certs/ca.crt \
+  --ssl-key=/etc/mysql/certs/tls.key \
+  --ssl-cert=/etc/mysql/certs/tls.crt \
+{{- end }}
   --connect-timeout 2"
 
 mysql_status_query () {
@@ -28,22 +34,25 @@ mysql_status_query () {
 }
 
 if ! $MYSQL -e 'select 1' > /dev/null 2>&1 ; then
-    exit 1
+  exit 1
 fi
 
 if [ "x$(mysql_status_query wsrep_ready)" != "xON" ]; then
-    # WSREP says the node can receive queries
-    exit 1
+  # WSREP says the node can receive queries
+  exit 1
 fi
+
 if [ "x$(mysql_status_query wsrep_connected)" != "xON" ]; then
-    # WSREP connected
-    exit 1
+  # WSREP connected
+  exit 1
 fi
+
 if [ "x$(mysql_status_query wsrep_cluster_status)" != "xPrimary" ]; then
-    # Not in primary cluster
-    exit 1
+  # Not in primary cluster
+  exit 1
 fi
+
 if [ "x$(mysql_status_query wsrep_local_state_comment)" != "xSynced" ]; then
-    # WSREP not synced
-    exit 1
+  # WSREP not synced
+  exit 1
 fi
diff --git a/mariadb/templates/bin/_start.py.tpl b/mariadb/templates/bin/_start.py.tpl
index 1275fb5f5..6b65e5c83 100644
--- a/mariadb/templates/bin/_start.py.tpl
+++ b/mariadb/templates/bin/_start.py.tpl
@@ -104,6 +104,8 @@ else:
 if check_env_var("MYSQL_DBAUDIT_PASSWORD"):
     mysql_dbaudit_password = os.environ['MYSQL_DBAUDIT_PASSWORD']
 
+mysql_x509 = os.getenv('MARIADB_X509', "")
+
 if mysql_dbadmin_username == mysql_dbsst_username:
     logger.critical(
         "The dbadmin username should not match the sst user username")
@@ -270,33 +272,35 @@ def mysqld_bootstrap():
                 # is locked and cannot login
                 "DELETE FROM mysql.user WHERE user != 'mariadb.sys' ;\n"  # nosec
                 "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n"
-                "GRANT ALL ON *.* TO '{0}'@'%' WITH GRANT OPTION ;\n"
+                "GRANT ALL ON *.* TO '{0}'@'%' {4} WITH GRANT OPTION; \n"
                 "DROP DATABASE IF EXISTS test ;\n"
-                "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}' ;\n"
-                "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n"
+                "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}';\n"
+                "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1';\n"
                 "FLUSH PRIVILEGES ;\n"
                 "SHUTDOWN ;".format(mysql_dbadmin_username, mysql_dbadmin_password,
-                                    mysql_dbsst_username, mysql_dbsst_password))
+                                    mysql_dbsst_username, mysql_dbsst_password,
+                                    mysql_x509))
         else:
             template = (
                 "DELETE FROM mysql.user WHERE user != 'mariadb.sys' ;\n"  # nosec
                 "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n"
-                "GRANT ALL ON *.* TO '{0}'@'%' WITH GRANT OPTION ;\n"
+                "GRANT ALL ON *.* TO '{0}'@'%' {6} WITH GRANT OPTION;\n"
                 "DROP DATABASE IF EXISTS test ;\n"
-                "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}' ;\n"
+                "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}';\n"
                 "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n"
-                "CREATE OR REPLACE USER '{4}'@'%' IDENTIFIED BY '{5}' ;\n"
-                "GRANT SELECT ON *.* TO '{4}'@'%' ;\n"
+                "CREATE OR REPLACE USER '{4}'@'%' IDENTIFIED BY '{5}';\n"
+                "GRANT SELECT ON *.* TO '{4}'@'%' {6};\n"
                 "FLUSH PRIVILEGES ;\n"
                 "SHUTDOWN ;".format(mysql_dbadmin_username, mysql_dbadmin_password,
                                     mysql_dbsst_username, mysql_dbsst_password,
-                                    mysql_dbaudit_username, mysql_dbaudit_password))
+                                    mysql_dbaudit_username, mysql_dbaudit_password,
+                                    mysql_x509))
         bootstrap_sql_file = tempfile.NamedTemporaryFile(suffix='.sql').name
         with open(bootstrap_sql_file, 'w') as f:
             f.write(template)
             f.close()
         run_cmd_with_logging([
-            'mysqld', '--bind-address=127.0.0.1',
+            'mysqld', '--user=mysql', '--bind-address=127.0.0.1',
             '--wsrep_cluster_address=gcomm://',
             "--init-file={0}".format(bootstrap_sql_file)
         ], logger)
@@ -780,7 +784,7 @@ def run_mysqld(cluster='existing'):
     mysqld_write_cluster_conf(mode='run')
     launch_leader_election()
     launch_cluster_monitor()
-    mysqld_cmd = ['mysqld']
+    mysqld_cmd = ['mysqld', '--user=mysql']
     if cluster == 'new':
         mysqld_cmd.append('--wsrep-new-cluster')
 
@@ -791,24 +795,26 @@ def run_mysqld(cluster='existing'):
         if not mysql_dbaudit_username:
             template = (
                 "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n"
-                "GRANT ALL ON *.* TO '{0}'@'%' WITH GRANT OPTION ;\n"
+                "GRANT ALL ON *.* TO '{0}'@'%' {4} WITH GRANT OPTION ;\n"
                 "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}' ;\n"
                 "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n"
                 "FLUSH PRIVILEGES ;\n"
-                "SHUTDOWN ;".format(mysql_dbadmin_username, mysql_dbadmin_password,
-                                    mysql_dbsst_username, mysql_dbsst_password))
-        else:
-            template = (
-                "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n"
-                "GRANT ALL ON *.* TO '{0}'@'%' WITH GRANT OPTION ;\n"
-                "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}' ;\n"
-                "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n"
-                "CREATE OR REPLACE USER '{4}'@'%' IDENTIFIED BY '{5}' ;\n"
-                "GRANT SELECT ON *.* TO '{4}'@'%' ;\n"
-                "FLUSH PRIVILEGES ;\n"
                 "SHUTDOWN ;".format(mysql_dbadmin_username, mysql_dbadmin_password,
                                     mysql_dbsst_username, mysql_dbsst_password,
-                                    mysql_dbaudit_username, mysql_dbaudit_password))
+                                    mysql_x509))
+        else:
+            template = (
+                "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n"
+                "GRANT ALL ON *.* TO '{0}'@'%' {6} WITH GRANT OPTION ;\n"
+                "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}' ;\n"
+                "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n"
+                "CREATE OR REPLACE USER '{4}'@'%' IDENTIFIED BY '{5}' ;\n"
+                "GRANT SELECT ON *.* TO '{4}'@'%' {6};\n"
+                "FLUSH PRIVILEGES ;\n"
+                "SHUTDOWN ;".format(mysql_dbadmin_username, mysql_dbadmin_password,
+                                    mysql_dbsst_username, mysql_dbsst_password,
+                                    mysql_dbaudit_username, mysql_dbaudit_password,
+                                    mysql_x509))
         bootstrap_sql_file = tempfile.NamedTemporaryFile(suffix='.sql').name
         with open(bootstrap_sql_file, 'w') as f:
             f.write(template)
diff --git a/mariadb/templates/certificates.yaml b/mariadb/templates/certificates.yaml
new file mode 100644
index 000000000..200f974ac
--- /dev/null
+++ b/mariadb/templates/certificates.yaml
@@ -0,0 +1,17 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.certificates -}}
+{{ dict "envAll" . "service" "oslo_db" "type" "default" | include "helm-toolkit.manifests.certificates" }}
+{{- end -}}
diff --git a/mariadb/templates/statefulset.yaml b/mariadb/templates/statefulset.yaml
index 70255b597..7ccc219bf 100644
--- a/mariadb/templates/statefulset.yaml
+++ b/mariadb/templates/statefulset.yaml
@@ -136,6 +136,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            {{- if $envAll.Values.manifests.certificates }}
+            - name: MARIADB_X509
+              value: "REQUIRE X509"
+            {{- end }}
             - name: MARIADB_REPLICAS
               value: {{ .Values.pod.replicas.server | quote }}
             - name: POD_NAME_PREFIX
@@ -229,6 +233,7 @@ spec:
               readOnly: true
             - name: mysql-data
               mountPath: /var/lib/mysql
+{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -248,6 +253,7 @@ spec:
           secret:
             secretName: mariadb-secrets
             defaultMode: 0444
+{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
         {{- if not .Values.volume.enabled }}
         - name: mysql-data
         {{- if .Values.volume.use_local_path_for_single_pod_cluster.enabled }}
diff --git a/mariadb/values.yaml b/mariadb/values.yaml
index cabf6d136..18de2ee5f 100644
--- a/mariadb/values.yaml
+++ b/mariadb/values.yaml
@@ -20,7 +20,6 @@ release_group: null
 
 images:
   tags:
-    # 10.2.31
     mariadb: openstackhelm/mariadb@sha256:5f05ce5dce71c835c6361a05705da5cce31114934689ec87dfa48b8f8c600f70
     ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0
     error_pages: gcr.io/google_containers/defaultbackend:1.4
@@ -416,6 +415,15 @@ conf:
       wsrep_sst_auth={{ .Values.endpoints.oslo_db.auth.sst.username }}:{{ .Values.endpoints.oslo_db.auth.sst.password }}
       wsrep_sst_method=mariabackup
 
+      {{ if .Values.manifests.certificates }}
+      # TLS
+      ssl_ca=/etc/mysql/certs/ca.crt
+      ssl_key=/etc/mysql/certs/tls.key
+      ssl_cert=/etc/mysql/certs/tls.crt
+      # tls_version = TLSv1.2,TLSv1.3
+      {{ end }}
+
+
       [mysqldump]
       max-allowed-packet=16M
 
@@ -423,6 +431,15 @@ conf:
       default_character_set=utf8
       protocol=tcp
       port={{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+      {{ if .Values.manifests.certificates }}
+      # TLS
+      ssl_ca=/etc/mysql/certs/ca.crt
+      ssl_key=/etc/mysql/certs/tls.key
+      ssl_cert=/etc/mysql/certs/tls.crt
+      # tls_version = TLSv1.2,TLSv1.3
+      ssl-verify-server-cert
+      {{ end }}
+
     config_override: null
     # Any configuration here will override the base config.
     # config_override: |-
@@ -445,6 +462,11 @@ secrets:
     remote_rgw_user: mariadb-backup-user
   mariadb:
     backup_restore: mariadb-backup-restore
+  tls:
+    oslo_db:
+      server:
+        public: mariadb-tls-server
+        internal: mariadb-tls-direct
 
 # typically overridden by environmental
 # values, but should include all endpoints
@@ -589,6 +611,7 @@ network_policy:
       - {}
 
 manifests:
+  certificates: false
   configmap_bin: true
   configmap_etc: true
   configmap_ingress_conf: true
diff --git a/mariadb/values_overrides/tls.yaml b/mariadb/values_overrides/tls.yaml
new file mode 100644
index 000000000..f89d5e94b
--- /dev/null
+++ b/mariadb/values_overrides/tls.yaml
@@ -0,0 +1,23 @@
+---
+pod:
+  security_context:
+    server:
+      container:
+        perms:
+          readOnlyRootFilesystem: false
+        mariadb:
+          runAsUser: 0
+          allowPrivilegeEscalation: true
+          readOnlyRootFilesystem: false
+endpoints:
+  oslo_db:
+    host_fqdn_override:
+      default:
+        tls:
+          secretName: mariadb-tls-direct
+          issuerRef:
+            name: ca-issuer
+            kind: Issuer
+manifests:
+  certificates: true
+...