Upgrade mariadb storage during upgrade tasks

When a tripleo major upgrade or FFU causes an update or mariadb
to a new major version (e.g. 10.1 -> 10.3), some internal DB
tables must be upgraded (myisam tables), and sometimes the
existing user tables may be migrated to new mariadb defaults.

Move the db-specific upgrade steps into a dedicated script and
make sure that it is called at the right time while upgrading
the undercloud and/or the overcloud.

Closes-Bug: #1913438

Change-Id: I92353622994b28c895d95bdcbe348a73b6c6bb99
(cherry picked from commit 712cfcc71b)
(cherry picked from commit 724d65804d)

container_config_scripts/mysql_upgrade_db.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+# Wait until we know the mysql server is up and responding
+timeout ${DB_MAX_TIMEOUT:-60} /bin/bash -c 'until mysqladmin -uroot ping 2>/dev/null; do sleep 1; done'
+
+# After an upgrade, make sure that the running mysql had a chance to
+# update its data table on disk.
+mysql_upgrade
+
+# Upgrade to 10.3: the default table row format changed from COMPACT
+# to DYNAMIC, so upgrade the existing tables.
+compact_tables=$(mysql -se 'SELECT CONCAT("`",TABLE_SCHEMA,"`.`",TABLE_NAME,"`") FROM information_schema.tables WHERE ENGINE = "InnoDB" and ROW_FORMAT = "Compact";');
+for i in $compact_tables; do echo converting row format of table $i; mysql -e "ALTER TABLE $i ROW_FORMAT=DYNAMIC;"; done;

deployment/database/mysql-base.yaml

@@ -93,6 +93,12 @@ conditions:
   key_size_override_unset: {equals: [{get_param: MysqlCertificateKeySize}, '']}
 
 outputs:
+  container_config_scripts:
+    description: Shared docker config scripts
+    value:
+      mysql_upgrade_db.sh:
+        mode: "0755"
+        content: { get_file: ../../container_config_scripts/mysql_upgrade_db.sh }
   role_data:
     description: Service MySQL using composable services.
     value:

deployment/database/mysql-container-puppet.yaml

@@ -58,6 +58,9 @@ conditions:
 
 resources:
 
+  ContainersCommon:
+    type: ../containers-common.yaml
+
   MysqlBase:
     type: ./mysql-base.yaml
     properties:
@@ -128,6 +131,10 @@ outputs:
             - path: /etc/pki/tls/private/mysql.key
               owner: mysql:mysql
               optional: true
+      container_config_scripts:
+        map_merge:
+          - {get_attr: [ContainersCommon, container_config_scripts]}
+          - {get_attr: [MysqlBase, container_config_scripts]}
       docker_config:
         # Kolla_bootstrap runs before permissions set by kolla_config
         step_1:
@@ -206,6 +213,23 @@ outputs:
             volumes: *mysql_volumes
             environment:
               KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
+          mysql_upgrade_db:
+            # update mysql db on disk after a version upgrade (idempotent)
+            config_volume: mysql
+            start_order: 3
+            detach: false
+            image: *mysql_image
+            volumes:
+              list_concat:
+                - *mysql_volumes
+                - - /var/lib/config-data/puppet-generated/mysql/root:/root:rw
+                  - /var/lib/container-config-scripts/:/container-config-scripts/:ro
+            environment:
+              KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
+            net: host
+            user: root
+            command:
+              - '/container-config-scripts/mysql_upgrade_db.sh'
         step_3:
           # sync credentials config on the running container if it was
           # changed by the docker_puppet_task during step 2
@@ -255,13 +279,12 @@ outputs:
             - {'path': /var/log/containers/mysql, 'setype': 'container_file_t', 'mode': '0750'}
             - {'path': /var/lib/mysql, 'setype': 'container_file_t'}
       upgrade_tasks:
-          # LP 1810136
-          # After upgrade, the new mariadb (e.g. 10.3) might not be able
-          # to replay the redo log of an older one (e.g. 10.1) if mysql
-          # stopped unexpectedly. If the container image differs, force
-          # mysql to stop and clean the redo log here to avoid issue after
-          # upgrade.
-        - name: Stop MySQL server and ensure redo log is cleaned up before upgrade
+          # When mariadb is upgraded to a new major release, one must run
+          # mysql_upgrade to upgrade the DB's system tables, and potentially
+          # run other storage upgrade. We want to that as early as possible
+          # so the database is fully upgraded when services are restarted
+          # during deploy steps.
+        - name: Stop MySQL server and upgrade the database if needed
           when: step|int == 2
           block:
             - name: Get Mysql container image name before upgrade
@@ -282,18 +305,22 @@ outputs:
                   list_join:
                     - ' '
                     - - 'kolla_set_configs;'
-                      - 'mysqld_safe --user=mysql --skip-networking &'
+                      - 'mysqld_safe --user=mysql --skip-networking --log-error=/var/log/mariadb/mariadb-upgrade.log &'
                       - 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
                       - 'mysqladmin shutdown'
             - name: Bind mounts for temporary clean-up container
               set_fact:
-                mysql_clean_up_volumes: *mysql_volumes
+                mysql_upgrade_volumes: *mysql_volumes
             - name: Stop the current mysql container
               systemd:
                 state: stopped
                 name: tripleo_mysql
               when: pre_upgrade_mysql_image != post_upgrade_mysql_image
             - name: Clean up redo log by running a transient mysql server
+              # After upgrade, the new mariadb (e.g. 10.3) might not be able
+              # to replay the redo log of an older one (e.g. 10.1) if mysql
+              # stopped unexpectedly. So run a temporary server to cleanup
+              # the redo log now before upgrade.
               shell:
                 str_replace:
                   template:
@@ -301,10 +328,6 @@ outputs:
                   params:
                     ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
                     IMAGE: "{{ pre_upgrade_mysql_image }}"
-                    VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}"
+                    VOLUMES: "-v {{ mysql_upgrade_volumes | join(' -v ') }}"
                     SCRIPT: "{{ mysql_clean_up_script }}"
               when: pre_upgrade_mysql_image != post_upgrade_mysql_image
-      post_upgrade_tasks:
-        - name: Check and upgrade Mysql database after major version upgrade
-          command: "{{ container_cli }} exec -u root mysql mysql_upgrade"
-          when: step|int == 2

deployment/database/mysql-pacemaker-puppet.yaml

@@ -243,10 +243,10 @@ outputs:
                     - 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "GRANT PROCESS ON *.* TO ''clustercheck''@''localhost'' WITH GRANT OPTION;"'
                     - 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "DELETE FROM mysql.user WHERE user = ''root'' AND host NOT IN (''%'',''localhost'');"'
                     - 'timeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown'
-            volumes: &mysql_volumes
+            volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
-                -
+                - &mysql_volumes
                   - /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z
                   - /var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z
                   - /var/lib/mysql:/var/lib/mysql:rw,z
@@ -599,42 +599,42 @@ outputs:
             - name: Mysql upgrade script
               set_fact:
                 mysql_upgrade_script:
-                  # idempotency: mysql_upgrade leaves a marker file
-                  # in datadir, it does nothing if it has already been
-                  # executed for the current version of MariaDB.
                   list_join:
-                    - ' '
-                    - - 'kolla_set_configs;'
-                      - 'if mysqladmin ping --silent; then exit 0; fi;'
-                      - 'chown -R mysql:mysql /var/lib/mysql;'
-                      - 'chown -R mysql:mysql /var/log/mariadb;'
-                      - 'mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off &'
-                      - 'timeout 60 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
-                      - 'mysql_upgrade;'
-                      - 'mysqladmin shutdown'
+                    - "\n"
+                    - # start a temporary mariadb server for running the upgrade
+                      - |
+                        kolla_set_configs
+                        if mysqladmin ping --silent; then exit 0; fi
+                        chown -R mysql:mysql /var/lib/mysql
+                        chown -R mysql:mysql /var/log/mysql
+                        mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off --log-error=/var/log/mysql/mysqld-upgrade.log &
+                      # an idempotent script takes care of all upgrade steps
+                      # we inline the content here because at the time this is executed,
+                      # the script is not yet copied in /var/lib/container-config-scripts
+                      - { get_file: ../../container_config_scripts/mysql_upgrade_db.sh }
+                      # stop the temporary server
+                      - mysqladmin shutdown
             - name: Bind mounts for temporary container
               set_fact:
-                mysql_upgrade_db_bind_mounts: *mysql_volumes
-            - name: Make sure Mysql upgrade temporary directory exists
-              file:
-                path: /tmp/mariadb-upgrade
-                state: directory
-                owner: root
-                group: root
-                mode: 0700
+                mysql_upgrade_db_bind_mounts:
+                  list_concat:
+                    - {get_attr: [ContainersCommon, volumes]}
+                    - *mysql_volumes
+                    - - /var/log/containers/mysql:/var/log/mysql:rw,z
             - name: Upgrade Mysql database from a temporary container
               shell:
                 str_replace:
                   template:
                     'CONTAINER_CLI run --rm --log-driver=k8s-file --log-opt path=LOG_DIR/db-upgrade.log \
-                    -u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "UPGRADE_SCRIPT"'
+                    -u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "$UPGRADE_SCRIPT"'
                   params:
                     UPGRADE_ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
                     UPGRADE_IMAGE: *mysql_image_pcmklatest
-                    UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | union(['/tmp/mariadb-upgrade:/var/log/mariadb:rw,z']) | join(' -v ')}}"
-                    UPGRADE_SCRIPT: "{{mysql_upgrade_script}}"
+                    UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}}"
                     CONTAINER_CLI: "{{ container_cli }}"
                     LOG_DIR: '/var/log/containers/mysql'
+              environment:
+                    UPGRADE_SCRIPT: "{{ mysql_upgrade_script }}"
 
       external_upgrade_tasks:
         - vars: