From ad3516951017d742aabd2771f69ffdfd9554196c Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Thu, 17 Dec 2020 13:23:58 +0100 Subject: [PATCH] WIP upgrade mariadb storage during upgrade tasks Change-Id: I92353622994b28c895d95bdcbe348a73b6c6bb99 --- .../database/mysql-container-puppet.yaml | 63 +++++++++++++++---- .../database/mysql-pacemaker-puppet.yaml | 2 + 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/deployment/database/mysql-container-puppet.yaml b/deployment/database/mysql-container-puppet.yaml index 4fa6fdff9e..9db650d0f5 100644 --- a/deployment/database/mysql-container-puppet.yaml +++ b/deployment/database/mysql-container-puppet.yaml @@ -255,13 +255,13 @@ outputs: - {'path': /var/log/containers/mysql, 'setype': 'container_file_t', 'mode': '0750'} - {'path': /var/lib/mysql, 'setype': 'container_file_t'} upgrade_tasks: - # LP 1810136 - # After upgrade, the new mariadb (e.g. 10.3) might not be able - # to replay the redo log of an older one (e.g. 10.1) if mysql - # stopped unexpectedly. If the container image differs, force - # mysql to stop and clean the redo log here to avoid issue after - # upgrade. - - name: Stop MySQL server and ensure redo log is cleaned up before upgrade + # When mariadb is upgraded to a new major release, one must run + # mysql_upgrade to upgrade the DB's system tables, and potentially + # run other storage upgrade. We want to that as early as possible + # so the database is fully upgraded when services are restarted + # during deploy steps. + # restarted, so the during deploy steps. + - name: Stop MySQL server and upgrade the database if needed when: step|int == 2 block: - name: Get Mysql container image name before upgrade @@ -285,15 +285,40 @@ outputs: - 'mysqld_safe --user=mysql --skip-networking &' - 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';' - 'mysqladmin shutdown' + - name: Mysql upgrade script + set_fact: + # The purpose of this script is to run mysql_upgrade + # and upgrade the database storage format if required + mysql_upgrade_script: + list_join: + - ' ' + - - 'kolla_set_configs;' + - 'mysqld_safe --user=mysql --skip-networking &' + - 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';' + - 'mysql_upgrade;' + - 'compact_tables=\$(mysql -se ''SELECT CONCAT(\"`\",TABLE_SCHEMA,\"`.`\",TABLE_NAME,\"`\") FROM information_schema.tables WHERE ENGINE = \"InnoDB\" and ROW_FORMAT = \"Compact\";'');' + - 'for i in \$compact_tables; do echo converting row format of table \$i; echo mysql -e \"ALTER TABLE \$i ROW_FORMAT=DYNAMIC;\"; done;' + - 'mysqladmin shutdown' - name: Bind mounts for temporary clean-up container set_fact: - mysql_clean_up_volumes: *mysql_volumes + mysql_upgrade_volumes: *mysql_volumes + - name: Make sure Mysql upgrade temporary directory exists + file: + path: /tmp/mariadb-upgrade + state: directory + owner: root + group: root + mode: 0700 - name: Stop the current mysql container systemd: state: stopped name: tripleo_mysql when: pre_upgrade_mysql_image != post_upgrade_mysql_image - name: Clean up redo log by running a transient mysql server + # After upgrade, the new mariadb (e.g. 10.3) might not be able + # to replay the redo log of an older one (e.g. 10.1) if mysql + # stopped unexpectedly. So run a temporary server to cleanup + # the redo now before upgrade. shell: str_replace: template: @@ -301,10 +326,22 @@ outputs: params: ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"' IMAGE: "{{ pre_upgrade_mysql_image }}" - VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}" + VOLUMES: "-v {{ mysql_upgrade_volumes | join(' -v ') }}" SCRIPT: "{{ mysql_clean_up_script }}" when: pre_upgrade_mysql_image != post_upgrade_mysql_image - post_upgrade_tasks: - - name: Check and upgrade Mysql database after major version upgrade - command: "{{ container_cli }} exec -u root mysql mysql_upgrade" - when: step|int == 2 + - name: Upgrade Mysql database from a temporary container + # Moving from 10.1 to 10.3, InnoDB's default row storage format + # changes, so the existing OpenStack tables have to be migrated + # to the new format before services are restarted. + shell: + str_replace: + template: + '{{ container_cli }} run --rm --log-driver=k8s-file --log-opt path=LOG_DIR/db-upgrade.log \ + -u root --net=host ENV VOLUMES "IMAGE" /bin/bash -ecx "SCRIPT"' + params: + ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"' + IMAGE: "{{ post_upgrade_mysql_image }}" + VOLUMES: "-v {{ mysql_upgrade_volumes | union(['/tmp/mariadb-upgrade:/var/log/mariadb:rw,z']) | join(' -v ')}}" + SCRIPT: "{{mysql_upgrade_script}}" + LOG_DIR: '/var/log/containers/mysql' + when: pre_upgrade_mysql_image != post_upgrade_mysql_image diff --git a/deployment/database/mysql-pacemaker-puppet.yaml b/deployment/database/mysql-pacemaker-puppet.yaml index e14ba73e7a..4dfa2c2ed9 100644 --- a/deployment/database/mysql-pacemaker-puppet.yaml +++ b/deployment/database/mysql-pacemaker-puppet.yaml @@ -621,6 +621,8 @@ outputs: - 'mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off &' - 'timeout 60 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';' - 'mysql_upgrade;' + - 'compact_tables=\$(mysql -se ''SELECT CONCAT(\"`\",TABLE_SCHEMA,\"`.`\",TABLE_NAME,\"`\") FROM information_schema.tables WHERE ENGINE = \"InnoDB\" and ROW_FORMAT = \"Compact\";'');' + - 'for i in \$compact_tables; do echo converting row format of table \$i; echo mysql -e \"ALTER TABLE \$i ROW_FORMAT=DYNAMIC;\"; done;' - 'mysqladmin shutdown' - name: Bind mounts for temporary container set_fact: