diff --git a/container_config_scripts/mysql_upgrade_db.sh b/container_config_scripts/mysql_upgrade_db.sh new file mode 100755 index 0000000000..610bf1307a --- /dev/null +++ b/container_config_scripts/mysql_upgrade_db.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Wait until we know the mysql server is up and responding +timeout ${DB_MAX_TIMEOUT:-60} /bin/bash -c 'until mysqladmin -uroot ping 2>/dev/null; do sleep 1; done' + +# After an upgrade, make sure that the running mysql had a chance to +# update its data table on disk. +mysql_upgrade + +# Upgrade to 10.3: the default table row format changed from COMPACT +# to DYNAMIC, so upgrade the existing tables. +compact_tables=$(mysql -se 'SELECT CONCAT("`",TABLE_SCHEMA,"`.`",TABLE_NAME,"`") FROM information_schema.tables WHERE ENGINE = "InnoDB" and ROW_FORMAT = "Compact";'); +for i in $compact_tables; do echo converting row format of table $i; mysql -e "ALTER TABLE $i ROW_FORMAT=DYNAMIC;"; done; diff --git a/deployment/database/mysql-base.yaml b/deployment/database/mysql-base.yaml index e98216f47c..6899fb2cf5 100644 --- a/deployment/database/mysql-base.yaml +++ b/deployment/database/mysql-base.yaml @@ -99,6 +99,12 @@ conditions: key_size_override_unset: {equals: [{get_param: MysqlCertificateKeySize}, '']} outputs: + container_config_scripts: + description: Shared docker config scripts + value: + mysql_upgrade_db.sh: + mode: "0755" + content: { get_file: ../../container_config_scripts/mysql_upgrade_db.sh } role_data: description: Service MySQL using composable services. value: diff --git a/deployment/database/mysql-container-puppet.yaml b/deployment/database/mysql-container-puppet.yaml index 4fa6fdff9e..aa22f96917 100644 --- a/deployment/database/mysql-container-puppet.yaml +++ b/deployment/database/mysql-container-puppet.yaml @@ -58,6 +58,9 @@ conditions: resources: + ContainersCommon: + type: ../containers-common.yaml + MysqlBase: type: ./mysql-base.yaml properties: @@ -128,6 +131,10 @@ outputs: - path: /etc/pki/tls/private/mysql.key owner: mysql:mysql optional: true + container_config_scripts: + map_merge: + - {get_attr: [ContainersCommon, container_config_scripts]} + - {get_attr: [MysqlBase, container_config_scripts]} docker_config: # Kolla_bootstrap runs before permissions set by kolla_config step_1: @@ -206,6 +213,23 @@ outputs: volumes: *mysql_volumes environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS + mysql_upgrade_db: + # update mysql db on disk after a version upgrade (idempotent) + config_volume: mysql + start_order: 3 + detach: false + image: *mysql_image + volumes: + list_concat: + - *mysql_volumes + - - /var/lib/config-data/puppet-generated/mysql/root:/root:rw + - /var/lib/container-config-scripts/:/container-config-scripts/:ro + environment: + KOLLA_CONFIG_STRATEGY: COPY_ALWAYS + net: host + user: root + command: + - '/container-config-scripts/mysql_upgrade_db.sh' step_3: # sync credentials config on the running container if it was # changed by the docker_puppet_task during step 2 @@ -255,13 +279,12 @@ outputs: - {'path': /var/log/containers/mysql, 'setype': 'container_file_t', 'mode': '0750'} - {'path': /var/lib/mysql, 'setype': 'container_file_t'} upgrade_tasks: - # LP 1810136 - # After upgrade, the new mariadb (e.g. 10.3) might not be able - # to replay the redo log of an older one (e.g. 10.1) if mysql - # stopped unexpectedly. If the container image differs, force - # mysql to stop and clean the redo log here to avoid issue after - # upgrade. - - name: Stop MySQL server and ensure redo log is cleaned up before upgrade + # When mariadb is upgraded to a new major release, one must run + # mysql_upgrade to upgrade the DB's system tables, and potentially + # run other storage upgrade. We want to that as early as possible + # so the database is fully upgraded when services are restarted + # during deploy steps. + - name: Stop MySQL server and upgrade the database if needed when: step|int == 2 block: - name: Get Mysql container image name before upgrade @@ -282,18 +305,22 @@ outputs: list_join: - ' ' - - 'kolla_set_configs;' - - 'mysqld_safe --user=mysql --skip-networking &' + - 'mysqld_safe --user=mysql --skip-networking --log-error=/var/log/mariadb/mariadb-upgrade.log &' - 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';' - 'mysqladmin shutdown' - name: Bind mounts for temporary clean-up container set_fact: - mysql_clean_up_volumes: *mysql_volumes + mysql_upgrade_volumes: *mysql_volumes - name: Stop the current mysql container systemd: state: stopped name: tripleo_mysql when: pre_upgrade_mysql_image != post_upgrade_mysql_image - name: Clean up redo log by running a transient mysql server + # After upgrade, the new mariadb (e.g. 10.3) might not be able + # to replay the redo log of an older one (e.g. 10.1) if mysql + # stopped unexpectedly. So run a temporary server to cleanup + # the redo log now before upgrade. shell: str_replace: template: @@ -301,10 +328,6 @@ outputs: params: ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"' IMAGE: "{{ pre_upgrade_mysql_image }}" - VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}" + VOLUMES: "-v {{ mysql_upgrade_volumes | join(' -v ') }}" SCRIPT: "{{ mysql_clean_up_script }}" when: pre_upgrade_mysql_image != post_upgrade_mysql_image - post_upgrade_tasks: - - name: Check and upgrade Mysql database after major version upgrade - command: "{{ container_cli }} exec -u root mysql mysql_upgrade" - when: step|int == 2 diff --git a/deployment/database/mysql-pacemaker-puppet.yaml b/deployment/database/mysql-pacemaker-puppet.yaml index e14ba73e7a..12b111953c 100644 --- a/deployment/database/mysql-pacemaker-puppet.yaml +++ b/deployment/database/mysql-pacemaker-puppet.yaml @@ -253,10 +253,10 @@ outputs: - 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "GRANT PROCESS ON *.* TO ''clustercheck''@''localhost'' WITH GRANT OPTION;"' - 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "DELETE FROM mysql.user WHERE user = ''root'' AND host NOT IN (''%'',''localhost'');"' - 'timeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown' - volumes: &mysql_volumes + volumes: list_concat: - {get_attr: [ContainersCommon, volumes]} - - + - &mysql_volumes - /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z - /var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z - /var/lib/mysql:/var/lib/mysql:rw,z @@ -609,42 +609,42 @@ outputs: - name: Mysql upgrade script set_fact: mysql_upgrade_script: - # idempotency: mysql_upgrade leaves a marker file - # in datadir, it does nothing if it has already been - # executed for the current version of MariaDB. list_join: - - ' ' - - - 'kolla_set_configs;' - - 'if mysqladmin ping --silent; then exit 0; fi;' - - 'chown -R mysql:mysql /var/lib/mysql;' - - 'chown -R mysql:mysql /var/log/mariadb;' - - 'mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off &' - - 'timeout 60 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';' - - 'mysql_upgrade;' - - 'mysqladmin shutdown' + - "\n" + - # start a temporary mariadb server for running the upgrade + - | + kolla_set_configs + if mysqladmin ping --silent; then exit 0; fi + chown -R mysql:mysql /var/lib/mysql + chown -R mysql:mysql /var/log/mysql + mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off --log-error=/var/log/mysql/mysqld-upgrade.log & + # an idempotent script takes care of all upgrade steps + # we inline the content here because at the time this is executed, + # the script is not yet copied in /var/lib/container-config-scripts + - { get_file: ../../container_config_scripts/mysql_upgrade_db.sh } + # stop the temporary server + - mysqladmin shutdown - name: Bind mounts for temporary container set_fact: - mysql_upgrade_db_bind_mounts: *mysql_volumes - - name: Make sure Mysql upgrade temporary directory exists - file: - path: /tmp/mariadb-upgrade - state: directory - owner: root - group: root - mode: 0700 + mysql_upgrade_db_bind_mounts: + list_concat: + - {get_attr: [ContainersCommon, volumes]} + - *mysql_volumes + - - /var/log/containers/mysql:/var/log/mysql:rw,z - name: Upgrade Mysql database from a temporary container shell: str_replace: template: 'CONTAINER_CLI run --rm --log-driver=k8s-file --log-opt path=LOG_DIR/db-upgrade.log \ - -u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "UPGRADE_SCRIPT"' + -u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "$UPGRADE_SCRIPT"' params: UPGRADE_ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"' UPGRADE_IMAGE: *mysql_image_pcmklatest - UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | union(['/tmp/mariadb-upgrade:/var/log/mariadb:rw,z']) | join(' -v ')}}" - UPGRADE_SCRIPT: "{{mysql_upgrade_script}}" + UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}}" CONTAINER_CLI: "{{ container_cli }}" LOG_DIR: '/var/log/containers/mysql' + environment: + UPGRADE_SCRIPT: "{{ mysql_upgrade_script }}" external_upgrade_tasks: - vars: