openstack
/
tripleo-heat-templates
Code Issues Proposed changes

Upgrade mariadb storage during upgrade tasks 

When a tripleo major upgrade or FFU causes an update or mariadb
to a new major version (e.g. 10.1 -> 10.3), some internal DB
tables must be upgraded (myisam tables), and sometimes some
the existing user tables may be migrated to new mariadb defaults.

Move the db-specific upgrade steps into a dedicated script and
make sure that it is called at the right time while upgrading
the undercloud and/or the overcloud.

Closes-Bug: #1913438

Change-Id: I92353622994b28c895d95bdcbe348a73b6c6bb99
(cherry picked from commit ad35169510)
This commit is contained in:
Damien Ciabrini 2020-12-17 13:23:58 +01:00
parent 28f98ae885
commit 9bdf4b1685
4 changed files with 78 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
container_config_scripts/mysql_upgrade_db.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash
set -e
# Wait until we know the mysql server is up and responding
timeout ${DB_MAX_TIMEOUT:-60} /bin/bash -c 'until mysqladmin -uroot ping 2>/dev/null; do sleep 1; done'
# After an upgrade, make sure that the running mysql had a chance to
# update its data table on disk.
mysql_upgrade
# Upgrade to 10.3: the default table row format changed from COMPACT
# to DYNAMIC, so upgrade the existing tables.
compact_tables=$(mysql -se 'SELECT CONCAT("`",TABLE_SCHEMA,"`.`",TABLE_NAME,"`") FROM information_schema.tables WHERE ENGINE = "InnoDB" and ROW_FORMAT = "Compact";');
for i in $compact_tables; do echo converting row format of table $i; mysql -e "ALTER TABLE $i ROW_FORMAT=DYNAMIC;"; done;

6
deployment/database/mysql-base.yaml
View File

@ -80,6 +80,12 @@ conditions:
key_size_override_unset: {equals: [{get_param: MysqlCertificateKeySize}, '']}
outputs:
container_config_scripts:
description: Shared docker config scripts
value:
mysql_upgrade_db.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/mysql_upgrade_db.sh }
role_data:
description: Service MySQL using composable services.
value:

46
deployment/database/mysql-container-puppet.yaml
View File

@ -125,6 +125,7 @@ outputs:
container_config_scripts:
map_merge:
- {get_attr: [ContainersCommon, container_config_scripts]}
- {get_attr: [MysqlBase, container_config_scripts]}
- neutron_db_rename.sh:
mode: "0755"
content: { get_file: ../../container_config_scripts/neutron_db_rename.sh }
@ -206,6 +207,23 @@ outputs:
volumes: *mysql_volumes
environment:
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
mysql_upgrade_db:
# update mysql db on disk after a version upgrade (idempotent)
config_volume: mysql
start_order: 3
detach: false
image: *mysql_image
volumes:
list_concat:
- *mysql_volumes
- - /var/lib/config-data/puppet-generated/mysql/root:/root:rw
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
environment:
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
net: host
user: root
command:
- '/container-config-scripts/mysql_upgrade_db.sh'
mysql_neutron_db_rename:
start_order: 3
detach: false
@ -273,13 +291,13 @@ outputs:
- {'path': /var/log/containers/mysql, 'setype': 'svirt_sandbox_file_t', 'mode': '0750'}
- {'path': /var/lib/mysql, 'setype': 'svirt_sandbox_file_t'}
upgrade_tasks:
# LP 1810136
# After upgrade, the new mariadb (e.g. 10.3) might not be able
# to replay the redo log of an older one (e.g. 10.1) if mysql
# stopped unexpectedly. If the container image differs, force
# mysql to stop and clean the redo log here to avoid issue after
# upgrade.
- name: Stop MySQL server and ensure redo log is cleaned up before upgrade
# When mariadb is upgraded to a new major release, one must run
# mysql_upgrade to upgrade the DB's system tables, and potentially
# run other storage upgrade. We want to that as early as possible
# so the database is fully upgraded when services are restarted
# during deploy steps.
# restarted, so the during deploy steps.
- name: Stop MySQL server and upgrade the database if needed
when: step|int == 2
block:
- name: Retrieve mysql container configuration from paunch
@ -313,18 +331,22 @@ outputs:
list_join:
- ' '
- - 'kolla_set_configs;'
- 'mysqld_safe --user=mysql --skip-networking &'
- 'mysqld_safe --user=mysql --skip-networking --log-error=/var/log/mariadb/mariadb-upgrade.log &'
- 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
- 'mysqladmin shutdown'
- name: Bind mounts for temporary clean-up container
set_fact:
mysql_clean_up_volumes: *mysql_volumes
mysql_upgrade_volumes: *mysql_volumes
- name: Stop the current mysql container
shell: |
{{ mysql_json.cli }} update --restart=unless-stopped mysql
{{ mysql_json.cli }} stop mysql
when: mysql_json.image is defined and mysql_json.image != post_upgrade_mysql_image
- name: Clean up redo log by running a transient mysql server
# After upgrade, the new mariadb (e.g. 10.3) might not be able
# to replay the redo log of an older one (e.g. 10.1) if mysql
# stopped unexpectedly. So run a temporary server to cleanup
# the redo now before upgrade.
shell:
str_replace:
template:
@ -332,10 +354,6 @@ outputs:
params:
ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
IMAGE: "{{ mysql_json.image }}"
VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}"
VOLUMES: "-v {{ mysql_upgrade_volumes | join(' -v ') }}"
SCRIPT: "{{ mysql_clean_up_script }}"
when: mysql_json.image is defined and mysql_json.image != post_upgrade_mysql_image
post_upgrade_tasks:
- name: Check and upgrade Mysql database after major version upgrade
command: "{{ container_cli }} exec -u root mysql mysql_upgrade"
when: step|int == 2

50
deployment/database/mysql-pacemaker-puppet.yaml
View File

@ -243,10 +243,10 @@ outputs:
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "GRANT PROCESS ON *.* TO ''clustercheck''@''localhost'' WITH GRANT OPTION;"'
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "DELETE FROM mysql.user WHERE user = ''root'' AND host NOT IN (''%'',''localhost'');"'
- 'timeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown'
volumes: &mysql_volumes
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- &mysql_volumes
- /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z
- /var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z
- /var/lib/mysql:/var/lib/mysql:rw,z
@ -564,35 +564,34 @@ outputs:
- name: Mysql upgrade script
set_fact:
mysql_upgrade_script:
# idempotency: mysql_upgrade leaves a marker file
# in datadir, it does nothing if it has already been
# executed for the current version of MariaDB.
list_join:
- ' '
- - 'kolla_set_configs;'
- 'if mysqladmin ping --silent; then exit 0; fi;'
- 'chown -R mysql:mysql /var/lib/mysql;'
- 'chown -R mysql:mysql /var/log/mariadb;'
- 'mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off &'
- 'timeout 60 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
- 'mysql_upgrade;'
- 'mysqladmin shutdown'
- "\n"
- # start a temporary mariadb server for running the upgrade
- |
kolla_set_configs
if mysqladmin ping --silent; then exit 0; fi
chown -R mysql:mysql /var/lib/mysql
chown -R mysql:mysql /var/log/mysql
mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off --log-error=/var/log/mysql/mysqld-upgrade.log &
# an idempotent script takes care of all upgrade steps
# we inline the content here because at the time this is executed,
# the script is not yet copied in /var/lib/container-config-scripts
- { get_file: ../../container_config_scripts/mysql_upgrade_db.sh }
# stop the temporary server
- mysqladmin shutdown
- name: Bind mounts for temporary container
set_fact:
mysql_upgrade_db_bind_mounts: *mysql_volumes
- name: Make sure Mysql upgrade temporary directory exists
file:
path: /tmp/mariadb-upgrade
state: directory
owner: root
group: root
mode: 0700
mysql_upgrade_db_bind_mounts:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
- *mysql_volumes
- - /var/log/containers/mysql:/var/log/mysql:rw,z
- name: Upgrade Mysql database from a temporary container
shell:
str_replace:
template:
'CONTAINER_CLI run --rm CONTAINER_CLI_EXTRA_ARGS \
-u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "UPGRADE_SCRIPT"'
-u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "$UPGRADE_SCRIPT"'
params:
CONTAINER_CLI_EXTRA_ARGS:
if:
@ -601,9 +600,10 @@ outputs:
- '--log-driver=k8s-file --log-opt path=/var/log/containers/mysql/db-upgrade.log'
UPGRADE_ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
UPGRADE_IMAGE: *mysql_image_pcmklatest
UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | union(['/tmp/mariadb-upgrade:/var/log/mariadb:rw,z']) | join(' -v ')}}"
UPGRADE_SCRIPT: "{{mysql_upgrade_script}}"
UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}}"
CONTAINER_CLI: "{{ container_cli }}"
environment:
UPGRADE_SCRIPT: "{{ mysql_upgrade_script }}"
external_upgrade_tasks:
# https://bugs.launchpad.net/tripleo/+bug/1753247