Upgrade mariadb storage during upgrade tasks
When a tripleo major upgrade or FFU causes an update or mariadb to a new major version (e.g. 10.1 -> 10.3), some internal DB tables must be upgraded (myisam tables), and sometimes the existing user tables may be migrated to new mariadb defaults. Move the db-specific upgrade steps into a dedicated script and make sure that it is called at the right time while upgrading the undercloud and/or the overcloud. Closes-Bug: #1913438 Change-Id: I92353622994b28c895d95bdcbe348a73b6c6bb99
This commit is contained in:
parent
c488d97b55
commit
712cfcc71b
15
container_config_scripts/mysql_upgrade_db.sh
Executable file
15
container_config_scripts/mysql_upgrade_db.sh
Executable file
@ -0,0 +1,15 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Wait until we know the mysql server is up and responding
|
||||||
|
timeout ${DB_MAX_TIMEOUT:-60} /bin/bash -c 'until mysqladmin -uroot ping 2>/dev/null; do sleep 1; done'
|
||||||
|
|
||||||
|
# After an upgrade, make sure that the running mysql had a chance to
|
||||||
|
# update its data table on disk.
|
||||||
|
mysql_upgrade
|
||||||
|
|
||||||
|
# Upgrade to 10.3: the default table row format changed from COMPACT
|
||||||
|
# to DYNAMIC, so upgrade the existing tables.
|
||||||
|
compact_tables=$(mysql -se 'SELECT CONCAT("`",TABLE_SCHEMA,"`.`",TABLE_NAME,"`") FROM information_schema.tables WHERE ENGINE = "InnoDB" and ROW_FORMAT = "Compact";');
|
||||||
|
for i in $compact_tables; do echo converting row format of table $i; mysql -e "ALTER TABLE $i ROW_FORMAT=DYNAMIC;"; done;
|
@ -99,6 +99,12 @@ conditions:
|
|||||||
key_size_override_unset: {equals: [{get_param: MysqlCertificateKeySize}, '']}
|
key_size_override_unset: {equals: [{get_param: MysqlCertificateKeySize}, '']}
|
||||||
|
|
||||||
outputs:
|
outputs:
|
||||||
|
container_config_scripts:
|
||||||
|
description: Shared docker config scripts
|
||||||
|
value:
|
||||||
|
mysql_upgrade_db.sh:
|
||||||
|
mode: "0755"
|
||||||
|
content: { get_file: ../../container_config_scripts/mysql_upgrade_db.sh }
|
||||||
role_data:
|
role_data:
|
||||||
description: Service MySQL using composable services.
|
description: Service MySQL using composable services.
|
||||||
value:
|
value:
|
||||||
|
@ -58,6 +58,9 @@ conditions:
|
|||||||
|
|
||||||
resources:
|
resources:
|
||||||
|
|
||||||
|
ContainersCommon:
|
||||||
|
type: ../containers-common.yaml
|
||||||
|
|
||||||
MysqlBase:
|
MysqlBase:
|
||||||
type: ./mysql-base.yaml
|
type: ./mysql-base.yaml
|
||||||
properties:
|
properties:
|
||||||
@ -128,6 +131,10 @@ outputs:
|
|||||||
- path: /etc/pki/tls/private/mysql.key
|
- path: /etc/pki/tls/private/mysql.key
|
||||||
owner: mysql:mysql
|
owner: mysql:mysql
|
||||||
optional: true
|
optional: true
|
||||||
|
container_config_scripts:
|
||||||
|
map_merge:
|
||||||
|
- {get_attr: [ContainersCommon, container_config_scripts]}
|
||||||
|
- {get_attr: [MysqlBase, container_config_scripts]}
|
||||||
docker_config:
|
docker_config:
|
||||||
# Kolla_bootstrap runs before permissions set by kolla_config
|
# Kolla_bootstrap runs before permissions set by kolla_config
|
||||||
step_1:
|
step_1:
|
||||||
@ -206,6 +213,23 @@ outputs:
|
|||||||
volumes: *mysql_volumes
|
volumes: *mysql_volumes
|
||||||
environment:
|
environment:
|
||||||
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
|
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
|
||||||
|
mysql_upgrade_db:
|
||||||
|
# update mysql db on disk after a version upgrade (idempotent)
|
||||||
|
config_volume: mysql
|
||||||
|
start_order: 3
|
||||||
|
detach: false
|
||||||
|
image: *mysql_image
|
||||||
|
volumes:
|
||||||
|
list_concat:
|
||||||
|
- *mysql_volumes
|
||||||
|
- - /var/lib/config-data/puppet-generated/mysql/root:/root:rw
|
||||||
|
- /var/lib/container-config-scripts/:/container-config-scripts/:ro
|
||||||
|
environment:
|
||||||
|
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
|
||||||
|
net: host
|
||||||
|
user: root
|
||||||
|
command:
|
||||||
|
- '/container-config-scripts/mysql_upgrade_db.sh'
|
||||||
step_3:
|
step_3:
|
||||||
# sync credentials config on the running container if it was
|
# sync credentials config on the running container if it was
|
||||||
# changed by the docker_puppet_task during step 2
|
# changed by the docker_puppet_task during step 2
|
||||||
@ -255,13 +279,12 @@ outputs:
|
|||||||
- {'path': /var/log/containers/mysql, 'setype': 'container_file_t', 'mode': '0750'}
|
- {'path': /var/log/containers/mysql, 'setype': 'container_file_t', 'mode': '0750'}
|
||||||
- {'path': /var/lib/mysql, 'setype': 'container_file_t'}
|
- {'path': /var/lib/mysql, 'setype': 'container_file_t'}
|
||||||
upgrade_tasks:
|
upgrade_tasks:
|
||||||
# LP 1810136
|
# When mariadb is upgraded to a new major release, one must run
|
||||||
# After upgrade, the new mariadb (e.g. 10.3) might not be able
|
# mysql_upgrade to upgrade the DB's system tables, and potentially
|
||||||
# to replay the redo log of an older one (e.g. 10.1) if mysql
|
# run other storage upgrade. We want to that as early as possible
|
||||||
# stopped unexpectedly. If the container image differs, force
|
# so the database is fully upgraded when services are restarted
|
||||||
# mysql to stop and clean the redo log here to avoid issue after
|
# during deploy steps.
|
||||||
# upgrade.
|
- name: Stop MySQL server and upgrade the database if needed
|
||||||
- name: Stop MySQL server and ensure redo log is cleaned up before upgrade
|
|
||||||
when: step|int == 2
|
when: step|int == 2
|
||||||
block:
|
block:
|
||||||
- name: Get Mysql container image name before upgrade
|
- name: Get Mysql container image name before upgrade
|
||||||
@ -282,18 +305,22 @@ outputs:
|
|||||||
list_join:
|
list_join:
|
||||||
- ' '
|
- ' '
|
||||||
- - 'kolla_set_configs;'
|
- - 'kolla_set_configs;'
|
||||||
- 'mysqld_safe --user=mysql --skip-networking &'
|
- 'mysqld_safe --user=mysql --skip-networking --log-error=/var/log/mariadb/mariadb-upgrade.log &'
|
||||||
- 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
|
- 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
|
||||||
- 'mysqladmin shutdown'
|
- 'mysqladmin shutdown'
|
||||||
- name: Bind mounts for temporary clean-up container
|
- name: Bind mounts for temporary clean-up container
|
||||||
set_fact:
|
set_fact:
|
||||||
mysql_clean_up_volumes: *mysql_volumes
|
mysql_upgrade_volumes: *mysql_volumes
|
||||||
- name: Stop the current mysql container
|
- name: Stop the current mysql container
|
||||||
systemd:
|
systemd:
|
||||||
state: stopped
|
state: stopped
|
||||||
name: tripleo_mysql
|
name: tripleo_mysql
|
||||||
when: pre_upgrade_mysql_image != post_upgrade_mysql_image
|
when: pre_upgrade_mysql_image != post_upgrade_mysql_image
|
||||||
- name: Clean up redo log by running a transient mysql server
|
- name: Clean up redo log by running a transient mysql server
|
||||||
|
# After upgrade, the new mariadb (e.g. 10.3) might not be able
|
||||||
|
# to replay the redo log of an older one (e.g. 10.1) if mysql
|
||||||
|
# stopped unexpectedly. So run a temporary server to cleanup
|
||||||
|
# the redo log now before upgrade.
|
||||||
shell:
|
shell:
|
||||||
str_replace:
|
str_replace:
|
||||||
template:
|
template:
|
||||||
@ -301,10 +328,6 @@ outputs:
|
|||||||
params:
|
params:
|
||||||
ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
|
ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
|
||||||
IMAGE: "{{ pre_upgrade_mysql_image }}"
|
IMAGE: "{{ pre_upgrade_mysql_image }}"
|
||||||
VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}"
|
VOLUMES: "-v {{ mysql_upgrade_volumes | join(' -v ') }}"
|
||||||
SCRIPT: "{{ mysql_clean_up_script }}"
|
SCRIPT: "{{ mysql_clean_up_script }}"
|
||||||
when: pre_upgrade_mysql_image != post_upgrade_mysql_image
|
when: pre_upgrade_mysql_image != post_upgrade_mysql_image
|
||||||
post_upgrade_tasks:
|
|
||||||
- name: Check and upgrade Mysql database after major version upgrade
|
|
||||||
command: "{{ container_cli }} exec -u root mysql mysql_upgrade"
|
|
||||||
when: step|int == 2
|
|
||||||
|
@ -253,10 +253,10 @@ outputs:
|
|||||||
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "GRANT PROCESS ON *.* TO ''clustercheck''@''localhost'' WITH GRANT OPTION;"'
|
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "GRANT PROCESS ON *.* TO ''clustercheck''@''localhost'' WITH GRANT OPTION;"'
|
||||||
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "DELETE FROM mysql.user WHERE user = ''root'' AND host NOT IN (''%'',''localhost'');"'
|
- 'mysql -uroot -p"${DB_ROOT_PASSWORD}" -e "DELETE FROM mysql.user WHERE user = ''root'' AND host NOT IN (''%'',''localhost'');"'
|
||||||
- 'timeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown'
|
- 'timeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown'
|
||||||
volumes: &mysql_volumes
|
volumes:
|
||||||
list_concat:
|
list_concat:
|
||||||
- {get_attr: [ContainersCommon, volumes]}
|
- {get_attr: [ContainersCommon, volumes]}
|
||||||
-
|
- &mysql_volumes
|
||||||
- /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z
|
- /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z
|
||||||
- /var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z
|
- /var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z
|
||||||
- /var/lib/mysql:/var/lib/mysql:rw,z
|
- /var/lib/mysql:/var/lib/mysql:rw,z
|
||||||
@ -609,42 +609,42 @@ outputs:
|
|||||||
- name: Mysql upgrade script
|
- name: Mysql upgrade script
|
||||||
set_fact:
|
set_fact:
|
||||||
mysql_upgrade_script:
|
mysql_upgrade_script:
|
||||||
# idempotency: mysql_upgrade leaves a marker file
|
|
||||||
# in datadir, it does nothing if it has already been
|
|
||||||
# executed for the current version of MariaDB.
|
|
||||||
list_join:
|
list_join:
|
||||||
- ' '
|
- "\n"
|
||||||
- - 'kolla_set_configs;'
|
- # start a temporary mariadb server for running the upgrade
|
||||||
- 'if mysqladmin ping --silent; then exit 0; fi;'
|
- |
|
||||||
- 'chown -R mysql:mysql /var/lib/mysql;'
|
kolla_set_configs
|
||||||
- 'chown -R mysql:mysql /var/log/mariadb;'
|
if mysqladmin ping --silent; then exit 0; fi
|
||||||
- 'mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off &'
|
chown -R mysql:mysql /var/lib/mysql
|
||||||
- 'timeout 60 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';'
|
chown -R mysql:mysql /var/log/mysql
|
||||||
- 'mysql_upgrade;'
|
mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off --log-error=/var/log/mysql/mysqld-upgrade.log &
|
||||||
- 'mysqladmin shutdown'
|
# an idempotent script takes care of all upgrade steps
|
||||||
|
# we inline the content here because at the time this is executed,
|
||||||
|
# the script is not yet copied in /var/lib/container-config-scripts
|
||||||
|
- { get_file: ../../container_config_scripts/mysql_upgrade_db.sh }
|
||||||
|
# stop the temporary server
|
||||||
|
- mysqladmin shutdown
|
||||||
- name: Bind mounts for temporary container
|
- name: Bind mounts for temporary container
|
||||||
set_fact:
|
set_fact:
|
||||||
mysql_upgrade_db_bind_mounts: *mysql_volumes
|
mysql_upgrade_db_bind_mounts:
|
||||||
- name: Make sure Mysql upgrade temporary directory exists
|
list_concat:
|
||||||
file:
|
- {get_attr: [ContainersCommon, volumes]}
|
||||||
path: /tmp/mariadb-upgrade
|
- *mysql_volumes
|
||||||
state: directory
|
- - /var/log/containers/mysql:/var/log/mysql:rw,z
|
||||||
owner: root
|
|
||||||
group: root
|
|
||||||
mode: 0700
|
|
||||||
- name: Upgrade Mysql database from a temporary container
|
- name: Upgrade Mysql database from a temporary container
|
||||||
shell:
|
shell:
|
||||||
str_replace:
|
str_replace:
|
||||||
template:
|
template:
|
||||||
'CONTAINER_CLI run --rm --log-driver=k8s-file --log-opt path=LOG_DIR/db-upgrade.log \
|
'CONTAINER_CLI run --rm --log-driver=k8s-file --log-opt path=LOG_DIR/db-upgrade.log \
|
||||||
-u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "UPGRADE_SCRIPT"'
|
-u root --net=host UPGRADE_ENV UPGRADE_VOLUMES "UPGRADE_IMAGE" /bin/bash -ecx "$UPGRADE_SCRIPT"'
|
||||||
params:
|
params:
|
||||||
UPGRADE_ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
|
UPGRADE_ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"'
|
||||||
UPGRADE_IMAGE: *mysql_image_pcmklatest
|
UPGRADE_IMAGE: *mysql_image_pcmklatest
|
||||||
UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | union(['/tmp/mariadb-upgrade:/var/log/mariadb:rw,z']) | join(' -v ')}}"
|
UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}}"
|
||||||
UPGRADE_SCRIPT: "{{mysql_upgrade_script}}"
|
|
||||||
CONTAINER_CLI: "{{ container_cli }}"
|
CONTAINER_CLI: "{{ container_cli }}"
|
||||||
LOG_DIR: '/var/log/containers/mysql'
|
LOG_DIR: '/var/log/containers/mysql'
|
||||||
|
environment:
|
||||||
|
UPGRADE_SCRIPT: "{{ mysql_upgrade_script }}"
|
||||||
|
|
||||||
external_upgrade_tasks:
|
external_upgrade_tasks:
|
||||||
- vars:
|
- vars:
|
||||||
|
Loading…
Reference in New Issue
Block a user