From f5ffc4d4c76ded2e4f637d5f577c32b383f319b3 Mon Sep 17 00:00:00 2001 From: Jiri Stransky Date: Mon, 26 Aug 2019 10:20:04 +0200 Subject: [PATCH] Fix MariaDB staged upgrade This patch fixes 2 issues on MariaDB staged upgrade: * After we landed service stopping on the data transfer step, the actual MariaDB staged upgrade moved from step 1 to step 2 of external_upgrade_tasks, but we only ran steps 0 and 1 during the upgrade. Increase the number of steps to run step 2 as well. * The mysql_upgrade container was being re-run on each upgrade-scaleup even on nodes where it already ran before and MariaDB was already running there, conflicting with the upgrade container. We fix this by only running the MariaDB upgrade if there's no MariaDB running yet on the particular node. Change-Id: I2feba6c22cdf07ea6af406c5edd27f62ff285d87 Co-Authored-By: Damien Ciabrini Closes-Bug: #1841415 (cherry picked from commit 22f26d7a94db952d679e143c162b6e45b8e8b3d3) --- common/deploy-steps.j2 | 2 +- deployment/database/mysql-pacemaker-puppet.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/common/deploy-steps.j2 b/common/deploy-steps.j2 index ad47bfcfb9..9053610481 100644 --- a/common/deploy-steps.j2 +++ b/common/deploy-steps.j2 @@ -19,7 +19,7 @@ {% set external_update_steps_max = 2 -%} {% set pre_upgrade_rolling_steps_max = 1 -%} {% set upgrade_steps_max = 6 -%} -{% set external_upgrade_steps_max = 2 -%} +{% set external_upgrade_steps_max = 3 -%} {% set post_upgrade_steps_max = 4 -%} {% set fast_forward_upgrade_steps_max = 9 -%} {% set fast_forward_upgrade_prep_steps_max = 3 -%} diff --git a/deployment/database/mysql-pacemaker-puppet.yaml b/deployment/database/mysql-pacemaker-puppet.yaml index 359b686c3e..204ebbd82b 100644 --- a/deployment/database/mysql-pacemaker-puppet.yaml +++ b/deployment/database/mysql-pacemaker-puppet.yaml @@ -563,6 +563,13 @@ outputs: - step|int == 3 block: *mysql_fetch_retag_container_tasks - name: Check and upgrade Mysql database after major version upgrade + # Note: during upgrade to Stein, a new pacemaker cluster is recreated, + # controller nodes added sequentially to this new cluster, and the upgrade + # workflow (upgrade tasks, deploy/convertge) is ran once per controller. + # This mysql upgrade block must run only once per controller, before + # the controller is added into the cluster (by mysql_init_bundle) and + # before pacemaker has a chance to start galera on that controller. + # So do not exercise the upgrade if mysql is already running. when: step|int == 3 block: # mariadb package changes ownership of /var/lib/mysql on package @@ -580,6 +587,7 @@ outputs: list_join: - ' ' - - 'kolla_set_configs;' + - 'if mysqladmin ping --silent; then exit 0; fi;' - 'chown -R mysql:mysql /var/lib/mysql;' - 'chown -R mysql:mysql /var/log/mariadb;' - 'mysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off &'