Merge "Dump and restore galera db during major upgrades"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
a70464cd0d
@ -17,7 +17,81 @@ fi
|
||||
STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk '{ print $2 }')
|
||||
pcs property set stonith-enabled=false
|
||||
|
||||
# If for some reason rpm-python are missing we want to error out early enough
|
||||
if [ ! rpm -q rpm-python &> /dev/null ]; then
|
||||
echo_error "ERROR: upgrade cannot start without rpm-python installed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# In case the mysql package is updated, the database on disk must be
|
||||
# upgraded as well. This typically needs to happen during major
|
||||
# version upgrades (e.g. 5.5 -> 5.6, 5.5 -> 10.1...)
|
||||
#
|
||||
# Because in-place upgrades are not supported across 2+ major versions
|
||||
# (e.g. 5.5 -> 10.1), we rely on logical upgrades via dump/restore cycle
|
||||
# https://bugzilla.redhat.com/show_bug.cgi?id=1341968
|
||||
#
|
||||
# The default is to determine automatically if upgrade is needed based
|
||||
# on mysql package versionning, but this can be overriden manually
|
||||
# to support specific upgrade scenario
|
||||
|
||||
# Where to backup current database if mysql need to be upgraded
|
||||
MYSQL_BACKUP_DIR=/var/tmp/mysql_upgrade_osp
|
||||
MYSQL_TEMP_UPGRADE_BACKUP_DIR=/var/lib/mysql-temp-upgrade-backup
|
||||
# Spare disk ratio for extra safety
|
||||
MYSQL_BACKUP_SIZE_RATIO=1.2
|
||||
|
||||
# Shall we upgrade mysql data directory during the stack upgrade?
|
||||
if [ "$mariadb_do_major_upgrade" = "auto" ]; then
|
||||
ret=$(is_mysql_upgrade_needed)
|
||||
if [ $ret = "1" ]; then
|
||||
DO_MYSQL_UPGRADE=1
|
||||
else
|
||||
DO_MYSQL_UPGRADE=0
|
||||
fi
|
||||
echo "mysql upgrade required: $DO_MYSQL_UPGRADE"
|
||||
elif [ "$mariadb_do_major_upgrade" = 0 ]; then
|
||||
DO_MYSQL_UPGRADE=0
|
||||
else
|
||||
DO_MYSQL_UPGRADE=1
|
||||
fi
|
||||
|
||||
if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
|
||||
if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
|
||||
if [ -d "$MYSQL_BACKUP_DIR" ]; then
|
||||
echo_error "Error: $MYSQL_BACKUP_DIR exists already. Likely an upgrade failed previously"
|
||||
exit 1
|
||||
fi
|
||||
mkdir "$MYSQL_BACKUP_DIR"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo_error "Error: could not create temporary backup directory $MYSQL_BACKUP_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# the /root/.my.cnf is needed because we set the mysql root
|
||||
# password from liberty onwards
|
||||
backup_flags="--defaults-extra-file=/root/.my.cnf -u root --flush-privileges --all-databases --single-transaction"
|
||||
# While not ideal, this step allows us to calculate exactly how much space the dump
|
||||
# will need. Our main goal here is avoiding any chance of corruption due to disk space
|
||||
# exhaustion
|
||||
backup_size=$(mysqldump $backup_flags 2>/dev/null | wc -c)
|
||||
database_size=$(du -cb /var/lib/mysql | tail -1 | awk '{ print $1 }')
|
||||
free_space=$(df -B1 --output=avail "$MYSQL_BACKUP_DIR" | tail -1)
|
||||
|
||||
# we need at least space for a new mysql database + dump of the existing one,
|
||||
# times a small factor for additional safety room
|
||||
# note: bash doesn't do floating point math or floats in if statements,
|
||||
# so use python to apply the ratio and cast it back to integer
|
||||
required_space=$(python -c "from __future__ import print_function; print(\"%d\" % int((($database_size + $backup_size) * $MYSQL_BACKUP_SIZE_RATIO)))")
|
||||
if [ $required_space -ge $free_space ]; then
|
||||
echo_error "Error: not enough free space in $MYSQL_BACKUP_DIR ($required_space bytes required)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mysqldump $backup_flags > "$MYSQL_BACKUP_DIR/openstack_database.sql"
|
||||
cp -rdp /etc/my.cnf* "$MYSQL_BACKUP_DIR"
|
||||
fi
|
||||
|
||||
pcs resource disable httpd
|
||||
check_resource httpd stopped 1800
|
||||
pcs resource disable openstack-core
|
||||
@ -54,9 +128,63 @@ while systemctl is-active pacemaker; do
|
||||
fi
|
||||
done
|
||||
|
||||
# The reason we do an sql dump *and* we move the old dir out of
|
||||
# the way is because it gives us an extra level of safety in case
|
||||
# something goes wrong during the upgrade. Once the restore is
|
||||
# successful we go ahead and remove it. If the directory exists
|
||||
# we bail out as it means the upgrade process had issues in the last
|
||||
# run.
|
||||
if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
|
||||
if [ -d $MYSQL_TEMP_UPGRADE_BACKUP_DIR ]; then
|
||||
echo_error "ERROR: mysql backup dir already exist"
|
||||
exit 1
|
||||
fi
|
||||
mv /var/lib/mysql $MYSQL_TEMP_UPGRADE_BACKUP_DIR
|
||||
fi
|
||||
|
||||
yum -y install python-zaqarclient # needed for os-collect-config
|
||||
yum -y -q update
|
||||
|
||||
# We need to ensure at least those two configuration settings, otherwise
|
||||
# mariadb 10.1+ won't activate galera replication.
|
||||
# wsrep_cluster_address must only be set though, its value does not
|
||||
# matter because it's overriden by the galera resource agent.
|
||||
cat >> /etc/my.cnf.d/galera.cnf <<EOF
|
||||
[mysqld]
|
||||
wsrep_on = ON
|
||||
wsrep_cluster_address = gcomm://localhost
|
||||
EOF
|
||||
|
||||
if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
|
||||
if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
|
||||
# Scripts run via heat have no HOME variable set and this confuses
|
||||
# mysqladmin
|
||||
export HOME=/root
|
||||
mkdir /var/lib/mysql || /bin/true
|
||||
chown mysql:mysql /var/lib/mysql
|
||||
chmod 0755 /var/lib/mysql
|
||||
restorecon -R /var/lib/mysql/
|
||||
mysql_install_db --datadir=/var/lib/mysql --user=mysql
|
||||
chown -R mysql:mysql /var/lib/mysql/
|
||||
mysqld_safe --wsrep-new-cluster &
|
||||
# We have a populated /root/.my.cnf with root/password here so
|
||||
# we need to temporarily rename it because the newly created
|
||||
# db is empty and no root password is set
|
||||
mv /root/.my.cnf /root/.my.cnf.temporary
|
||||
timeout 60 sh -c 'while ! mysql -e "" &> /dev/null; do sleep 1; done'
|
||||
mysql -u root < "$MYSQL_BACKUP_DIR/openstack_database.sql"
|
||||
mv /root/.my.cnf.temporary /root/.my.cnf
|
||||
mysqladmin -u root shutdown
|
||||
# The import was successful so we may remove the folder
|
||||
rm -r "$MYSQL_BACKUP_DIR"
|
||||
fi
|
||||
fi
|
||||
|
||||
# If we reached here without error we can safely blow away the origin
|
||||
# mysql dir from every controller
|
||||
if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
|
||||
rm -r $MYSQL_TEMP_UPGRADE_BACKUP_DIR
|
||||
fi
|
||||
|
||||
# Let's reset the stonith back to true if it was true, before starting the cluster
|
||||
if [ $STONITH_STATE == "true" ]; then
|
||||
|
||||
@ -20,6 +20,12 @@ parameters:
|
||||
type: string
|
||||
description: Nova Compute upgrade level
|
||||
default: ''
|
||||
MySqlMajorUpgrade:
|
||||
type: string
|
||||
description: Can be auto,yes,no and influences if the major upgrade should do or detect an automatic mysql upgrade
|
||||
constraints:
|
||||
- allowed_values: ['auto', 'yes', 'no']
|
||||
default: 'auto'
|
||||
|
||||
resources:
|
||||
# TODO(jistr): for Mitaka->Newton upgrades and further we can use
|
||||
@ -39,6 +45,12 @@ resources:
|
||||
upgrade_level_nova_compute='UPGRADE_LEVEL_NOVA_COMPUTE'
|
||||
params:
|
||||
UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute}
|
||||
- str_replace:
|
||||
template: |
|
||||
#!/bin/bash
|
||||
mariadb_do_major_upgrade='MYSQL_MAJOR_UPGRADE'
|
||||
params:
|
||||
MYSQL_MAJOR_UPGRADE: {get_param: MySqlMajorUpgrade}
|
||||
- get_file: pacemaker_common_functions.sh
|
||||
- get_file: major_upgrade_pacemaker_migrations.sh
|
||||
- get_file: major_upgrade_controller_pacemaker_1.sh
|
||||
|
||||
@ -13,6 +13,50 @@
|
||||
# been already applied, it should be possible to call the function
|
||||
# again without damaging the deployment or failing the upgrade.
|
||||
|
||||
# If the major version of mysql is going to change after the major
|
||||
# upgrade, the database must be upgraded on disk to avoid failures
|
||||
# due to internal incompatibilities between major mysql versions
|
||||
# https://bugs.launchpad.net/tripleo/+bug/1587449
|
||||
# This function detects whether a database upgrade is required
|
||||
# after a mysql package upgrade. It returns 0 when no major upgrade
|
||||
# has to take place, 1 otherwise.
|
||||
function is_mysql_upgrade_needed {
|
||||
# The name of the package which provides mysql might differ
|
||||
# after the upgrade. Consider the generic package name, which
|
||||
# should capture the major version change (e.g. 5.5 -> 10.1)
|
||||
local name="mariadb"
|
||||
local output
|
||||
local ret
|
||||
set +e
|
||||
output=$(yum -q check-update $name)
|
||||
ret=$?
|
||||
set -e
|
||||
if [ $ret -ne 100 ]; then
|
||||
# no updates so we exit
|
||||
echo "0"
|
||||
return
|
||||
fi
|
||||
|
||||
local currentepoch=$(rpm -q --qf "%{epoch}" $name)
|
||||
local currentversion=$(rpm -q --qf "%{version}" $name)
|
||||
local currentrelease=$(rpm -q --qf "%{release}" $name)
|
||||
local newoutput=$(repoquery -a --pkgnarrow=updates --qf "%{epoch} %{version} %{release}\n" $name)
|
||||
local newepoch=$(echo "$newoutput" | awk '{ print $1 }')
|
||||
local newversion=$(echo "$newoutput" | awk '{ print $2 }')
|
||||
local newrelease=$(echo "$newoutput" | awk '{ print $3 }')
|
||||
|
||||
# With this we trigger the dump restore/path if we change either epoch or
|
||||
# version in the package If only the release tag changes we do not do it
|
||||
# FIXME: we could refine this by trying to parse the mariadb version
|
||||
# into X.Y.Z and trigger the update only if X and/or Y change.
|
||||
output=$(python -c "import rpm; rc = rpm.labelCompare((\"$currentepoch\", \"$currentversion\", None), (\"$newepoch\", \"$newversion\", None)); print rc")
|
||||
if [ "$output" != "-1" ]; then
|
||||
echo "0"
|
||||
return
|
||||
fi
|
||||
echo "1"
|
||||
}
|
||||
|
||||
function add_missing_openstack_core_constraints {
|
||||
# The CIBs are saved under /root as they might contain sensitive data
|
||||
CIB="/root/migration.cib"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user