cf465d87b1
Currently when we call the major-upgrade step we do the following:
"""
...
if [[ -n $(is_bootstrap_node) ]]; then
check_clean_cluster
fi
...
if [[ -n $(is_bootstrap_node) ]]; then
migrate_full_to_ng_ha
fi
...
for service in $(services_to_migrate); do
manage_systemd_service stop "${service%%-clone}"
...
done
"""
The problem with the above code is that it is open to the following race
condition:
1. Code gets run first on a non-bootstrap controller node so we start
stopping a bunch of services
2. Pacemaker notices will notice that services are down and will mark
the service as stopped
3. Code gets run on the bootstrap node (controller-0) and the
check_clean_cluster function will fail and exit
4. Eventually also the script on the non-bootstrap controller node will
timeout and exit because the cluster never shut down (it never actually
started the shutdown because we failed at 3)
Let's make sure we first only call the HA NG migration step as a
separate heat step. Only afterwards we start shutting down the systemd
services on all nodes.
We also need to move the STONITH_STATE variable into a file because it
is being used across two different scripts (1 and 2) and we need to
store that state.
Co-Authored-By: Athlan-Guyot Sofer <sathlang@redhat.com>
Closes-Bug: #1640407
Change-Id: Ifb9b9e633fcc77604cca2590071656f4b2275c60
(cherry picked from commit dde12b075f)
69 lines
2.3 KiB
Bash
Executable File
69 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -eu
|
|
|
|
cluster_form_timeout=600
|
|
cluster_settle_timeout=1800
|
|
galera_sync_timeout=600
|
|
|
|
if [[ -n $(is_bootstrap_node) ]]; then
|
|
pcs cluster start --all
|
|
|
|
tstart=$(date +%s)
|
|
while pcs status 2>&1 | grep -E '(cluster is not currently running)|(OFFLINE:)'; do
|
|
sleep 5
|
|
tnow=$(date +%s)
|
|
if (( tnow-tstart > cluster_form_timeout )) ; then
|
|
echo_error "ERROR: timed out forming the cluster"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if ! timeout -k 10 $cluster_settle_timeout crm_resource --wait; then
|
|
echo_error "ERROR: timed out waiting for cluster to finish transition"
|
|
exit 1
|
|
fi
|
|
|
|
for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Stopped | awk '{ print $1 }'); do
|
|
pcs resource enable $vip
|
|
check_resource_pacemaker $vip started 60
|
|
done
|
|
fi
|
|
|
|
start_or_enable_service galera
|
|
check_resource galera started 600
|
|
start_or_enable_service redis
|
|
check_resource redis started 600
|
|
# We need mongod which is now a systemd service up and running before calling
|
|
# ceilometer-dbsync. There is still a race here: mongod might not be up on all nodes
|
|
# so ceilometer-dbsync will fail a couple of times before that. As it retries indefinitely
|
|
# we should be good.
|
|
# Due to LP Bug https://bugs.launchpad.net/tripleo/+bug/1627254 am using systemctl directly atm
|
|
systemctl start mongod
|
|
check_resource mongod started 600
|
|
|
|
if [[ -n $(is_bootstrap_node) ]]; then
|
|
tstart=$(date +%s)
|
|
while ! clustercheck; do
|
|
sleep 5
|
|
tnow=$(date +%s)
|
|
if (( tnow-tstart > galera_sync_timeout )) ; then
|
|
echo_error "ERROR galera sync timed out"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# Run all the db syncs
|
|
# TODO: check if this can be triggered in puppet and removed from here
|
|
ceilometer-dbsync --config-file=/etc/ceilometer/ceilometer.conf
|
|
cinder-manage db sync
|
|
glance-manage --config-file=/etc/glance/glance-registry.conf db_sync
|
|
heat-manage --config-file /etc/heat/heat.conf db_sync
|
|
keystone-manage db_sync
|
|
neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini upgrade head
|
|
nova-manage db sync
|
|
nova-manage api_db sync
|
|
nova-manage db online_data_migrations
|
|
sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head
|
|
fi
|