Merge "A few major-upgrade issues"
This commit is contained in:
commit
7565e03a82
@ -19,7 +19,7 @@ STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk
|
|||||||
pcs property set stonith-enabled=false
|
pcs property set stonith-enabled=false
|
||||||
|
|
||||||
# Migrate to HA NG
|
# Migrate to HA NG
|
||||||
if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
|
if [[ -n $(is_bootstrap_node) ]]; then
|
||||||
migrate_full_to_ng_ha
|
migrate_full_to_ng_ha
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@ -29,9 +29,26 @@ fi
|
|||||||
# is going to take a long time because rabbit is down. By having the service stopped
|
# is going to take a long time because rabbit is down. By having the service stopped
|
||||||
# systemctl try-restart is a noop
|
# systemctl try-restart is a noop
|
||||||
|
|
||||||
for $service in $(services_to_migrate); do
|
for service in $(services_to_migrate); do
|
||||||
manage_systemd_service stop "${service%%-clone}"
|
manage_systemd_service stop "${service%%-clone}"
|
||||||
check_resource_systemd "${service%%-clone}" stopped 600
|
# So the reason for not reusing check_resource_systemd is that
|
||||||
|
# I have observed systemctl is-active returning unknown with at least
|
||||||
|
# one service that was stopped (See LP 1627254)
|
||||||
|
timeout=600
|
||||||
|
tstart=$(date +%s)
|
||||||
|
tend=$(( $tstart + $timeout ))
|
||||||
|
check_interval=3
|
||||||
|
while (( $(date +%s) < $tend )); do
|
||||||
|
if [[ "$(systemctl is-active ${service%%-clone})" = "active" ]]; then
|
||||||
|
echo "$service still active, sleeping $check_interval seconds."
|
||||||
|
sleep $check_interval
|
||||||
|
else
|
||||||
|
# we do not care if it is inactive, unknown or failed as long as it is
|
||||||
|
# not running
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
# In case the mysql package is updated, the database on disk must be
|
# In case the mysql package is updated, the database on disk must be
|
||||||
@ -46,7 +63,7 @@ done
|
|||||||
# on mysql package versionning, but this can be overriden manually
|
# on mysql package versionning, but this can be overriden manually
|
||||||
# to support specific upgrade scenario
|
# to support specific upgrade scenario
|
||||||
|
|
||||||
if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
|
if [[ -n $(is_bootstrap_node) ]]; then
|
||||||
if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
|
if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
|
||||||
mysqldump $backup_flags > "$MYSQL_BACKUP_DIR/openstack_database.sql"
|
mysqldump $backup_flags > "$MYSQL_BACKUP_DIR/openstack_database.sql"
|
||||||
cp -rdp /etc/my.cnf* "$MYSQL_BACKUP_DIR"
|
cp -rdp /etc/my.cnf* "$MYSQL_BACKUP_DIR"
|
||||||
@ -68,7 +85,7 @@ if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
# Swift isn't controled by pacemaker
|
# Swift isn't controlled by pacemaker
|
||||||
systemctl_swift stop
|
systemctl_swift stop
|
||||||
|
|
||||||
tstart=$(date +%s)
|
tstart=$(date +%s)
|
||||||
|
@ -76,7 +76,7 @@ systemctl_swift start
|
|||||||
# We need to start the systemd services we explicitely stopped at step _1.sh
|
# We need to start the systemd services we explicitely stopped at step _1.sh
|
||||||
# FIXME: Should we let puppet during the convergence step do the service enabling or
|
# FIXME: Should we let puppet during the convergence step do the service enabling or
|
||||||
# should we add it here?
|
# should we add it here?
|
||||||
for $service in $(services_to_migrate); do
|
for service in $(services_to_migrate); do
|
||||||
manage_systemd_service stop "${service%%-clone}"
|
manage_systemd_service start "${service%%-clone}"
|
||||||
check_resource_systemd "${service%%-clone}" started 600
|
check_resource_systemd "${service%%-clone}" started 600
|
||||||
done
|
done
|
||||||
|
@ -109,10 +109,11 @@ function services_to_migrate {
|
|||||||
# during the conversion
|
# during the conversion
|
||||||
# 2. Remove all the colocation constraints and then the ordering constraints, except the
|
# 2. Remove all the colocation constraints and then the ordering constraints, except the
|
||||||
# ones related to haproxy/VIPs which exist in Newton as well
|
# ones related to haproxy/VIPs which exist in Newton as well
|
||||||
# 3. Remove all the resources that won't be managed by pacemaker in newton. Note that they
|
# 3. Take the cluster out of maintenance-mode and do a resource cleanup
|
||||||
# will show up as ORPHANED but they will keep running normally via systemd. They will be
|
# 4. Remove all the resources that won't be managed by pacemaker in newton. The
|
||||||
# enabled to start at boot by puppet during the converge step
|
# outcome will be
|
||||||
# 4. Take the cluster out of maintenance-mode and do a resource cleanup
|
# that they are stopped and removed from pacemakers control
|
||||||
|
# 5. Do a resource cleanup to make sure the cluster is in a clean state
|
||||||
function migrate_full_to_ng_ha {
|
function migrate_full_to_ng_ha {
|
||||||
if [[ -n $(pcmk_running) ]]; then
|
if [[ -n $(pcmk_running) ]]; then
|
||||||
pcs property set maintenance-mode=true
|
pcs property set maintenance-mode=true
|
||||||
@ -135,32 +136,35 @@ function migrate_full_to_ng_ha {
|
|||||||
log_debug "Deleting ordering constraint $constraint from CIB"
|
log_debug "Deleting ordering constraint $constraint from CIB"
|
||||||
pcs constraint remove "$constraint"
|
pcs constraint remove "$constraint"
|
||||||
done
|
done
|
||||||
|
# At this stage all the pacemaker resources are removed from the CIB.
|
||||||
|
# Once we remove the maintenance-mode those systemd resources will keep
|
||||||
|
# on running. They shall be systemd enabled via the puppet converge
|
||||||
|
# step later on
|
||||||
|
pcs property set maintenance-mode=false
|
||||||
|
|
||||||
# At this stage there are no constraints whatsoever except the haproxy/ip ones
|
# At this stage there are no constraints whatsoever except the haproxy/ip ones
|
||||||
# which we want to keep. We now delete each resource that will move to systemd
|
# which we want to keep. We now disable and then delete each resource
|
||||||
# Note that the corresponding systemd resource will stay running, which means that
|
# that will move to systemd.
|
||||||
# later when we do the "yum update", things will be a bit slower because each
|
# We want the systemd resources be stopped before doing "yum update",
|
||||||
# "systemctl try-restart <service>" is not a no-op any longer because the service is up
|
# that way "systemctl try-restart <service>" is no-op because the
|
||||||
# and running and it will be restarted with rabbitmq being down.
|
# service was down already
|
||||||
PCS_STATUS_OUTPUT="$(pcs status)"
|
PCS_STATUS_OUTPUT="$(pcs status)"
|
||||||
for resource in $(services_to_migrate) "delay-clone" "openstack-core-clone"; do
|
for resource in $(services_to_migrate) "delay-clone" "openstack-core-clone"; do
|
||||||
if echo "$PCS_STATUS_OUTPUT" | grep "$resource"; then
|
if echo "$PCS_STATUS_OUTPUT" | grep "$resource"; then
|
||||||
log_debug "Deleting $resource from the CIB"
|
log_debug "Deleting $resource from the CIB"
|
||||||
|
if ! pcs resource disable "$resource" --wait=600; then
|
||||||
# We need to add --force because the cluster is in maintenance mode and the resource
|
echo_error "ERROR: resource $resource failed to be disabled"
|
||||||
# is unmanaged. The if serves to make this idempotent
|
exit 1
|
||||||
|
fi
|
||||||
pcs resource delete --force "$resource"
|
pcs resource delete --force "$resource"
|
||||||
else
|
else
|
||||||
log_debug "Service $service not found as a pacemaker resource, not trying to delete."
|
log_debug "Service $service not found as a pacemaker resource, not trying to delete."
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# At this stage all the pacemaker resources are removed from the CIB. Once we remove the
|
# We need to do a pcs resource cleanup here + crm_resource --wait to
|
||||||
# maintenance-mode those systemd resources will keep on running. They shall be systemd enabled
|
# make sure the cluster is in a clean state before we stop everything,
|
||||||
# via the puppet converge step later on
|
# upgrade and restart everything
|
||||||
pcs property set maintenance-mode=false
|
|
||||||
# We need to do a pcs resource cleanup here + crm_resource --wait to make sure the
|
|
||||||
# cluster is in a clean state before we stop everything, upgrade and restart everything
|
|
||||||
pcs resource cleanup
|
pcs resource cleanup
|
||||||
# We are making sure here that the cluster is stable before proceeding
|
# We are making sure here that the cluster is stable before proceeding
|
||||||
if ! timeout -k 10 600 crm_resource --wait; then
|
if ! timeout -k 10 600 crm_resource --wait; then
|
||||||
|
Loading…
Reference in New Issue
Block a user