Merge "HA: fix <service>_restart_bundle with minor update workflow" into stable/queens

This commit is contained in:
Zuul 2019-09-27 03:03:11 +00:00 committed by Gerrit Code Review
commit e225059694
10 changed files with 87 additions and 146 deletions

View File

@ -43,6 +43,12 @@ parameters:
description: Specifies the default CA cert to use if TLS is used for
services in the internal network.
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pacemaker resource to restart when
a config change is detected and the resource is being restarted
type: number
conditions:
internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]}
@ -83,6 +89,13 @@ outputs:
exit 0
fi
exit $rc
pacemaker_restart_bundle.sh:
mode: "0755"
content:
str_replace:
template: { get_file: ../../docker_config_scripts/pacemaker_restart_bundle.sh }
params:
__PCMKTIMEOUT__: {get_param: PcmkConfigRestartTimeout}
volumes_base:
description: Base volume list
@ -116,6 +129,17 @@ outputs:
# required for bootstrap_host_exec
- /etc/puppet:/etc/puppet:ro
pacemaker_restart_volumes:
description: Common volumes for the pacemaker restart containers.
value:
list_concat:
- *volumes_base
- - /var/lib/docker-config-scripts/pacemaker_restart_bundle.sh:/pacemaker_restart_bundle.sh:ro
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
# required for bootstrap_host_exec, facter
- /etc/puppet:/etc/puppet:ro
docker_puppet_apply_volumes:
description: Common volumes needed to run the docker_puppet_apply.sh from docker_config_scripts
value:

View File

@ -52,11 +52,6 @@ parameters:
default: false
description: Whether to run config management (e.g. Puppet) in debug mode.
type: boolean
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
DeployIdentifier:
default: ''
type: string
@ -190,22 +185,12 @@ outputs:
user: root
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'cinder_backup'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show openstack-cinder-backup; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT openstack-cinder-backup; echo "openstack-cinder-backup restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh openstack-cinder-backup cinder_backup
image: {get_param: DockerCinderBackupImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro
- {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
- - /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro
cinder_backup_init_bundle:
start_order: 1
detach: false

View File

@ -40,11 +40,6 @@ parameters:
default: false
description: Whether to run config management (e.g. Puppet) in debug mode.
type: boolean
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
DeployIdentifier:
default: ''
type: string
@ -175,22 +170,12 @@ outputs:
user: root
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'cinder_volume'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show openstack-cinder-volume; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT openstack-cinder-volume; echo "openstack-cinder-volume restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh openstack-cinder-volume cinder_volume
image: {get_param: DockerCinderVolumeImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro
- {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
- - /var/lib/config-data/puppet-generated/cinder/:/var/lib/kolla/config_files/src:ro
cinder_volume_init_bundle:
start_order: 1
detach: false

View File

@ -61,11 +61,6 @@ parameters:
description: >
Setting this to a unique value will re-run any deployment tasks which
perform configuration on a Heat stack-update.
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
resources:
@ -261,22 +256,12 @@ outputs:
user: root
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'mysql'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show galera-bundle; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT galera-bundle; echo "galera-bundle restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh galera-bundle mysql
image: {get_param: DockerMysqlImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/mysql/:/var/lib/kolla/config_files/src:ro
- {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
- - /var/lib/config-data/puppet-generated/mysql/:/var/lib/kolla/config_files/src:ro
mysql_init_bundle:
start_order: 1
detach: false

View File

@ -47,11 +47,6 @@ parameters:
default: false
description: Enable IPv6 in Redis
type: boolean
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
DeployIdentifier:
default: ''
type: string
@ -222,22 +217,12 @@ outputs:
user: root
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'redis'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show redis-bundle; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT redis-bundle; echo "redis-bundle restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh redis-bundle redis
image: {get_param: DockerRedisConfigImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro
- {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
- - /var/lib/config-data/puppet-generated/redis/:/var/lib/kolla/config_files/src:ro
- redis_init_bundle:
start_order: 2
detach: false

View File

@ -79,11 +79,6 @@ parameters:
default: false
description: Whether to run config management (e.g. Puppet) in debug mode.
type: boolean
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
DeployIdentifier:
default: ''
type: string
@ -238,22 +233,12 @@ outputs:
config_volume: haproxy
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'haproxy'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show haproxy-bundle; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT haproxy-bundle; echo "haproxy-bundle restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh haproxy-bundle haproxy
image: {get_param: DockerHAProxyImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/haproxy/:/var/lib/kolla/config_files/src:ro
- {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
- - /var/lib/config-data/puppet-generated/haproxy/:/var/lib/kolla/config_files/src:ro
haproxy_init_bundle:
start_order: 3
detach: false

View File

@ -40,11 +40,6 @@ parameters:
default: false
description: Whether to run config management (e.g. Puppet) in debug mode.
type: boolean
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
DeployIdentifier:
default: ''
type: string
@ -169,22 +164,12 @@ outputs:
user: root
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'manila_share'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show openstack-manila-share; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT openstack-manila-share; echo "openstack-manila-share restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh openstack-manila-share manila_share
image: {get_param: DockerManilaShareImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/manila/:/var/lib/kolla/config_files/src:ro
- {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
- - /var/lib/config-data/puppet-generated/manila/:/var/lib/kolla/config_files/src:ro
manila_share_init_bundle:
start_order: 1
detach: false

View File

@ -48,11 +48,6 @@ parameters:
default: false
description: Whether to run config management (e.g. Puppet) in debug mode.
type: boolean
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
DeployIdentifier:
default: ''
type: string
@ -154,22 +149,9 @@ outputs:
user: root
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'ovn_dbs'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show ovn-dbs-bundle; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT ovn-dbs-bundle; echo "ovn-dbs-bundle restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh ovn-dbs-bundle ovn_dbs
image: {get_param: DockerOvnDbsConfigImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/ovn_dbs/:/var/lib/kolla/config_files/src:ro
volumes: {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
ovn_dbs_init_bundle:
start_order: 1
detach: false

View File

@ -44,11 +44,6 @@ parameters:
default: false
description: Whether to run config management (e.g. Puppet) in debug mode.
type: boolean
PcmkConfigRestartTimeout:
default: 600
description: Time in seconds to wait for a pcmk resource to restart when
a config change is detected and the resource is being restarted
type: number
DeployIdentifier:
default: ''
type: string
@ -209,22 +204,12 @@ outputs:
user: root
environment:
- TRIPLEO_MINOR_UPDATE
command:
- '/usr/bin/bootstrap_host_exec'
- 'rabbitmq'
- str_replace:
template:
'if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ] && /usr/sbin/pcs resource show rabbitmq-bundle; then /usr/sbin/pcs resource restart --wait=PCMKTIMEOUT rabbitmq-bundle; echo "rabbitmq-bundle restart invoked"; fi'
params:
PCMKTIMEOUT: {get_param: PcmkConfigRestartTimeout}
command: /pacemaker_restart_bundle.sh rabbitmq-bundle rabbitmq
image: {get_param: DockerRabbitmqImage}
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
-
- /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro
- /dev/shm:/dev/shm:rw
- /var/lib/config-data/puppet-generated/rabbitmq/:/var/lib/kolla/config_files/src:ro
- {get_attr: [ContainersCommon, pacemaker_restart_volumes]}
- - /var/lib/config-data/puppet-generated/rabbitmq/:/var/lib/kolla/config_files/src:ro
rabbitmq_init_bundle:
start_order: 1
detach: false

View File

@ -0,0 +1,40 @@
#!/bin/bash
set -u
# ./pacemaker_restart_bundle.sh galera-bundle galera
RESOURCE=$1
TRIPLEO_SERVICE=$2
# try to restart only if resource has been created already
if /usr/sbin/pcs resource show $RESOURCE; then
if [ x"${TRIPLEO_MINOR_UPDATE,,}" != x"true" ]; then
# During a stack update, this script is called in parallel on
# every node the resource runs on, after the service's configs
# have been updated on all nodes. So we need to run pcs only
# once (e.g. on the service's boostrap node).
echo "$(date -u): Restarting ${RESOURCE} globally"
/usr/bin/bootstrap_host_exec $TRIPLEO_SERVICE /sbin/pcs resource restart --wait=__PCMKTIMEOUT__ $RESOURCE
else
# During a minor update workflow however, a host gets fully
# updated before updating the next one. So unlike stack
# update, at the time this script is called, the service's
# configs aren't updated on all nodes yet. So only restart the
# resource locally, where it's guaranteed that the config is
# up to date.
HOST=$(facter hostname)
# XPath rationale: as long as there is a bundle running
# locally and it is managed by pacemaker, no matter the state
# of any inner pcmk_remote or ocf resource, we should restart
# it to give it a chance to read the new config.
# XPath rationale 2: if the resource is being stopped, the
# attribute "target_role" will be present in the output of
# crm_mon. Do not restart the resource if that is the case.
if crm_mon -r --as-xml | xmllint --format --xpath "//bundle[@id='${RESOURCE}']/replica/resource[@managed='true' and (not(boolean(@target_role)) or (boolean(@target_role) and @target_role!='Stopped'))]/node[@name='${HOST}']/../.." - &>/dev/null; then
echo "$(date -u): Restarting ${RESOURCE} locally on '${HOST}'"
/sbin/pcs resource restart --wait=__PCMKTIMEOUT__ $RESOURCE "${HOST}"
else
echo "$(date -u): Resource ${RESOURCE} currently not running on '${HOST}', no restart needed"
fi
fi
fi