openstack/tripleo-heat-templates
Code Issues Proposed changes

Prevent service bootstrap node facts from colliding with each other

Browse Source 
Many services currently set an `is_bootstrap_node` fact, meaning they
override each other's results when the fact is being set. As long as
the fact doesn't belong into a particular step but it's executed on
every step, nothing bad happens, as the correct is_bootstrap_node
setting directly precedes any service upgrade tasks. However, we
intend to put the fact setting into step 0 in change
Ib04b051e8f4275e06be0cafa81e2111c9cced9b7 and at that point the name
collision would break upgrades (only one service would "win" in
setting the is_bootstrap_node fact).

This patch changes the is_bootstrap_node facts in upgrade_tasks to use
per-service naming.

Note that fast_forward_upgrade_tasks use their own is_boostrap_node
logic. We've uncovered some weirdness there while looking into the
is_boostrap_node issue, but the fix is not a low hanging fruit and
likely we'll be completely redoing the FFU tasks for Q->T
upgrade. So the FFU tasks are left alone for now.

Change-Id: I9c585d3cb282b7e4eb0bacb3cf6909e04a9a495e
Closes-Bug: #1810408
This commit is contained in:
Jiri Stransky 2019-01-03 13:50:09 +01:00
parent 697cefb95e
commit 54fb81ecd9
16 changed files with 71 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
deployment/ironic/ironic-api-container-puppet.yaml
View File

@ -266,8 +266,8 @@ outputs:
- when: step|int == 0
tags: common
block:
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{ironic_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_ironic_api_bootstrap_node fact
set_fact: is_ironic_api_bootstrap_node={{ironic_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Ensure all online data migrations for Ironic have been applied
shell: |
if {{ container_cli }} ps | grep ironic_api; then
@ -278,7 +278,7 @@ outputs:
docker exec ironic_api ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations
fi
tags: pre-upgrade
when: is_bootstrap_node|bool
when: is_ironic_api_bootstrap_node|bool
- when: step|int == 3
block:
- name: Set fact for removal of openstack-ironic-api package

6
docker/services/cinder-api.yaml
View File

@ -228,8 +228,8 @@ outputs:
- when: step|int == 0
tags: common
block:
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{cinder_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_cinder_api_bootstrap_node fact
set_fact: is_cinder_api_bootstrap_node={{cinder_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Ensure all online data migrations for Cinder have been applied
shell: |
if {{ container_cli }} ps | grep cinder_api; then
@ -240,7 +240,7 @@ outputs:
docker exec cinder_api cinder-manage db online_data_migrations
fi
tags: pre-upgrade
when: is_bootstrap_node|bool
when: is_cinder_api_bootstrap_node|bool
- when: step|int == 3
block:
- name: Set fact for removal of httpd package

6
docker/services/nova-api.yaml
View File

@ -325,8 +325,8 @@ outputs:
- when: step|int == 0
tags: common
block:
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{nova_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_nova_api_bootstrap_node fact
set_fact: is_nova_api_bootstrap_node={{nova_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Ensure all online data migrations for Nova have been applied
shell: |
if {{ container_cli }} ps | grep nova_api; then
@ -337,7 +337,7 @@ outputs:
docker exec nova_api nova-manage db online_data_migrations
fi
tags: pre-upgrade
when: is_bootstrap_node|bool
when: is_nova_api_bootstrap_node|bool
- when: step|int == 3
block:
- name: Set fact for removal of openstack-nova-api package

8
docker/services/pacemaker/cinder-backup.yaml
View File

@ -324,9 +324,9 @@ outputs:
- name: Set fact cinder_backup_pcs_res
set_fact:
cinder_backup_pcs_res: "{{cinder_backup_pcs_res_result|succeeded}}"
- name: set is_bootstrap_node fact
- name: set is_cinder_backup_bootstrap_node fact
tags: common
set_fact: is_bootstrap_node={{cinder_backup_short_bootstrap_node_name|lower == ansible_hostname|lower}}
set_fact: is_cinder_backup_bootstrap_node={{cinder_backup_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Cinder-Backup baremetal to container upgrade tasks
when:
- step|int == 1
@ -339,7 +339,7 @@ outputs:
check_mode: false
ignore_errors: true
register: cinder_backup_res
- when: (is_bootstrap_node) and (cinder_backup_res|succeeded)
- when: (is_cinder_backup_bootstrap_node) and (cinder_backup_res|succeeded)
block:
- name: Disable the openstack-cinder-backup cluster resource
pacemaker_resource:
@ -363,7 +363,7 @@ outputs:
when:
- step|int == 1
- cinder_backup_containerized|bool
- is_bootstrap_node
- is_cinder_backup_bootstrap_node
- cinder_backup_pcs_res|bool
block:
- name: Disable the cinder_backup cluster resource before container upgrade

8
docker/services/pacemaker/cinder-volume.yaml
View File

@ -290,9 +290,9 @@ outputs:
- name: Set fact cinder_volume_pcs_res
set_fact:
cinder_volume_pcs_res: "{{cinder_volume_pcs_res_result|succeeded}}"
- name: set is_bootstrap_node fact
- name: set is_cinder_volume_bootstrap_node fact
tags: common
set_fact: is_bootstrap_node={{cinder_volume_short_bootstrap_node_name|lower == ansible_hostname|lower}}
set_fact: is_cinder_volume_bootstrap_node={{cinder_volume_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Cinder-Volume baremetal to container upgrade tasks
when:
- step|int == 1
@ -305,7 +305,7 @@ outputs:
check_mode: false
ignore_errors: true
register: cinder_volume_res
- when: (is_bootstrap_node) and (cinder_volume_res|succeeded)
- when: (is_cinder_volume_bootstrap_node) and (cinder_volume_res|succeeded)
block:
- name: Disable the openstack-cinder-volume cluster resource
pacemaker_resource:
@ -329,7 +329,7 @@ outputs:
when:
- step|int == 1
- cinder_volume_containerized|bool
- is_bootstrap_node
- is_cinder_volume_bootstrap_node
- cinder_volume_pcs_res|bool
block:
- name: Disable the cinder_volume cluster resource before container upgrade

8
docker/services/pacemaker/database/mysql.yaml
View File

@ -360,8 +360,8 @@ outputs:
- name: Check if Mysql is already containerized
set_fact:
mysql_containerized: "{{mysql_kolla_config.stat.isdir | default(false)}}"
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{mysql_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_mysql_bootstrap_node fact
set_fact: is_mysql_bootstrap_node={{mysql_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Prepare the switch to new galera container image name in pacemaker
when: mysql_containerized|bool
block:
@ -397,7 +397,7 @@ outputs:
check_mode: true
ignore_errors: true
register: galera_res
- when: (is_bootstrap_node) and (galera_res|succeeded)
- when: (is_mysql_bootstrap_node) and (galera_res|succeeded)
block:
- name: Disable the galera cluster resource
pacemaker_resource:
@ -425,7 +425,7 @@ outputs:
when:
- step|int == 1
- mysql_containerized|bool
- is_bootstrap_node
- is_mysql_bootstrap_node
- galera_pcs_res|bool
block:
- name: Disable the galera cluster resource before container upgrade

8
docker/services/pacemaker/database/redis.yaml
View File

@ -339,8 +339,8 @@ outputs:
- name: Check if redis is already containerized
set_fact:
redis_containerized: "{{redis_kolla_config.stat.isdir | default(false)}}"
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{redis_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_redis_bootstrap_node fact
set_fact: is_redis_bootstrap_node={{redis_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Prepare the switch to new redis container image name in pacemaker
when: redis_containerized|bool
block:
@ -376,7 +376,7 @@ outputs:
check_mode: false
ignore_errors: true
register: redis_res
- when: (is_bootstrap_node) and (redis_res|succeeded)
- when: (is_redis_bootstrap_node) and (redis_res|succeeded)
block:
- name: Disable the redis cluster resource
pacemaker_resource:
@ -400,7 +400,7 @@ outputs:
when:
- step|int == 1
- redis_containerized|bool
- is_bootstrap_node
- is_redis_bootstrap_node
- redis_pcs_res|bool
block:
- name: Disable the redis cluster resource before container upgrade

10
docker/services/pacemaker/haproxy.yaml
View File

@ -344,14 +344,14 @@ outputs:
- name: Check if haproxy is already containerized
set_fact:
haproxy_containerized: "{{haproxy_kolla_config.stat.isdir | default(false)}}"
- name: set is_bootstrap_node fact
- name: set is_haproxy_bootstrap_node fact
tags: common
set_fact: is_bootstrap_node={{haproxy_short_bootstrap_node_name|lower == ansible_hostname|lower}}
set_fact: is_haproxy_bootstrap_node={{haproxy_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Mount TLS cert if needed
when:
- step|int == 1
- haproxy_containerized|bool
- is_bootstrap_node
- is_haproxy_bootstrap_node
block:
- name: Check haproxy public certificate configuration in pacemaker
command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-cert']"
@ -459,7 +459,7 @@ outputs:
check_mode: true
ignore_errors: true
register: haproxy_res
- when: (is_bootstrap_node) and (haproxy_res|succeeded)
- when: (is_haproxy_bootstrap_node) and (haproxy_res|succeeded)
block:
- name: Disable the haproxy cluster resource.
pacemaker_resource:
@ -481,7 +481,7 @@ outputs:
when:
- step|int == 1
- haproxy_containerized|bool
- is_bootstrap_node
- is_haproxy_bootstrap_node
- haproxy_pcs_res|bool
block:
- name: Disable the haproxy cluster resource before container upgrade

8
docker/services/pacemaker/manila-share.yaml
View File

@ -291,9 +291,9 @@ outputs:
- name: Set fact manila_share_pcs_res
set_fact:
manila_share_pcs_res: "{{manila_share_pcs_res_result|succeeded}}"
- name: set is_bootstrap_node fact
- name: set is_manila_share_bootstrap_node fact
tags: common
set_fact: is_bootstrap_node={{manila_share_short_bootstrap_node_name|lower == ansible_hostname|lower}}
set_fact: is_manila_share_bootstrap_node={{manila_share_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Manila-Share baremetal to container upgrade tasks
when:
- step|int == 1
@ -306,7 +306,7 @@ outputs:
check_mode: false
ignore_errors: true
register: manila_share_res
- when: (is_bootstrap_node) and (manila_share_res|succeeded)
- when: (is_manila_share_bootstrap_node) and (manila_share_res|succeeded)
block:
- name: Disable the openstack-manila-share cluster resource
pacemaker_resource:
@ -330,7 +330,7 @@ outputs:
when:
- step|int == 1
- manila_share_containerized|bool
- is_bootstrap_node
- is_manila_share_bootstrap_node
- manila_share_pcs_res|bool
block:
- name: Disable the Manila-Share cluster resource before container upgrade

8
docker/services/pacemaker/notify-rabbitmq.yaml
View File

@ -313,8 +313,8 @@ outputs:
- name: Check if Rabbitmq is already containerized
set_fact:
rabbit_containerized: "{{rabbit_kolla_config.stat.isdir | default(false)}}"
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{notify_rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_notify_rabbitmq_bootstrap_node fact
set_fact: is_notify_rabbitmq_bootstrap_node={{notify_rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Rabbitmq baremetal to container upgrade tasks
when:
- step|int == 1
@ -327,7 +327,7 @@ outputs:
state: show
ignore_errors: true
register: rabbitmq_res
- when: (is_bootstrap_node) and (rabbitmq_res|succeeded)
- when: (is_notify_rabbitmq_bootstrap_node) and (rabbitmq_res|succeeded)
block:
- name: Disable the rabbitmq cluster resource.
pacemaker_resource:
@ -351,7 +351,7 @@ outputs:
when:
- step|int == 1
- rabbit_containerized|bool
- is_bootstrap_node
- is_notify_rabbitmq_bootstrap_node
block:
- name: Check rabbitmq logging configuration in pacemaker
command: cibadmin --query --xpath "//storage-mapping[@id='rabbitmq-log']"

12
docker/services/pacemaker/ovn-dbs.yaml
View File

@ -214,8 +214,8 @@ outputs:
- name: Check if ovn-dbs is already containerized
set_fact:
ovn_dbs_containerized: "{{ovn_dbs_log_file.stat.exists | default(false)}}"
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{ovn_dbs_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_ovn_dbs_bootstrap_node fact
set_fact: is_ovn_dbs_bootstrap_node={{ovn_dbs_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: ovn-dbs fetch and retag container image for pacemaker
when:
- step|int == 3
@ -255,7 +255,7 @@ outputs:
when:
- step|int == 5
- ovn_dbs_containerized|bool
- is_bootstrap_node
- is_ovn_dbs_bootstrap_node
block:
- name: Get the present image used by ovn-dbs-bundle
shell: "pcs resource show ovn-dbs-bundle | grep image | awk '{ split($2, image, \"=\"); print image[2] }'"
@ -281,8 +281,8 @@ outputs:
set_fact:
ovn_dbs_docker_image: {get_param: DockerOvnDbsImage}
ovn_dbs_docker_image_latest: *ovn_dbs_image_pcmklatest
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{ovn_dbs_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_ovn_dbs_bootstrap_node fact
set_fact: is_ovn_dbs_bootstrap_node={{ovn_dbs_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Prepare the switch to new ovn-dbs container image name in pacemaker
when:
- step|int == 0
@ -326,7 +326,7 @@ outputs:
when:
- step|int == 1
- ovn_dbs_containerized|bool
- is_bootstrap_node
- is_ovn_dbs_bootstrap_node
- ovndbs_pcs_res|succeeded
block: *ovn_dbs_update_bundle_with_new_image
- name: Retag the pacemaker image if containerized

8
docker/services/pacemaker/rabbitmq.yaml
View File

@ -309,8 +309,8 @@ outputs:
- name: Check if Rabbitmq is already containerized
set_fact:
rabbit_containerized: "{{rabbit_kolla_config.stat.isdir | default(false)}}"
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_rabbitmq_bootstrap_node fact
set_fact: is_rabbitmq_bootstrap_node={{rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Rabbitmq baremetal to container upgrade tasks
when:
- step|int == 1
@ -323,7 +323,7 @@ outputs:
state: show
ignore_errors: true
register: rabbitmq_res
- when: (is_bootstrap_node) and (rabbitmq_res|succeeded)
- when: (is_rabbitmq_bootstrap_node) and (rabbitmq_res|succeeded)
block:
- name: Disable the rabbitmq cluster resource.
pacemaker_resource:
@ -369,7 +369,7 @@ outputs:
when:
- step|int == 1
- rabbit_containerized|bool
- is_bootstrap_node
- is_rabbitmq_bootstrap_node
- rabbit_pcs_res|succeeded
block:
- name: Disable the rabbitmq cluster resource before container upgrade

8
docker/services/pacemaker/rpc-rabbitmq.yaml
View File

@ -310,8 +310,8 @@ outputs:
- name: Check if Rabbitmq is already containerized
set_fact:
rabbit_containerized: "{{rabbit_kolla_config.stat.isdir | default(false)}}"
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{rpc_rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: set is_rpc_rabbitmq_bootstrap_node fact
set_fact: is_rpc_rabbitmq_bootstrap_node={{rpc_rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Rabbitmq baremetal to container upgrade tasks
when:
- step|int == 1
@ -324,7 +324,7 @@ outputs:
state: show
ignore_errors: true
register: rabbitmq_res
- when: (is_bootstrap_node) and (rabbitmq_res|succeeded)
- when: (is_rpc_rabbitmq_bootstrap_node) and (rabbitmq_res|succeeded)
block:
- name: Disable the rabbitmq cluster resource.
pacemaker_resource:
@ -370,7 +370,7 @@ outputs:
when:
- step|int == 1
- rabbit_containerized|bool
- is_bootstrap_node
- is_rpc_rabbitmq_bootstrap_node
- rabbit_pcs_res|succeeded
block:
- name: Disable the rabbitmq cluster resource before container upgrade

6
puppet/services/gnocchi-api.yaml
View File

@ -188,11 +188,11 @@ outputs:
- name: Stop gnocchi_api service (running under httpd)
when: step|int == 1
service: name=httpd state=stopped
- name: set is_bootstrap_node fact
- name: set is_gnocchi_api_bootstrap_node fact
tags: common
set_fact: is_bootstrap_node={{gnocchi_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
set_fact: is_gnocchi_api_bootstrap_node={{gnocchi_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Setup gnocchi db during upgrade
command: gnocchi-upgrade
when:
- step|int == 5
- is_bootstrap_node|bool
- is_gnocchi_api_bootstrap_node|bool

6
puppet/services/haproxy-public-tls-inject.yaml
View File

@ -84,8 +84,8 @@ outputs:
stat:
path: "{{cert_path}}"
- name: set is_bootstrap_node fact
set_fact: is_bootstrap_node={{haproxy_short_bootstrap_node_name == ansible_hostname}}
- name: set is_haproxy_bootstrap_node fact
set_fact: is_haproxy_bootstrap_node={{haproxy_short_bootstrap_node_name == ansible_hostname}}
- name: get haproxy status
register: haproxy_state
@ -113,7 +113,7 @@ outputs:
- name: get pcs resource name for haproxy container
when:
- bootstrap_node is defined
- is_bootstrap_node
- is_haproxy_bootstrap_node
- pacemaker_state.status.ActiveState == 'active'
- attr_cert_path.stat.exists
- attr_cert_path.stat.isdir

26
puppet/services/nova-api.yaml
View File

@ -249,14 +249,14 @@ outputs:
list_concat:
- get_attr: [ApacheServiceBase, role_data, upgrade_tasks]
-
- name: set is_bootstrap_node fact
- name: set is_nova_api_bootstrap_node fact
tags: common
set_fact: is_bootstrap_node={{nova_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
set_fact: is_nova_api_bootstrap_node={{nova_api_short_bootstrap_node_name|lower == ansible_hostname|lower}}
- name: Extra migration for nova tripleo/+bug/1656791
tags: pre-upgrade
when:
- step|int == 0
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
command: nova-manage db online_data_migrations
- name: Stop and disable nova_api service (pre-upgrade not under httpd)
when: step|int == 2
@ -264,7 +264,7 @@ outputs:
- name: Create puppet manifest to set transport_url in nova.conf
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
copy:
dest: /root/nova-api_upgrade_manifest.pp
mode: 0600
@ -283,7 +283,7 @@ outputs:
- name: Run puppet apply to set tranport_url in nova.conf
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
command: puppet apply --summarize --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules --detailed-exitcodes /root/nova-api_upgrade_manifest.pp
register: puppet_apply_nova_api_upgrade
failed_when: puppet_apply_nova_api_upgrade.rc not in [0,2]
@ -291,12 +291,12 @@ outputs:
- name: Setup cell_v2 (map cell0)
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
shell: nova-manage cell_v2 map_cell0 --database_connection=$(hiera nova::cell0_database_connection)
- name: Setup cell_v2 (create default cell)
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
# (owalsh) puppet-nova expects the cell name 'default'
# (owalsh) pass the db uri explicitly to avoid https://bugs.launchpad.net/tripleo/+bug/1662344
shell: nova-manage cell_v2 create_cell --name='default' --database_connection=$(hiera nova::database_connection)
@ -306,33 +306,33 @@ outputs:
- name: Setup cell_v2 (sync nova/cell DB)
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
command: nova-manage db sync
async: {get_param: NovaDbSyncTimeout}
poll: 10
- name: Setup cell_v2 (get cell uuid)
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
shell: nova-manage cell_v2 list_cells | sed -e '1,3d' -e '$d' | awk -F ' *| *' '$2 == "default" {print $4}'
register: nova_api_cell_uuid
- name: Setup cell_v2 (migrate hosts)
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
command: nova-manage cell_v2 discover_hosts --cell_uuid {{nova_api_cell_uuid.stdout}} --verbose
- name: Setup cell_v2 (migrate instances)
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
command: nova-manage cell_v2 map_instances --cell_uuid {{nova_api_cell_uuid.stdout}}
- name: Sync nova_api DB
command: nova-manage api_db sync
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
- name: Online data migration for nova
when:
- step|int == 5
- is_bootstrap_node|bool
- is_nova_api_bootstrap_node|bool
command: nova-manage db online_data_migrations