kayobe/ansible/overcloud-deprovision.yml
Pierre Riteau c7aaecfc38 Speed up calls to Bifrost
For overcloud commands involving Bifrost (e.g. overcloud provision), we
use the Bifrost dynamic inventory. This runs bifrost_inventory.py which
gathers node and port information for all bare metal hosts. Because this
gets executed for each node in our inventory, it produces a large number
of API requests to Ironic, which grows as the number of hosts increases.

Halve the number of calls using the BIFROST_NODE_NAMES environment
variable, which restricts fetching port information to just the node
being queried. bifrost_inventory.py still performs one call for each
node, which must be fixed separately in Bifrost [1].

Time to query overcloud nodes' hardware introspection data for 44 hosts:

* before patch: 823 seconds
* after patch: 415 seconds

With the patch to Bifrost [1], this is further reduced to 46 seconds.

[1] https://review.opendev.org/c/openstack/bifrost/+/882950

Change-Id: I341075115442a38c327e3ade74b81b1b9e7e85c6
(cherry picked from commit 3e03884c47)
2023-08-30 14:33:24 +00:00

144 lines
5.4 KiB
YAML

---
# Use bifrost to deprovision the overcloud nodes.
- name: Ensure the overcloud nodes are deprovisioned
hosts: overcloud
tags:
- deprovision
vars:
# Set to False to avoid waiting for the nodes to become active.
wait_available: True
wait_available_timeout: 600
wait_available_interval: 10
# List of states from which we can get to available.
deprovisionable_states:
- available
- active
- error
- wait call-back
- deploying
- deploy failed
# List of valid states while a node is being deprovisioned.
deleting_states:
# The API is asynchronous, so allow the initial state.
- active
- deleting
- cleaning
- clean wait
# Retries to use when using Ironic API and hitting node locked errors.
ironic_retries: 6
ironic_retry_interval: 5
seed_host: "{{ groups['seed'][0] }}"
gather_facts: no
tasks:
- name: Check the ironic node's initial provision state
command: >
docker exec bifrost_deploy
bash -c '
export OS_CLOUD=bifrost &&
export OS_BAREMETAL_API_VERSION=1.34 &&
export BIFROST_INVENTORY_SOURCE=ironic &&
export BIFROST_NODE_NAMES="{{ inventory_hostname }}" &&
ansible baremetal
--connection local
--inventory /etc/bifrost/inventory/
-e @/etc/bifrost/bifrost.yml
-e @/etc/bifrost/dib.yml
--limit {{ inventory_hostname }}
-m command
-a "baremetal node show {% raw %}{{ inventory_hostname }}{% endraw %} -f value -c provision_state"'
register: show_result
changed_when: False
delegate_to: "{{ seed_host }}"
vars:
# NOTE: Without this, the seed's ansible_host variable will not be
# respected when using delegate_to.
ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}"
- name: Set a fact containing the ironic node's initial provision state
set_fact:
initial_provision_state: "{{ show_result.stdout_lines[1] }}"
- name: Fail if the ironic node is in an unexpected provision state
fail:
msg: >
Ironic node for {{ inventory_hostname }} is in an unexpected
initial provision state: {{ initial_provision_state }}. Expected
states are: {{ deprovisionable_states | join(',') }}.
when: initial_provision_state not in deprovisionable_states
- name: Ensure the ironic node is deprovisioned
command: >
docker exec bifrost_deploy
bash -c '
export OS_CLOUD=bifrost &&
export BIFROST_INVENTORY_SOURCE=ironic &&
export BIFROST_NODE_NAMES="{{ inventory_hostname }}" &&
ansible baremetal -vvvv
--connection local
--inventory /etc/bifrost/inventory/
-e @/etc/bifrost/bifrost.yml
-e @/etc/bifrost/dib.yml
--limit {{ inventory_hostname }}
-m command
-a "baremetal node undeploy {% raw %}{{ inventory_hostname }}{% endraw %}"'
register: delete_result
until: delete_result is successful or 'is locked by host' in delete_result.stdout
retries: "{{ ironic_retries }}"
delay: "{{ ironic_retry_interval }}"
when: initial_provision_state != 'available'
delegate_to: "{{ seed_host }}"
vars:
# NOTE: Without this, the seed's ansible_host variable will not be
# respected when using delegate_to.
ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}"
- name: Wait for the ironic node to become available
command: >
docker exec bifrost_deploy
bash -c '
export OS_CLOUD=bifrost &&
export OS_BAREMETAL_API_VERSION=1.34 &&
export BIFROST_INVENTORY_SOURCE=ironic &&
export BIFROST_NODE_NAMES="{{ inventory_hostname }}" &&
ansible baremetal
--connection local
--inventory /etc/bifrost/inventory/
-e @/etc/bifrost/bifrost.yml
-e @/etc/bifrost/dib.yml
--limit {{ inventory_hostname }}
-m command
-a "baremetal node show {% raw %}{{ inventory_hostname }}{% endraw %} -f value -c provision_state"'
register: show_result
# Wait until the node is no longer in one of the deleting states.
until: not show_result.stdout_lines[1:] | intersect(deleting_states)
retries: "{{ wait_available_timeout // wait_available_interval }}"
delay: "{{ wait_available_interval }}"
when:
- wait_available | bool
- initial_provision_state != 'available'
changed_when: False
delegate_to: "{{ seed_host }}"
vars:
# NOTE: Without this, the seed's ansible_host variable will not be
# respected when using delegate_to.
ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}"
- name: Set a fact containing the ironic node's final provision state
set_fact:
final_provision_state: "{{ show_result.stdout_lines[1] }}"
when:
- wait_available | bool
- initial_provision_state != 'available'
- name: Fail if the ironic node is not available
fail:
msg: >
Ironic node for {{ inventory_hostname }} is in an unexpected
provision state after deprovisioning. Ironic provision state:
{{ final_provision_state }}. Expected: available.
when:
- wait_available | bool
- initial_provision_state != 'available'
- final_provision_state != 'available'