3e03884c47
For overcloud commands involving Bifrost (e.g. overcloud provision), we use the Bifrost dynamic inventory. This runs bifrost_inventory.py which gathers node and port information for all bare metal hosts. Because this gets executed for each node in our inventory, it produces a large number of API requests to Ironic, which grows as the number of hosts increases. Halve the number of calls using the BIFROST_NODE_NAMES environment variable, which restricts fetching port information to just the node being queried. bifrost_inventory.py still performs one call for each node, which must be fixed separately in Bifrost [1]. Time to query overcloud nodes' hardware introspection data for 44 hosts: * before patch: 823 seconds * after patch: 415 seconds With the patch to Bifrost [1], this is further reduced to 46 seconds. [1] https://review.opendev.org/c/openstack/bifrost/+/882950 Change-Id: I341075115442a38c327e3ade74b81b1b9e7e85c6
157 lines
6.1 KiB
YAML
157 lines
6.1 KiB
YAML
---
|
|
# Use bifrost to inspect the overcloud nodes' hardware.
|
|
|
|
- name: Ensure the overcloud nodes' hardware is inspected
|
|
hosts: overcloud
|
|
tags:
|
|
- hardware-inspect
|
|
vars:
|
|
# Set to False to avoid waiting for the nodes to become active.
|
|
wait_inspected: True
|
|
wait_inspected_timeout: "{{ kolla_bifrost_inspection_timeout }}"
|
|
wait_inspected_interval: 10
|
|
# List of states from which we can get to inspecting.
|
|
inspectable_states:
|
|
- enroll
|
|
- manageable
|
|
- available
|
|
- inspect failed
|
|
# List of valid states while a node is being inspected.
|
|
inspecting_states:
|
|
- inspecting
|
|
- inspect wait
|
|
# Retries to use when using Ironic API and hitting node locked errors.
|
|
ironic_retries: 6
|
|
ironic_retry_interval: 5
|
|
seed_host: "{{ groups['seed'][0] }}"
|
|
gather_facts: no
|
|
tasks:
|
|
- name: Check the ironic node's initial provision state
|
|
command: >
|
|
docker exec bifrost_deploy
|
|
bash -c '
|
|
export OS_CLOUD=bifrost &&
|
|
export BIFROST_INVENTORY_SOURCE=ironic &&
|
|
export BIFROST_NODE_NAMES="{{ inventory_hostname }}" &&
|
|
ansible baremetal
|
|
--connection local
|
|
--inventory /etc/bifrost/inventory/
|
|
-e @/etc/bifrost/bifrost.yml
|
|
-e @/etc/bifrost/dib.yml
|
|
--limit {{ inventory_hostname }}
|
|
-m command
|
|
-a "baremetal node show {% raw %}{{ inventory_hostname }}{% endraw %} -f value -c provision_state"'
|
|
register: show_result
|
|
changed_when: False
|
|
delegate_to: "{{ seed_host }}"
|
|
vars:
|
|
# NOTE: Without this, the seed's ansible_host variable will not be
|
|
# respected when using delegate_to.
|
|
ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}"
|
|
|
|
- name: Set a fact containing the ironic node's initial provision state
|
|
set_fact:
|
|
initial_provision_state: "{{ show_result.stdout_lines[1] }}"
|
|
|
|
- name: Fail if the ironic node is in an unexpected provision state
|
|
fail:
|
|
msg: >
|
|
Ironic node for {{ inventory_hostname }} is in an unexpected
|
|
initial provision state: {{ initial_provision_state }}. Expected
|
|
states are: {{ inspectable_states | join(',') }}.
|
|
when: initial_provision_state not in inspectable_states
|
|
|
|
- name: Ensure the ironic node is manageable
|
|
command: >
|
|
docker exec bifrost_deploy
|
|
bash -c '
|
|
export OS_CLOUD=bifrost &&
|
|
export BIFROST_INVENTORY_SOURCE=ironic &&
|
|
export BIFROST_NODE_NAMES="{{ inventory_hostname }}" &&
|
|
ansible baremetal -vvvv
|
|
--connection local
|
|
--inventory /etc/bifrost/inventory/
|
|
-e @/etc/bifrost/bifrost.yml
|
|
-e @/etc/bifrost/dib.yml
|
|
--limit {{ inventory_hostname }}
|
|
-m command
|
|
-a "baremetal node manage {% raw %}{{ inventory_hostname }}{% endraw %}"'
|
|
register: manage_result
|
|
until: manage_result is successful or 'is locked by host' in manage_result.stdout
|
|
retries: "{{ ironic_retries }}"
|
|
delay: "{{ ironic_retry_interval }}"
|
|
when: initial_provision_state != 'manageable'
|
|
delegate_to: "{{ seed_host }}"
|
|
vars:
|
|
# NOTE: Without this, the seed's ansible_host variable will not be
|
|
# respected when using delegate_to.
|
|
ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}"
|
|
|
|
- name: Ensure the ironic node is inspected
|
|
command: >
|
|
docker exec bifrost_deploy
|
|
bash -c '
|
|
export OS_CLOUD=bifrost &&
|
|
export BIFROST_INVENTORY_SOURCE=ironic &&
|
|
export BIFROST_NODE_NAMES="{{ inventory_hostname }}" &&
|
|
ansible baremetal -vvvv
|
|
--connection local
|
|
--inventory /etc/bifrost/inventory/
|
|
-e @/etc/bifrost/bifrost.yml
|
|
-e @/etc/bifrost/dib.yml
|
|
--limit {{ inventory_hostname }}
|
|
-m command
|
|
-a "baremetal node inspect {% raw %}{{ inventory_hostname }}{% endraw %}"'
|
|
register: provide_result
|
|
until: provide_result is successful or 'is locked by host' in provide_result.stdout
|
|
retries: "{{ ironic_retries }}"
|
|
delay: "{{ ironic_retry_interval }}"
|
|
delegate_to: "{{ seed_host }}"
|
|
vars:
|
|
# NOTE: Without this, the seed's ansible_host variable will not be
|
|
# respected when using delegate_to.
|
|
ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}"
|
|
|
|
- name: Wait for the ironic node to be inspected
|
|
command: >
|
|
docker exec bifrost_deploy
|
|
bash -c '
|
|
export OS_CLOUD=bifrost &&
|
|
export BIFROST_INVENTORY_SOURCE=ironic &&
|
|
export BIFROST_NODE_NAMES="{{ inventory_hostname }}" &&
|
|
ansible baremetal
|
|
--connection local
|
|
--inventory /etc/bifrost/inventory/
|
|
-e @/etc/bifrost/bifrost.yml
|
|
-e @/etc/bifrost/dib.yml
|
|
--limit {{ inventory_hostname }}
|
|
-m command
|
|
-a "baremetal node show {% raw %}{{ inventory_hostname }}{% endraw %} -f value -c provision_state"'
|
|
register: show_result
|
|
# Wait until the node is no longer in one of the inspecting states.
|
|
until: not show_result.stdout_lines[1:] | intersect(inspecting_states)
|
|
retries: "{{ wait_inspected_timeout | int // wait_inspected_interval | int }}"
|
|
delay: "{{ wait_inspected_interval }}"
|
|
when: wait_inspected | bool
|
|
changed_when: False
|
|
delegate_to: "{{ seed_host }}"
|
|
vars:
|
|
# NOTE: Without this, the seed's ansible_host variable will not be
|
|
# respected when using delegate_to.
|
|
ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}"
|
|
|
|
- name: Set a fact containing the final provision state
|
|
set_fact:
|
|
final_provision_state: "{{ show_result.stdout_lines[1] }}"
|
|
when: wait_inspected | bool
|
|
|
|
- name: Fail if any of the nodes are not manageable
|
|
fail:
|
|
msg: >
|
|
Ironic node for {{ inventory_hostname }} is in an unexpected
|
|
provision state after inspecting. Ironic provision state:
|
|
{{ final_provision_state }}. Expected: manageable.
|
|
when:
|
|
- wait_inspected | bool
|
|
- final_provision_state != 'manageable'
|