Introduce max fail percentage to playbooks

This allows us to continue execution until a certain proportion of hosts
fail. This can be useful at scale, where failures are common, and
restarting a deployment is time-consuming.

The default max failure percentage is 100, keeping the default
behaviour. A global max failure percentage may be set via
kayobe_max_fail_percentage, and individual playbooks may define a max
failure percentage via <playbook>_max_fail_percentage.

Related Kolla Ansible patch:
https://review.opendev.org/c/openstack/kolla-ansible/+/805598

Change-Id: Ib81c72b63be5765cca664c38141ffc769640cf07

ansible/apparmor-libvirt.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure AppArmor is disabled for containerised libvirt
   hosts: compute
+  max_fail_percentage: >-
+    {{ apparmor_libvirt_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - apparmor-libvirt
   vars:

ansible/apt.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure APT is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ apt_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:

ansible/baremetal-compute-inspect.yml

@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are inspected in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_inspect_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     controller_host: "{{ groups['controllers'][0] }}"
     venv: "{{ virtualenv_path }}/openstacksdk"

ansible/baremetal-compute-introspection-data-save.yml

@@ -18,6 +18,11 @@
 - name: Ensure the baremetal compute nodes' hardware introspection data is saved
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_introspection_data_save_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"

ansible/baremetal-compute-manage.yml

@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are manageable in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_manage_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstacksdk"
     ansible_python_interpreter: "{{ venv }}/bin/python"

ansible/baremetal-compute-provide.yml

@@ -20,6 +20,11 @@
 - name: Ensure baremetal compute nodes are available in ironic
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_provide_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstacksdk"
     ansible_python_interpreter: "{{ venv }}/bin/python"

ansible/baremetal-compute-register.yml

@@ -18,6 +18,11 @@
 - name: Ensure baremetal compute nodes are registered in ironic
   hosts: baremetal-compute
   gather_facts: false
+  max_fail_percentage: >-
+    {{ baremetal_compute_register_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - baremetal
   vars:

ansible/baremetal-compute-rename.yml

@@ -21,6 +21,11 @@
 - name: Rename baremetal compute nodes
   hosts: baremetal-compute
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_rename_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"

ansible/baremetal-compute-serial-console.yml

@@ -53,6 +53,11 @@
 - name: Enable serial console
   hosts: "{{ console_compute_node_limit | default('baremetal-compute') }}"
   gather_facts: False
+  max_fail_percentage: >-
+    {{ baremetal_compute_serial_console_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     venv: "{{ virtualenv_path }}/openstack-cli"
     controller_host: "{{ groups['controllers'][0] }}"

ansible/compute-libvirt-host.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure the libvirt daemon is configured
   hosts: compute
+  max_fail_percentage: >-
+    {{ compute_libvirt_host_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - libvirt-host
   tasks:

ansible/compute-node-discovery.yml

@@ -4,6 +4,11 @@
 - name: Ensure baremetal compute nodes are PXE booted
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ compute_node_discovery_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     controller_host: "{{ groups['controllers'][0] }}"
   tasks:

ansible/dell-compute-node-boot-mode.yml

@@ -7,6 +7,11 @@
 - name: Ensure Dell baremetal compute nodes boot mode is set
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_boot_mode_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to the required boot mode. One of 'bios' or 'uefi'.
     drac_boot_mode: "bios"

ansible/dell-compute-node-discovery.yml

@@ -7,6 +7,11 @@
 - name: Ensure Dell baremetal compute nodes are PXE booted
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_discovery_max_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to the index of the inteface on which to enable PXE.
     drac_pxe_interface: 1

ansible/dell-compute-node-inventory.yml

@@ -28,6 +28,11 @@
 - name: Ensure Dell baremetal compute nodes are present in the Ansible inventory
   hosts: baremetal-compute
   gather_facts: no
+  max_fail_percentage: >-
+    {{ dell_compute_node_inventory_fail_percentage |
+       default(baremetal_compute_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     compute_node_limit: ""
     compute_node_limit_list: "{{ compute_node_limit.split(':') }}"

ansible/dev-tools.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure development tools are installed
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ dev_tools_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - dev-tools
   roles:

ansible/disable-cloud-init.yml

@@ -5,6 +5,11 @@
 # and cause some issues in network configuration
 - name: Disable Cloud-init service
   hosts: overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ disable_cloud_init_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - disable-cloud-init
   roles:

ansible/disable-glean.yml

@@ -4,6 +4,11 @@
 # interfaces. In some cases this can lead to timeouts.
 - name: Ensure Glean is disabled and its artifacts are removed
   hosts: seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ disable_glean_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - disable-glean
   roles:

ansible/dnf.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure DNF repos are configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ dnf_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:

ansible/docker.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure docker is configured
   hosts: docker
+  max_fail_percentage: >-
+    {{ docker_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - docker
   tasks:

ansible/drac-bios.yml

@@ -2,6 +2,10 @@
 - name: Ensure that overcloud nodes' BIOS are configured
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_bios_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     bios_config:
       OneTimeBootMode: "OneTimeBootSeq"

ansible/drac-boot-order.yml

@@ -2,6 +2,10 @@
 - name: Ensure that overcloud nodes' boot order is configured
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_boot_order_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_host: "{{ ipmi_address }}"
     ansible_user: "{{ ipmi_username }}"

ansible/drac-facts.yml

@@ -2,6 +2,10 @@
 - name: Gather and display BIOS and RAID facts from iDRACs
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ drac_facts_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     # The role simply pulls in the drac_facts module.
     - role: stackhpc.drac-facts

ansible/dump-config.yml

@@ -8,6 +8,10 @@
 - name: Dump configuration from one or more hosts
   hosts: "{{ dump_hosts }}"
   gather_facts: "{{ dump_facts }}"
+  max_fail_percentage: >-
+    {{ dump_config_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - dump-config
   vars:

ansible/etc-hosts.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure /etc/hosts is configured
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ etc_hosts_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts
   tasks:

ansible/firewall.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure firewall is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ firewall_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - config
     - firewall

ansible/host-command-run.yml

@@ -1,7 +1,11 @@
 ---
 - name: Run a command
-  gather_facts: False
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  gather_facts: False
+  max_fail_percentage: >-
+    {{ host_command_run_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tasks:
     - name: Run a command
       shell: "{{ host_command_to_run }}"

ansible/host-package-update.yml

@@ -1,6 +1,10 @@
 ---
 - name: Update host packages
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ host_package_update_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Optionally set this to a list of packages to update. Default behaviour is
     # to update all packages.

ansible/infra-vm-deprovision.yml

@@ -15,6 +15,10 @@
 
 - name: Ensure defined infra VMs are destroyed
   hosts: hypervisors
+  max_fail_percentage: >-
+    {{ infra_vm_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-deprovision
   tasks:
@@ -27,6 +31,10 @@
 - name: Set facts about infra VMs
   gather_facts: false
   hosts: "{{ infra_vm_limit | default('infra-vms') }}"
+  max_fail_percentage: >-
+    {{ infra_vm_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-deprovision
   tasks:

ansible/infra-vm-provision.yml

@@ -15,6 +15,10 @@
 
 - name: Ensure defined infra VMs are deployed
   hosts: hypervisors
+  max_fail_percentage: >-
+    {{ infra_vm_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-provision
   tasks:
@@ -26,6 +30,10 @@
 - name: Wait for infra VMs to be accessible
   hosts: "{{ infra_vm_limit | default('infra-vms') }}"
   gather_facts: false
+  max_fail_percentage: >-
+    {{ infra_vm_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - infra-vm-provision
   tasks:

ansible/ip-allocation.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure IP addresses are allocated
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ ip_allocation_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ip-allocation
   gather_facts: no

ansible/ip-routing.yml

@@ -3,6 +3,11 @@
 
 - name: Ensure IP routing is enabled
   hosts: seed-hypervisor:seed
+  max_fail_percentage: >-
+    {{ ip_routing_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ip-routing
   roles:

ansible/kayobe-ansible-user.yml

@@ -9,6 +9,11 @@
 - name: Determine whether user bootstrapping is required
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-ansible-user
   tasks:
@@ -31,15 +36,24 @@
           attempting bootstrap
       when: ssh_result.unreachable | default(false)
 
-- name: Ensure python is installed
+- name: Ensure python is installed and the Kayobe Ansible user account exists
   hosts: kayobe_user_bootstrap_required_True
   gather_facts: no
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_user: "{{ bootstrap_user }}"
+    # We can't assume that a virtualenv exists at this point, so use the system
+    # python interpreter.
+    ansible_python_interpreter: /usr/bin/python3
     dnf_options:
       - "-y"
       - "{% if 'proxy' in dnf_config %}--setopt=proxy={{ dnf_config['proxy'] }}{% endif %}"
   tags:
+    - kayobe-ansible-user
     - ensure-python
   tasks:
     - name: Check if python is installed
@@ -53,29 +67,19 @@
       raw: "test -e /usr/bin/apt && (sudo apt -y update && sudo apt install -y python3-minimal) || (sudo dnf {{ dnf_options | select | join(' ') }} install python3)"
       when: check_python.rc != 0
 
-- name: Ensure the Kayobe Ansible user account exists
+    - import_role:
-  hosts: kayobe_user_bootstrap_required_True
+        name: singleplatform-eng.users
-  gather_facts: false
+      vars:
-  tags:
+        groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
-    - kayobe-ansible-user
+        users:
-  vars:
+          - username: "{{ kayobe_ansible_user }}"
-    ansible_user: "{{ bootstrap_user }}"
+            name: Kayobe deployment user
-    # We can't assume that a virtualenv exists at this point, so use the system
+            groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
-    # python interpreter.
+            append: True
-    ansible_python_interpreter: /usr/bin/python3
+            ssh_key:
-  roles:
+              - "{{ lookup('file', ssh_public_key_path) }}"
-    - role: singleplatform-eng.users
-      groups_to_create: "{{ [{'name': 'docker'}] if 'docker' in group_names else [] }}"
-      users:
-        - username: "{{ kayobe_ansible_user }}"
-          name: Kayobe deployment user
-          groups: "{{ ['docker'] if 'docker' in group_names else [] }}"
-          append: True
-          ssh_key:
-            - "{{ lookup('file', ssh_public_key_path) }}"
       become: True
 
-  post_tasks:
     - name: Ensure the Kayobe Ansible user has passwordless sudo
       copy:
         content: "{{ kayobe_ansible_user }} ALL=(ALL) NOPASSWD: ALL"
@@ -86,6 +90,11 @@
 - name: Verify that the Kayobe Ansible user account is accessible
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kayobe_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-ansible-user
   vars:

ansible/kayobe-target-venv.yml

@@ -5,6 +5,11 @@
 - name: Ensure a virtualenv exists for kayobe
   hosts: seed:seed-hypervisor:overcloud:infra-vms
   gather_facts: False
+  max_fail_percentage: >-
+    {{ kayobe_target_venv_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kayobe-target-venv
   tasks:

ansible/kolla-ansible-user.yml

@@ -2,6 +2,11 @@
 - name: Ensure the Kolla Ansible user account exists
   hosts: seed:overcloud
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kolla_ansible_user_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-ansible
     - kolla-ansible-user

ansible/kolla-bifrost-hostvars.yml

@@ -4,6 +4,10 @@
 - name: Ensure the Bifrost overcloud inventory is populated
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ kolla_bifrost_hostvars_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-bifrost-hostvars
   become: true

ansible/kolla-packages.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure Kolla Ansible packages are installed
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ kolla_packages_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-packages
   vars:

ansible/kolla-pip.yml

@@ -2,6 +2,11 @@
 - name: Configure local PyPi mirror for Kolla Ansible
   hosts: seed:overcloud
   gather_facts: false
+  max_fail_percentage: >-
+    {{ kolla_pip_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - kolla-ansible
     - kolla-pip

ansible/kolla-target-venv.yml

@@ -5,6 +5,11 @@
 - name: Ensure a virtualenv exists for kolla-ansible
   hosts: seed:overcloud
   gather_facts: False
+  max_fail_percentage: >-
+    {{ kolla_target_venv_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # kolla_overcloud_inventory_top_level_group_map looks like:
     # kolla_overcloud_inventory_top_level_group_map:

ansible/logging.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure Logging configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ logging_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   gather_facts: false
   vars:
     # NOTE(wszumski): Kayobe target env does not yet exist.

ansible/luks.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure encryption configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ luks_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - luks
   tasks:

ansible/lvm.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure LVM configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ lvm_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - lvm
     - upgrade-check

ansible/mdadm.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure software RAID configuration is applied
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ mdadm_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - mdadm
   roles:

ansible/network-connectivity.yml

@@ -1,6 +1,10 @@
 ---
 - name: Check network connectivity between hosts
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ network_connectivity_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this to an external IP address to check.
     nc_external_ip: 8.8.8.8

ansible/network.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure networking is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ network_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - config
     - network

ansible/overcloud-bios-raid.yml

@@ -9,6 +9,10 @@
 - name: Group overcloud nodes by their BMC type
   hosts: overcloud
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid
@@ -35,6 +39,10 @@
 - name: Check whether any changes to nodes' BIOS and RAID configuration are required
   hosts: overcloud_with_bmcs_of_type_idrac
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid
@@ -57,6 +65,10 @@
 - name: Ensure that overcloud BIOS and RAID volumes are configured
   hosts: overcloud_with_bmcs_of_type_idrac
   gather_facts: no
+  max_fail_percentage: >-
+    {{ overcloud_bios_raid_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - bios
     - raid

ansible/overcloud-deprovision.yml

@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes are deprovisioned
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_deprovision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - deprovision
   vars:

ansible/overcloud-etc-hosts-fixup.yml

@@ -9,6 +9,10 @@
 
 - name: Ensure overcloud hosts' /etc/hosts does not contain incorrect IPs
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_etc_hosts_fixup_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts-fixup
   tasks:
@@ -25,6 +29,10 @@
 
 - name: Ensure rabbitmq containers' /etc/hosts does not contain incorrect IPs
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_etc_hosts_fixup_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - etc-hosts-fixup
   vars:

ansible/overcloud-facts-gather.yml

@@ -1,6 +1,10 @@
 ---
 - name: Gather facts
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_facts_gather_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   gather_facts: false
   tasks:
     - name: Gather facts

ansible/overcloud-hardware-inspect.yml

@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes' hardware is inspected
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_hardware_inspect_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - hardware-inspect
   vars:

ansible/overcloud-introspection-data-save.yml

@@ -1,6 +1,10 @@
 ---
 - name: Ensure the overcloud nodes' hardware introspection data is saved
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_introspection_data_save_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     seed_host: "{{ groups['seed'][0] }}"
     # Override this to save results to another location.

ansible/overcloud-provision.yml

@@ -3,6 +3,10 @@
 
 - name: Ensure the overcloud nodes are provisioned
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_provision_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - provision
   vars:

ansible/overcloud-service-config-save.yml

@@ -1,6 +1,10 @@
 ---
 - name: Save overcloud service configuration
   hosts: overcloud
+  max_fail_percentage: >-
+    {{ overcloud_service_config_save_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - service-config-save
   vars:

ansible/physical-network.yml

@@ -5,6 +5,10 @@
 - name: Group hosts by their switch type and apply configuration filters
   hosts: switches
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     # Set this variable to True to configure the network for hardware
     # discovery.
@@ -90,6 +94,10 @@
 - name: Display switch configuration
   hosts: switches_in_display_mode_True
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tasks:
     - name: Display the candidate global switch configuration
       debug:
@@ -102,6 +110,10 @@
 - name: Ensure Arista physical switches are configured
   hosts: switches_of_type_arista:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -114,6 +126,10 @@
 - name: Ensure DellOS physical switches are configured
   hosts: switches_of_type_dellos6:switches_of_type_dellos9:switches_of_type_dellos10:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -127,6 +143,10 @@
 - name: Ensure Dell PowerConnect physical switches are configured
   hosts: switches_of_type_dell-powerconnect:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -139,6 +159,10 @@
 - name: Ensure Juniper physical switches are configured
   hosts: switches_of_type_junos:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -152,6 +176,10 @@
 - name: Ensure Mellanox physical switches are configured
   hosts: switches_of_type_mellanox:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 
@@ -164,6 +192,10 @@
 - name: Ensure Cumulus physical switches are configured with NCLU
   hosts: switches_of_type_nclu:&switches_in_display_mode_False
   gather_facts: no
+  max_fail_percentage: >-
+    {{ physical_network_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: ssh-known-host
 

ansible/pip.yml

@@ -1,6 +1,11 @@
 ---
 - name: Configure local PyPi mirror
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ pip_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - pip
   vars:

ansible/proxy.yml

@@ -1,5 +1,10 @@
 - name: Configure HTTP(S) proxy settings
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ proxy_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     ansible_python_interpreter: /usr/bin/python3
   tags:

ansible/selinux.yml

@@ -1,6 +1,11 @@
 ---
 - name: Configure SELinux state and reboot if required
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ selinux_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - selinux
   roles:

ansible/ssh-known-host.yml

@@ -2,6 +2,11 @@
 - name: Ensure known hosts are configured
   hosts: all
   gather_facts: no
+  max_fail_percentage: >-
+    {{ ssh_known_host_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ssh-known-host
   roles:

ansible/swap.yml

@@ -2,6 +2,11 @@
 - name: Configure swap
   hosts: seed-hypervisor:seed:overcloud:infra-vms
   become: true
+  max_fail_percentage: >-
+    {{ swap_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - swap
   tasks:

ansible/swift-block-devices.yml

@@ -1,6 +1,10 @@
 ---
 - name: Ensure Swift block devices are prepared
   hosts: "{{ swift_hosts }}"
+  max_fail_percentage: >-
+    {{ swift_block_devices_max_fail_percentage |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   vars:
     swift_hosts: storage
   tags:

ansible/sysctl.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure sysctl parameters are configured
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ sysctl_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - sysctl
   roles:

ansible/time.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure timezone is configured
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ time_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - timezone
   tasks:
@@ -24,6 +29,11 @@
 
 - name: Ensure Chrony is installed and configured
   hosts: ntp
+  max_fail_percentage: >-
+    {{ time_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - ntp
   tasks:

ansible/tuned.yml

@@ -1,6 +1,11 @@
 ---
 - name: Configure tuned profile
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ tuned_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - tuned
   roles:

ansible/users.yml

@@ -1,6 +1,11 @@
 ---
 - name: Ensure users exist
   hosts: seed:seed-hypervisor:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ users_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - users
   roles:

ansible/vgpu.yml

@@ -1,6 +1,11 @@
 ---
 - name: Configure IOMMU
   hosts: iommu
+  max_fail_percentage: >-
+    {{ vgpu_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - iommu
     - vgpu
@@ -15,6 +20,11 @@
 
 - name: Configure NVIDIA VGPUs
   hosts: vgpu
+  max_fail_percentage: >-
+    {{ vgpu_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - vgpu
   tasks:
@@ -28,6 +38,11 @@
 
 - name: Reboot when required
   hosts: iommu:vgpu
+  max_fail_percentage: >-
+    {{ vgpu_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - reboot
     - iommu

ansible/wipe-disks.yml

@@ -9,6 +9,11 @@
 
 - name: Ensure that all unmounted block devices are wiped
   hosts: seed-hypervisor:seed:overcloud:infra-vms
+  max_fail_percentage: >-
+    {{ wipe_disks_max_fail_percentage |
+       default(host_configure_max_fail_percentage) |
+       default(kayobe_max_fail_percentage) |
+       default(100) }}
   tags:
     - wipe-disks
   tasks:

doc/source/configuration/reference/ansible.rst

@@ -146,3 +146,30 @@ Similarly, for Kolla Ansible (notice the similar but different file names):
    :caption: ``$KAYOBE_CONFIG_PATH/kolla/globals.yml``
 
    kolla_ansible_setup_gather_subset: "all,!facter"
+
+Max failure percentage
+======================
+
+It is possible to specify a `maximum failure percentage
+<https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_error_handling.html#setting-a-maximum-failure-percentage>`__
+using ``kayobe_max_fail_percentage``. By default this is undefined, which is
+equivalent to a value of 100, meaning that Ansible will continue execution
+until all hosts have failed or completed. For example:
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/globals.yml``
+
+   kayobe_max_fail_percentage: 50
+
+A max fail percentage may be set for the ``kayobe * host configure`` commands
+using ``host_configure_max_fail_percentage``, or for a specific playbook using
+``<playbook>_max_fail_percentage`` where ``<playbook>`` is the playbook name
+with dashes replaced with underscores and without the ``.yml`` extension. For
+example:
+
+.. code-block:: yaml
+   :caption: ``$KAYOBE_CONFIG_PATH/globals.yml``
+
+   kayobe_max_fail_percentage: 50
+   host_configure_max_fail_percentage: 25
+   time_max_fail_percentage: 100

etc/kayobe/globals.yml

@@ -64,6 +64,10 @@
 # to not specify a gather subset.
 #kayobe_ansible_setup_gather_subset:
 
+# Global maximum failure percentage. By default this is undefined, which is
+# equivalent to a value of 100.
+#kayobe_max_fail_percentage:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

releasenotes/notes/max-fail-percentage-5f1d21bdcd138695.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds support for setting the max fail percentage for Ansible plays via
+    ``kayobe_max_fail_percentage``. It can also be set on a per-playbook basis,
+    e.g. ``time_max_fail_percentage``.