Support Ansible max_fail_percentage

This allows us to continue execution until a certain proportion of hosts
to fail. This can be useful at scale, where failures are common, and
restarting a deployment is time-consuming.

The default max failure percentage is 100, keeping the default
behaviour. A global max failure percentage may be set via
kolla_max_fail_percentage, and individual services may define a max
failure percentage via <service>_max_fail_percentage.

Note that all hosts in the inventory must be reachable for fact
gathering, even those not included in a --limit.

Closes-Bug: #1833737
Change-Id: I808474a75c0f0e8b539dc0421374b06cea44be4f

ansible/destroy.yml

@@ -1,5 +1,9 @@
 ---
 - name: Apply role destroy
   hosts: all
+  max_fail_percentage: >-
+    {{ destroy_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - destroy

ansible/gather-facts.yml

@@ -5,6 +5,10 @@
 - name: Gather facts for all hosts
   hosts: all
   any_errors_fatal: "{{ kolla_ansible_setup_any_errors_fatal | bool }}"
+  max_fail_percentage: >-
+    {{ gather_facts_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   serial: '{{ kolla_serial|default("0") }}'
   gather_facts: false
   tasks:
@@ -37,6 +41,10 @@
 - name: Gather facts for all hosts (if using --limit)
   hosts: all_using_limit_True
   any_errors_fatal: "{{ kolla_ansible_setup_any_errors_fatal | bool }}"
+  max_fail_percentage: >-
+    {{ gather_facts_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   serial: '{{ kolla_serial|default("0") }}'
   gather_facts: false
   vars:

ansible/kolla-host.yml

@@ -5,6 +5,10 @@
   hosts: baremetal
   serial: '{{ kolla_serial|default("0") }}'
   gather_facts: false
+  max_fail_percentage: >-
+    {{ baremetal_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: openstack.kolla.baremetal,
         tags: baremetal }

ansible/mariadb.yml

@@ -5,6 +5,10 @@
   hosts:
     - mariadb
     - '&enable_mariadb_True'
+  max_fail_percentage: >-
+    {{ mariadb_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - mariadb
   tasks:
@@ -18,6 +22,10 @@
     - '&enable_mariadb_True'
   # Restart in batches
   serial: "33%"
+  max_fail_percentage: >-
+    {{ mariadb_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - mariadb
   tasks:
@@ -32,6 +40,10 @@
     - '&enable_mariadb_True'
   # Start in batches
   serial: "33%"
+  max_fail_percentage: >-
+    {{ mariadb_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - mariadb
   tasks:
@@ -44,6 +56,10 @@
   hosts:
     - mariadb_bootstrap_restart
     - '&enable_mariadb_True'
+  max_fail_percentage: >-
+    {{ mariadb_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - mariadb
   tasks:
@@ -56,6 +72,10 @@
   hosts:
     - mariadb
     - '&enable_mariadb_True'
+  max_fail_percentage: >-
+    {{ mariadb_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - mariadb
   tasks:

ansible/mariadb_backup.yml

@@ -1,6 +1,10 @@
 ---
 - name: Backup MariaDB
   hosts: mariadb
+  max_fail_percentage: >-
+    {{ mariadb_backup_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: mariadb,
         tags: mariadb,

ansible/mariadb_recovery.yml

@@ -1,6 +1,10 @@
 ---
 - name: Recover mariadb
   hosts: mariadb
+  max_fail_percentage: >-
+    {{ mariadb_recovery_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: mariadb,
         tags: mariadb,

ansible/nova-libvirt-cleanup.yml

@@ -6,6 +6,10 @@
   hosts:
     - compute
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_libvirt_cleanup_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - nova-libvirt-cleanup
   tasks:

ansible/nova.yml

@@ -37,6 +37,10 @@
     - nova-api
     - nova-api-bootstrap
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     # * Create nova API & cell0 DBs & users
     # * API DB schema migrations
@@ -109,6 +113,10 @@
     - nova-api-deploy
     - nova-api-upgrade
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: nova
       when: enable_nova | bool
@@ -130,6 +138,10 @@
     - nova-cell-deploy
     - nova-cell-upgrade
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_cell_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: nova-cell
       when: enable_nova | bool
@@ -147,6 +159,10 @@
     - nova-api
     - nova-refresh-scheduler-cell-cache
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova
@@ -172,6 +188,10 @@
     - nova-api
     - nova-api-reload
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova
@@ -196,6 +216,10 @@
     - nova-cell
     - nova-cell-reload
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_cell_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova-cell
@@ -216,6 +240,10 @@
     - nova-api
     - nova-api-reload
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ nova_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - import_role:
         name: nova
@@ -238,6 +266,8 @@
     - nova-online-data-migrations
     - nova-api-online-data-migrations
   serial: '{{ kolla_serial|default("0") }}'
+  # Fail all hosts if any of these once-per-cell tasks fails.
+  any_errors_fatal: true
   tasks:
     - import_role:
         name: nova

ansible/prune-images.yml

@@ -5,5 +5,9 @@
   hosts: baremetal
   serial: '{{ kolla_serial|default("0") }}'
   gather_facts: false
+  max_fail_percentage: >-
+    {{ prune_images_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - prune-images

ansible/rabbitmq.yml

@@ -5,6 +5,10 @@
   hosts:
     - rabbitmq
     - '&enable_rabbitmq_True'
+  max_fail_percentage: >-
+    {{ rabbitmq_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - rabbitmq
   tasks:
@@ -30,6 +34,10 @@
     - '&enable_rabbitmq_True'
   # Restart in batches
   serial: "33%"
+  max_fail_percentage: >-
+    {{ rabbitmq_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - rabbitmq
   tasks:
@@ -45,6 +53,10 @@
   hosts:
     - rabbitmq
     - '&enable_rabbitmq_True'
+  max_fail_percentage: >-
+    {{ rabbitmq_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - rabbitmq
   tasks:
@@ -62,6 +74,10 @@
   hosts:
     - outward-rabbitmq
     - '&enable_outward_rabbitmq_True'
+  max_fail_percentage: >-
+    {{ rabbitmq_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - rabbitmq
   tasks:
@@ -86,6 +102,10 @@
     - '&enable_outward_rabbitmq_True'
   # Restart in batches
   serial: "33%"
+  max_fail_percentage: >-
+    {{ rabbitmq_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - rabbitmq
   tasks:
@@ -102,6 +122,10 @@
   hosts:
     - outward-rabbitmq
     - '&enable_outward_rabbitmq_True'
+  max_fail_percentage: >-
+    {{ rabbitmq_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - rabbitmq
   tasks:

ansible/site.yml

@@ -7,6 +7,10 @@
 - name: Group hosts based on configuration
   hosts: all
   gather_facts: false
+  max_fail_percentage: >-
+    {{ group_hosts_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tasks:
     - name: Group hosts based on Kolla action
       group_by:
@@ -81,6 +85,10 @@
   gather_facts: false
   # Apply only when kolla action is 'precheck'.
   hosts: kolla_action_precheck
+  max_fail_percentage: >-
+    {{ prechecks_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - role: prechecks
 
@@ -92,6 +100,10 @@
     - kolla-logs
     - kolla-toolbox
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ common_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - common
   roles:
@@ -103,6 +115,10 @@
     - loadbalancer
     - '&enable_loadbalancer_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ loadbalancer_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   tags:
     - haproxy
     - keepalived
@@ -362,6 +378,10 @@
     - collectd
     - '&enable_collectd_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ collectd_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: collectd,
         tags: collectd }
@@ -372,6 +392,10 @@
     - influxdb
     - '&enable_influxdb_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ influxdb_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: influxdb,
         tags: influxdb }
@@ -382,6 +406,10 @@
     - telegraf
     - '&enable_telegraf_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ telegraf_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: telegraf,
         tags: telegraf }
@@ -392,6 +420,10 @@
     - redis
     - '&enable_redis_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ redis_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: redis,
         tags: redis }
@@ -406,6 +438,10 @@
     - memcached
     - '&enable_memcached_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ memcached_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: memcached,
         tags: [memcache, memcached] }
@@ -426,6 +462,10 @@
     - prometheus-libvirt-exporter
     - '&enable_prometheus_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ prometheus_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: prometheus,
         tags: prometheus }
@@ -437,6 +477,10 @@
     - tgtd
     - '&enable_iscsid_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ iscsid_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: iscsi,
         tags: iscsi }
@@ -447,6 +491,10 @@
     - multipathd
     - '&enable_multipathd_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ multipathd_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: multipathd,
         tags: multipathd }
@@ -458,6 +506,10 @@
   hosts:
     - etcd
     - '&enable_etcd_True'
+  max_fail_percentage: >-
+    {{ etcd_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: etcd,
         tags: etcd }
@@ -468,6 +520,10 @@
     - keystone
     - '&enable_keystone_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ keystone_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: keystone,
         tags: keystone }
@@ -478,6 +534,10 @@
     - opensearch
     - '&enable_opensearch_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ opensearch_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: opensearch,
         tags: opensearch }
@@ -491,6 +551,10 @@
     - swift-proxy-server
     - '&enable_swift_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ swift_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: swift,
         tags: swift }
@@ -503,6 +567,10 @@
     - kolla-toolbox
     - '&enable_ceph_rgw_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ ceph_rgw_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: ceph-rgw,
         tags: ceph-rgw }
@@ -513,6 +581,10 @@
     - glance-api
     - '&enable_glance_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ glance_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: glance,
         tags: glance }
@@ -527,6 +599,10 @@
     - ironic-http
     - '&enable_ironic_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ ironic_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: ironic,
         tags: ironic }
@@ -540,6 +616,10 @@
     - cinder-volume
     - '&enable_cinder_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ cinder_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: cinder,
         tags: cinder }
@@ -550,6 +630,10 @@
     - placement-api
     - '&enable_placement_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ placement_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: placement,
         tags: placement }
@@ -564,6 +648,10 @@
     - openvswitch
     - '&enable_openvswitch_True_enable_ovs_dpdk_False'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ openvswitch_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: openvswitch,
         tags: openvswitch,
@@ -575,6 +663,10 @@
     - openvswitch
     - '&enable_openvswitch_True_enable_ovs_dpdk_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ ovs_dpdk_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: ovs-dpdk,
         tags: ovs-dpdk,
@@ -586,6 +678,10 @@
     - ovn-controller
     - '&enable_ovn_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ ovn_controller_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: ovn-controller,
         tags: [ovn, ovn-controller] }
@@ -598,6 +694,10 @@
     - ovn-sb-db
     - '&enable_ovn_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ ovn_db_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: ovn-db,
         tags: [ovn, ovn-db] }
@@ -618,6 +718,10 @@
     - manila-share
     - '&enable_neutron_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ neutron_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: neutron,
         tags: neutron }
@@ -628,6 +732,10 @@
     - compute
     - '&enable_kuryr_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ kuryr_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: kuryr,
         tags: kuryr }
@@ -639,6 +747,10 @@
     - hacluster-remote
     - '&enable_hacluster_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ hacluster_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: hacluster,
         tags: hacluster }
@@ -651,6 +763,10 @@
     - heat-engine
     - '&enable_heat_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ heat_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: heat,
         tags: heat }
@@ -661,6 +777,10 @@
     - horizon
     - '&enable_horizon_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ horizon_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: horizon,
         tags: horizon }
@@ -672,6 +792,10 @@
     - murano-engine
     - '&enable_murano_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ murano_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: murano,
         tags: murano }
@@ -687,6 +811,10 @@
     - solum-image-builder
     - '&enable_solum_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ solum_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: solum,
         tags: solum }
@@ -698,6 +826,10 @@
     - magnum-conductor
     - '&enable_magnum_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ magnum_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: magnum,
         tags: magnum }
@@ -711,6 +843,10 @@
     - mistral-event-engine
     - '&enable_mistral_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ mistral_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: mistral,
         tags: mistral }
@@ -722,6 +858,10 @@
     - sahara-engine
     - '&enable_sahara_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ sahara_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: sahara,
         tags: sahara }
@@ -735,6 +875,10 @@
     - manila-scheduler
     - '&enable_manila_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ manila_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: manila,
         tags: manila }
@@ -747,6 +891,10 @@
     - gnocchi-statsd
     - '&enable_gnocchi_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ gnocchi_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: gnocchi,
         tags: gnocchi }
@@ -760,6 +908,10 @@
     - ceilometer-ipmi
     - '&enable_ceilometer_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ ceilometer_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: ceilometer,
         tags: ceilometer }
@@ -773,6 +925,10 @@
     - aodh-notifier
     - '&enable_aodh_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ aodh_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: aodh,
         tags: aodh }
@@ -785,6 +941,10 @@
     - barbican-worker
     - '&enable_barbican_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ barbican_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: barbican,
         tags: barbican }
@@ -797,6 +957,10 @@
     - cyborg-conductor
     - '&enable_cyborg_True'
   serial: '{{ serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ cyborg_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: cyborg,
         tags: cyborg }
@@ -813,6 +977,10 @@
     - designate-backend-bind9
     - '&enable_designate_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ designate_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: designate,
         tags: designate }
@@ -825,6 +993,10 @@
     - trove-taskmanager
     - '&enable_trove_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ trove_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: trove,
         tags: trove }
@@ -837,6 +1009,10 @@
     - watcher-applier
     - '&enable_watcher_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ watcher_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: watcher,
         tags: watcher }
@@ -847,6 +1023,10 @@
     - grafana
     - '&enable_grafana_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ grafana_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: grafana,
         tags: grafana }
@@ -858,6 +1038,10 @@
     - cloudkitty-processor
     - '&enable_cloudkitty_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ cloudkitty_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: cloudkitty,
         tags: cloudkitty }
@@ -869,6 +1053,10 @@
     - freezer-scheduler
     - '&enable_freezer_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ freezer_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: freezer,
         tags: freezer }
@@ -882,6 +1070,10 @@
     - senlin-health-manager
     - '&enable_senlin_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ senlin_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: senlin,
         tags: senlin }
@@ -893,6 +1085,10 @@
     - tacker-conductor
     - '&enable_tacker_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ tacker_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: tacker,
         tags: tacker }
@@ -906,6 +1102,10 @@
     - octavia-worker
     - '&enable_octavia_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ octavia_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: octavia,
         tags: octavia }
@@ -919,6 +1119,10 @@
     - zun-cni-daemon
     - '&enable_zun_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ zun_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: zun,
         tags: zun }
@@ -933,6 +1137,10 @@
     - vitrage-persistor
     - '&enable_vitrage_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ vitrage_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: vitrage,
         tags: vitrage }
@@ -944,6 +1152,10 @@
     - blazar-manager
     - '&enable_blazar_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ blazar_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: blazar,
         tags: blazar }
@@ -957,6 +1169,10 @@
     - masakari-instancemonitor
     - '&enable_masakari_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ masakari_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: masakari,
         tags: masakari }
@@ -968,6 +1184,10 @@
     - venus-manager
     - '&enable_venus_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ venus_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: venus,
         tags: venus }
@@ -978,6 +1198,10 @@
     - skyline
     - '&enable_skyline_True'
   serial: '{{ kolla_serial|default("0") }}'
+  max_fail_percentage: >-
+    {{ skyline_max_fail_percentage |
+       default(kolla_max_fail_percentage) |
+       default(100) }}
   roles:
     - { role: skyline,
         tags: skyline }

doc/source/user/ansible-tuning.rst

@@ -139,3 +139,24 @@ facts via facter:
 .. code-block:: yaml
 
    kolla_ansible_setup_gather_subset: "all,!facter"
+
+Max failure percentage
+----------------------
+
+It is possible to specify a `maximum failure percentage
+<https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_error_handling.html#setting-a-maximum-failure-percentage>`__
+using ``kolla_max_fail_percentage``. By default this is undefined, which is
+equivalent to a value of 100, meaning that Ansible will continue execution
+until all hosts have failed or completed. For example:
+
+.. code-block:: yaml
+
+   kolla_max_fail_percentage: 50
+
+A max fail percentage may be set for specific services using
+``<service>_max_fail_percentage``. For example:
+
+.. code-block:: yaml
+
+   kolla_max_fail_percentage: 50
+   nova_max_fail_percentage: 25

etc/kolla/globals.yml

@@ -32,6 +32,12 @@ workaround_ansible_issue_8743: yes
 # scenarios with all facts cached (as there is no task to fail).
 #kolla_ansible_setup_any_errors_fatal: false
 
+# This variable may be used to set the maxiumum failure percentage for all
+# plays. More fine-grained control is possible via per-service variables, e.g.
+# nova_max_fail_percentage. The default behaviour is to set a max fail
+# percentage of 100, which is equivalent to not setting it.
+#kolla_max_fail_percentage:
+
 ###############
 # Kolla options
 ###############

releasenotes/notes/max-fail-percentage-00838dc468920ec1.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds support for setting the max fail percentage for Ansible plays via
+    ``kolla_max_fail_percentage``. It can also be set on a per-service basis,
+    e.g. ``nova_max_fail_percentage``.