Remove the "max_fail_percentage" option

This option can cause silent failures which are confusing and hard to track down. While the intention of this was to allow large scale deployments to succeed in cases where a single node fails due to transiant issues it has produced more problems in terms of confusion that it solves. This change removes the option from all production playbooks. Change-Id: I1dcbbf5bc8cc66f11dd8ddc22d2a177c5c0f31f1 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
2018-02-22 18:00:43 -06:00 · 2018-02-22 18:00:43 -06:00 · c2743f5cca
parent 5d61ed448c
commit c2743f5cca
31 changed files with 21 additions and 39 deletions
--- a/playbooks/ceph-install.yml
+++ b/playbooks/ceph-install.yml
@ -15,7 +15,6 @@

 - name: Install ceph mons
  hosts: ceph-mon
-  max_fail_percentage: 20
  user: root
  serial: 1
  pre_tasks:
@ -124,7 +123,6 @@

 - name: Install ceph osds
  hosts: ceph-osd
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: common-tasks/os-log-dir-setup.yml
--- a/playbooks/containers-lxc-create.yml
+++ b/playbooks/containers-lxc-create.yml
@ -34,7 +34,6 @@
 - name: Create container(s)
  hosts: all_lxc_containers
  gather_facts: false
-  max_fail_percentage: 20
  user: root
  roles:
    - role: "lxc_container_create"
--- a/playbooks/containers-lxc-destroy.yml
+++ b/playbooks/containers-lxc-destroy.yml
@ -30,7 +30,6 @@
 - name: Destroy lxc containers
  hosts: all_lxc_containers
  gather_facts: false
-  max_fail_percentage: 20
  user: root
  tasks:
    - name: Destroy a container
--- a/playbooks/containers-lxc-host.yml
+++ b/playbooks/containers-lxc-host.yml
@ -16,7 +16,6 @@
 - name: Basic lxc host setup
  hosts: "{{ lxc_host_group | default('lxc_hosts')}}"
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: common-tasks/set-upper-constraints.yml
--- a/playbooks/containers-nspawn-destroy.yml
+++ b/playbooks/containers-nspawn-destroy.yml
@ -30,7 +30,6 @@
 - name: Destroy nspawn containers
  hosts: all_nspawn_containers
  gather_facts: false
-  max_fail_percentage: 20
  user: root
  tasks:
    - name: Get container status
--- a/playbooks/etcd-install.yml
+++ b/playbooks/etcd-install.yml
@ -16,7 +16,6 @@
 - name: Install etcd server cluster
  hosts: etcd_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/galera-install.yml
+++ b/playbooks/galera-install.yml
@ -17,7 +17,6 @@
  hosts: galera_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
  serial: 1
-  max_fail_percentage: 0
  user: root
  tasks:
    - include: common-tasks/os-log-dir-setup.yml
@ -43,7 +42,6 @@
  hosts: galera_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
  serial: 1
-  max_fail_percentage: 20
  user: root
  roles:
    - role: haproxy_endpoints
--- a/playbooks/haproxy-install.yml
+++ b/playbooks/haproxy-install.yml
@ -41,7 +41,6 @@
 - name: Install haproxy
  hosts: haproxy
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - name: Remove legacy haproxy configuration files
--- a/playbooks/memcached-install.yml
+++ b/playbooks/memcached-install.yml
@ -16,7 +16,6 @@
 - name: Install memcached
  hosts: memcached
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/openstack-hosts-setup.yml
+++ b/playbooks/openstack-hosts-setup.yml
@ -39,7 +39,6 @@
 - name: Basic host setup
  hosts: "{{ openstack_host_group|default('hosts') }}"
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - name: Check for a supported Operating System
--- a/playbooks/os-aodh-install.yml
+++ b/playbooks/os-aodh-install.yml
@ -16,7 +16,6 @@
 - name: Install the aodh components
  hosts: aodh_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-barbican-install.yml
+++ b/playbooks/os-barbican-install.yml
@ -16,7 +16,6 @@
 - name: Installation and setup of barbican
  hosts: barbican_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-ceilometer-install.yml
+++ b/playbooks/os-ceilometer-install.yml
@ -16,7 +16,6 @@
 - name: Install the ceilometer components
  hosts: ceilometer_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-cinder-install.yml
+++ b/playbooks/os-cinder-install.yml
@ -139,7 +139,6 @@
  hosts: cinder_backup,cinder_volume,cinder_scheduler
  gather_facts: no
  serial: "{{ cinder_backend_serial | default(['1', '100%']) }}"
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
@ -165,7 +164,6 @@
  hosts: cinder_api
  gather_facts: no
  serial: "{{ cinder_api_serial | default(['1','100%']) }}"
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
--- a/playbooks/os-glance-install.yml
+++ b/playbooks/os-glance-install.yml
@ -80,7 +80,6 @@
 - name: Refresh local facts after all software changes are made
  hosts: glance_all
  gather_facts: no
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
--- a/playbooks/os-gnocchi-install.yml
+++ b/playbooks/os-gnocchi-install.yml
@ -15,7 +15,6 @@

 - name: Install Gnocchi components
  hosts: gnocchi_all
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-heat-install.yml
+++ b/playbooks/os-heat-install.yml
@ -16,7 +16,6 @@
 - name: Install heat server
  hosts: heat_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-horizon-install.yml
+++ b/playbooks/os-horizon-install.yml
@ -16,7 +16,6 @@
 - name: Install horizon server
  hosts: horizon_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-keystone-install.yml
+++ b/playbooks/os-keystone-install.yml
@ -62,7 +62,6 @@
  hosts: keystone_all
  serial: "{{ keystone_serial | default(['1', '100%']) }}"
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
@ -154,7 +153,6 @@
 - name: Finalise data migrations if required
  hosts: keystone_all
  gather_facts: no
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
--- a/playbooks/os-magnum-install.yml
+++ b/playbooks/os-magnum-install.yml
@ -18,7 +18,6 @@

 - name: Install magnum server
  hosts: magnum_all
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-nova-install.yml
+++ b/playbooks/os-nova-install.yml
@ -125,7 +125,6 @@
 - name: Refresh local facts after all software changes are made
  hosts: nova_all
  gather_facts: no
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
@ -171,7 +170,6 @@
  hosts: "nova_all:!nova_api_placement:!nova_console"
  gather_facts: no
  serial: "{{ nova_serial | default('100%') }}"
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
@ -196,7 +194,6 @@
  hosts: "nova_api_placement:nova_console"
  gather_facts: no
  serial: "{{ nova_api_serial | default(['1', '100%']) }}"
-  max_fail_percentage: 20
  user: root
  environment: "{{ deployment_environment_variables | default({}) }}"
  tags:
--- a/playbooks/os-octavia-install.yml
+++ b/playbooks/os-octavia-install.yml
@ -16,7 +16,6 @@
 - name: Install octavia server
  hosts: octavia_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-swift-install.yml
+++ b/playbooks/os-swift-install.yml
@ -16,7 +16,6 @@
 - name: Installation and setup of Swift
  hosts: swift_all:swift_remote_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
@ -61,7 +60,6 @@

 - name: Installation and setup of Swift
  hosts: swift_all
-  max_fail_percentage: 20
  user: root
  roles:
    - role: "rsyslog_client"
--- a/playbooks/os-swift-sync.yml
+++ b/playbooks/os-swift-sync.yml
@ -18,7 +18,6 @@
 - name: Synchronisation of swift ring and ssh keys
  hosts: swift_all:swift_remote_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  roles:
    - role: "os_swift"
--- a/playbooks/os-tacker-install.yml
+++ b/playbooks/os-tacker-install.yml
@ -16,7 +16,6 @@
 - name: Install the tacker components
  hosts: tacker_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/os-tempest-install.yml
+++ b/playbooks/os-tempest-install.yml
@ -16,7 +16,6 @@
 - name: Installation and setup of Tempest
  hosts: utility_all[0]
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  roles:
    - role: "os_tempest"
--- a/playbooks/rabbitmq-install.yml
+++ b/playbooks/rabbitmq-install.yml
@ -17,7 +17,6 @@
  hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}"
  serial: 1
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 0
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
@ -40,7 +39,6 @@
 # http://www.rabbitmq.com/clustering.html#upgrading
 - name: Stop RabbitMQ nodes that are not the upgrader
  hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}[1:]"
-  max_fail_percentage: 0
  user: root
  tasks:
    - name: "Stop RabbitMQ"
@ -52,7 +50,6 @@
 - name: Install rabbitmq server
  hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}"
  serial: 20%
-  max_fail_percentage: 20
  user: root
  roles:
    - role: "rabbitmq_server"
--- a/playbooks/repo-server.yml
+++ b/playbooks/repo-server.yml
@ -16,7 +16,6 @@
 - name: Setup repo servers
  hosts: repo_all
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:

--- a/playbooks/rsyslog-install.yml
+++ b/playbooks/rsyslog-install.yml
@ -16,7 +16,6 @@
 - name: Install rsyslog
  hosts: rsyslog
  gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/playbooks/utility-install.yml
+++ b/playbooks/utility-install.yml
@ -15,7 +15,6 @@

 - name: Setup the utility location(s)
  hosts: utility_all
-  max_fail_percentage: 20
  user: root
  pre_tasks:
    - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
--- a/releasenotes/notes/remove-fax-fail-5038b4b6eb4951b3.yaml
+++ b/releasenotes/notes/remove-fax-fail-5038b4b6eb4951b3.yaml
@ -0,0 +1,21 @@
+---
+other:
+  - The `max_fail_percentage` playbook option has been used with the default
+    playbooks since the first release of the playbooks back in **Icehouse**.
+    While the intention was to allow large-scale deployments to succeed in
+    cases where a single node fails due to transient issues, this option has
+    produced more problems that it solves. If a failure occurs that is transient
+    in nature but is under the set failure percentage the playbook will report a
+    success, which can cause silent failures depending on where the failure
+    happened. If a deployer finds themselves in this situation the problems are
+    are then compounded because the tools will report there are no known issues.
+    To ensure deployers have the best deployment experience and the most
+    accurate information a change has been made to remove the
+    `max_fail_percentage` option from all of the default playbooks. The removal
+    of this option has the side effect of requiring the deploy to skip specific
+    hosts should one need to be omitted from a run, but has the benefit of
+    eliminating silent, hard to track down, failures. To skip a failing host
+    for a given playbook run use the `--limit '!$HOSTNAME'` CLI switch for the
+    specific run. Once the issues have been resolved for the failing host rerun
+    the specific playbook without the `--limit` option to ensure everything
+    is in sync.