Remove the "max_fail_percentage" option

This option can cause silent failures which are confusing and hard to
track down. While the intention of this was to allow large scale
deployments to succeed in cases where a single node fails due to
transiant issues it has produced more problems in terms of confusion
that it solves. This change removes the option from all production
playbooks.

Change-Id: I1dcbbf5bc8cc66f11dd8ddc22d2a177c5c0f31f1
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
(cherry picked from commit c2743f5cca)
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

playbooks/ceph-install.yml

@@ -15,7 +15,6 @@
 
 - name: Install ceph mons
   hosts: ceph-mon
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-log-dir-setup.yml
@@ -132,7 +131,6 @@
 
 - name: Install ceph osds
   hosts: ceph-osd
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-log-dir-setup.yml

playbooks/etcd-install.yml

@@ -16,7 +16,6 @@
 - name: Install etcd server cluster
   hosts: etcd_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/galera-install.yml

@@ -17,7 +17,6 @@
   hosts: galera_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
   serial: 1
-  max_fail_percentage: 0
   user: root
   tasks:
     - include: common-tasks/os-log-dir-setup.yml
@@ -41,7 +40,6 @@
   hosts: galera_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
   serial: 1
-  max_fail_percentage: 20
   user: root
   roles:
     - role: "galera_server"

playbooks/haproxy-install.yml

@@ -39,7 +39,6 @@
 - name: Install haproxy
   hosts: haproxy
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - name: Remove legacy haproxy configuration files

playbooks/lxc-containers-create.yml

@@ -20,7 +20,6 @@
 - name: Create container(s)
   hosts: "{{ container_group|default('all_containers') }}"
   gather_facts: false
-  max_fail_percentage: 20
   user: root
   roles:
     - role: "lxc_container_create"

playbooks/lxc-containers-destroy.yml

@@ -16,7 +16,6 @@
 - name: Destroy lxc containers
   hosts: "{{ container_group|default('all_containers') }}"
   gather_facts: false
-  max_fail_percentage: 20
   user: root
   tasks:
     - name: Destroy a container

playbooks/lxc-hosts-setup.yml

@@ -16,7 +16,6 @@
 - name: Basic lxc host setup
   hosts: "{{ lxc_host_group | default('lxc_hosts')}}"
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/set-upper-constraints.yml

playbooks/memcached-install.yml

@@ -16,7 +16,6 @@
 - name: Install memcached
   hosts: memcached
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/openstack-hosts-setup.yml

@@ -39,7 +39,6 @@
 - name: Basic host setup
   hosts: "{{ openstack_host_group|default('hosts') }}"
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - name: Check for a supported Operating System

playbooks/os-aodh-install.yml

@@ -16,7 +16,6 @@
 - name: Install the aodh components
   hosts: aodh_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-barbican-install.yml

@@ -16,7 +16,6 @@
 - name: Installation and setup of barbican
   hosts: barbican_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-ceilometer-install.yml

@@ -16,7 +16,6 @@
 - name: Install the ceilometer components
   hosts: ceilometer_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-cinder-install.yml

@@ -140,7 +140,6 @@
   hosts: cinder_backup,cinder_volume,cinder_scheduler
   gather_facts: no
   serial: "{{ cinder_backend_serial | default(['1', '100%']) }}"
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
@@ -166,7 +165,6 @@
   hosts: cinder_api
   gather_facts: no
   serial: "{{ cinder_api_serial | default(['1','100%']) }}"
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:

playbooks/os-glance-install.yml

@@ -81,7 +81,6 @@
 - name: Refresh local facts after all software changes are made
   hosts: glance_all
   gather_facts: no
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:

playbooks/os-gnocchi-install.yml

@@ -15,7 +15,6 @@
 
 - name: Install Gnocchi components
   hosts: gnocchi_all
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-heat-install.yml

@@ -16,7 +16,6 @@
 - name: Install heat server
   hosts: heat_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-horizon-install.yml

@@ -16,7 +16,6 @@
 - name: Install horizon server
   hosts: horizon_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-keystone-install.yml

@@ -62,7 +62,6 @@
   hosts: keystone_all
   serial: "{{ keystone_serial | default(['1', '100%']) }}"
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
@@ -149,7 +148,6 @@
 - name: Finalise data migrations if required
   hosts: keystone_all
   gather_facts: no
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:

playbooks/os-magnum-install.yml

@@ -18,7 +18,6 @@
 
 - name: Install magnum server
   hosts: magnum_all
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-nova-install.yml

@@ -134,7 +134,6 @@
 - name: Refresh local facts after all software changes are made
   hosts: nova_all
   gather_facts: no
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
@@ -180,7 +179,6 @@
   hosts: "nova_all:!nova_api_placement:!nova_console"
   gather_facts: no
   serial: "{{ nova_serial | default('100%') }}"
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:
@@ -205,7 +203,6 @@
   hosts: "nova_api_placement:nova_console"
   gather_facts: no
   serial: "{{ nova_api_serial | default(['1', '100%']) }}"
-  max_fail_percentage: 20
   user: root
   environment: "{{ deployment_environment_variables | default({}) }}"
   tags:

playbooks/os-octavia-install.yml

@@ -16,7 +16,6 @@
 - name: Install octavia server
   hosts: octavia_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/os-swift-install.yml

@@ -16,7 +16,6 @@
 - name: Installation and setup of Swift
   hosts: swift_all:swift_remote_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml
@@ -59,7 +58,6 @@
 
 - name: Installation and setup of Swift
   hosts: swift_all
-  max_fail_percentage: 20
   user: root
   roles:
     - role: "rsyslog_client"

playbooks/os-swift-sync.yml

@@ -18,7 +18,6 @@
 - name: Synchronisation of swift ring and ssh keys
   hosts: swift_all:swift_remote_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   roles:
     - role: "os_swift"

playbooks/os-tempest-install.yml

@@ -16,7 +16,6 @@
 - name: Installation and setup of Tempest
   hosts: utility_all[0]
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   roles:
     - role: "os_tempest"

playbooks/rabbitmq-install.yml

@@ -17,7 +17,6 @@
   hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}"
   serial: 1
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 0
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml
@@ -38,7 +37,6 @@
 # http://www.rabbitmq.com/clustering.html#upgrading
 - name: Stop RabbitMQ nodes that are not the upgrader
   hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}[1:]"
-  max_fail_percentage: 0
   user: root
   tasks:
     - name: "Stop RabbitMQ"
@@ -50,7 +48,6 @@
 - name: Install rabbitmq server
   hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}"
   serial: 20%
-  max_fail_percentage: 20
   user: root
   roles:
     - role: "rabbitmq_server"

playbooks/repo-server.yml

@@ -16,7 +16,6 @@
 - name: Setup repo servers
   hosts: repo_all
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
 

playbooks/rsyslog-install.yml

@@ -16,7 +16,6 @@
 - name: Install rsyslog
   hosts: rsyslog
   gather_facts: "{{ osa_gather_facts | default(True) }}"
-  max_fail_percentage: 20
   user: root
   pre_tasks:
     - include: common-tasks/os-lxc-container-setup.yml

playbooks/utility-install.yml

@@ -15,7 +15,6 @@
 
 - name: Setup the utility location(s)
   hosts: utility_all
-  max_fail_percentage: 20
   user: root
   pre_tasks:
 

releasenotes/notes/remove-fax-fail-5038b4b6eb4951b3.yaml

@@ -0,0 +1,21 @@
+---
+other:
+  - The `max_fail_percentage` playbook option has been used with the default
+    playbooks since the first release of the playbooks back in **Icehouse**.
+    While the intention was to allow large-scale deployments to succeed in
+    cases where a single node fails due to transient issues, this option has
+    produced more problems that it solves. If a failure occurs that is transient
+    in nature but is under the set failure percentage the playbook will report a
+    success, which can cause silent failures depending on where the failure
+    happened. If a deployer finds themselves in this situation the problems are
+    are then compounded because the tools will report there are no known issues.
+    To ensure deployers have the best deployment experience and the most
+    accurate information a change has been made to remove the
+    `max_fail_percentage` option from all of the default playbooks. The removal
+    of this option has the side effect of requiring the deploy to skip specific
+    hosts should one need to be omitted from a run, but has the benefit of
+    eliminating silent, hard to track down, failures. To skip a failing host
+    for a given playbook run use the `--limit '!$HOSTNAME'` CLI switch for the
+    specific run. Once the issues have been resolved for the failing host rerun
+    the specific playbook without the `--limit` option to ensure everything
+    is in sync.