From c2743f5cca1930a553ce9f20b207cb17fa6b763d Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 22 Feb 2018 18:00:43 -0600 Subject: [PATCH] Remove the "max_fail_percentage" option This option can cause silent failures which are confusing and hard to track down. While the intention of this was to allow large scale deployments to succeed in cases where a single node fails due to transiant issues it has produced more problems in terms of confusion that it solves. This change removes the option from all production playbooks. Change-Id: I1dcbbf5bc8cc66f11dd8ddc22d2a177c5c0f31f1 Signed-off-by: Kevin Carter --- playbooks/ceph-install.yml | 2 -- playbooks/containers-lxc-create.yml | 1 - playbooks/containers-lxc-destroy.yml | 1 - playbooks/containers-lxc-host.yml | 1 - playbooks/containers-nspawn-destroy.yml | 1 - playbooks/etcd-install.yml | 1 - playbooks/galera-install.yml | 2 -- playbooks/haproxy-install.yml | 1 - playbooks/memcached-install.yml | 1 - playbooks/openstack-hosts-setup.yml | 1 - playbooks/os-aodh-install.yml | 1 - playbooks/os-barbican-install.yml | 1 - playbooks/os-ceilometer-install.yml | 1 - playbooks/os-cinder-install.yml | 2 -- playbooks/os-glance-install.yml | 1 - playbooks/os-gnocchi-install.yml | 1 - playbooks/os-heat-install.yml | 1 - playbooks/os-horizon-install.yml | 1 - playbooks/os-keystone-install.yml | 2 -- playbooks/os-magnum-install.yml | 1 - playbooks/os-nova-install.yml | 3 --- playbooks/os-octavia-install.yml | 1 - playbooks/os-swift-install.yml | 2 -- playbooks/os-swift-sync.yml | 1 - playbooks/os-tacker-install.yml | 1 - playbooks/os-tempest-install.yml | 1 - playbooks/rabbitmq-install.yml | 3 --- playbooks/repo-server.yml | 1 - playbooks/rsyslog-install.yml | 1 - playbooks/utility-install.yml | 1 - .../remove-fax-fail-5038b4b6eb4951b3.yaml | 21 +++++++++++++++++++ 31 files changed, 21 insertions(+), 39 deletions(-) create mode 100644 releasenotes/notes/remove-fax-fail-5038b4b6eb4951b3.yaml diff --git a/playbooks/ceph-install.yml b/playbooks/ceph-install.yml index cdaa630e0e..589e3f579d 100644 --- a/playbooks/ceph-install.yml +++ b/playbooks/ceph-install.yml @@ -15,7 +15,6 @@ - name: Install ceph mons hosts: ceph-mon - max_fail_percentage: 20 user: root serial: 1 pre_tasks: @@ -124,7 +123,6 @@ - name: Install ceph osds hosts: ceph-osd - max_fail_percentage: 20 user: root pre_tasks: - include: common-tasks/os-log-dir-setup.yml diff --git a/playbooks/containers-lxc-create.yml b/playbooks/containers-lxc-create.yml index 8b73adaa0a..c35f119c76 100644 --- a/playbooks/containers-lxc-create.yml +++ b/playbooks/containers-lxc-create.yml @@ -34,7 +34,6 @@ - name: Create container(s) hosts: all_lxc_containers gather_facts: false - max_fail_percentage: 20 user: root roles: - role: "lxc_container_create" diff --git a/playbooks/containers-lxc-destroy.yml b/playbooks/containers-lxc-destroy.yml index 5ebe594fcb..99eb45ecbe 100644 --- a/playbooks/containers-lxc-destroy.yml +++ b/playbooks/containers-lxc-destroy.yml @@ -30,7 +30,6 @@ - name: Destroy lxc containers hosts: all_lxc_containers gather_facts: false - max_fail_percentage: 20 user: root tasks: - name: Destroy a container diff --git a/playbooks/containers-lxc-host.yml b/playbooks/containers-lxc-host.yml index 580a438985..06ebeda726 100644 --- a/playbooks/containers-lxc-host.yml +++ b/playbooks/containers-lxc-host.yml @@ -16,7 +16,6 @@ - name: Basic lxc host setup hosts: "{{ lxc_host_group | default('lxc_hosts')}}" gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: common-tasks/set-upper-constraints.yml diff --git a/playbooks/containers-nspawn-destroy.yml b/playbooks/containers-nspawn-destroy.yml index fb4636f772..d09646e794 100644 --- a/playbooks/containers-nspawn-destroy.yml +++ b/playbooks/containers-nspawn-destroy.yml @@ -30,7 +30,6 @@ - name: Destroy nspawn containers hosts: all_nspawn_containers gather_facts: false - max_fail_percentage: 20 user: root tasks: - name: Get container status diff --git a/playbooks/etcd-install.yml b/playbooks/etcd-install.yml index 0e429a9db0..8c47e0065a 100644 --- a/playbooks/etcd-install.yml +++ b/playbooks/etcd-install.yml @@ -16,7 +16,6 @@ - name: Install etcd server cluster hosts: etcd_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/galera-install.yml b/playbooks/galera-install.yml index cf32b3dab4..72d29b7f71 100644 --- a/playbooks/galera-install.yml +++ b/playbooks/galera-install.yml @@ -17,7 +17,6 @@ hosts: galera_all gather_facts: "{{ osa_gather_facts | default(True) }}" serial: 1 - max_fail_percentage: 0 user: root tasks: - include: common-tasks/os-log-dir-setup.yml @@ -43,7 +42,6 @@ hosts: galera_all gather_facts: "{{ osa_gather_facts | default(True) }}" serial: 1 - max_fail_percentage: 20 user: root roles: - role: haproxy_endpoints diff --git a/playbooks/haproxy-install.yml b/playbooks/haproxy-install.yml index 0246e67cc8..303b69d5dd 100644 --- a/playbooks/haproxy-install.yml +++ b/playbooks/haproxy-install.yml @@ -41,7 +41,6 @@ - name: Install haproxy hosts: haproxy gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - name: Remove legacy haproxy configuration files diff --git a/playbooks/memcached-install.yml b/playbooks/memcached-install.yml index 42b9f7a4d5..aaefa22cfe 100644 --- a/playbooks/memcached-install.yml +++ b/playbooks/memcached-install.yml @@ -16,7 +16,6 @@ - name: Install memcached hosts: memcached gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/openstack-hosts-setup.yml b/playbooks/openstack-hosts-setup.yml index f5c708f1f5..485478735e 100644 --- a/playbooks/openstack-hosts-setup.yml +++ b/playbooks/openstack-hosts-setup.yml @@ -39,7 +39,6 @@ - name: Basic host setup hosts: "{{ openstack_host_group|default('hosts') }}" gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - name: Check for a supported Operating System diff --git a/playbooks/os-aodh-install.yml b/playbooks/os-aodh-install.yml index 694bebdd10..ebf3992fd6 100644 --- a/playbooks/os-aodh-install.yml +++ b/playbooks/os-aodh-install.yml @@ -16,7 +16,6 @@ - name: Install the aodh components hosts: aodh_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-barbican-install.yml b/playbooks/os-barbican-install.yml index 6362b55d00..7f66fc59bb 100644 --- a/playbooks/os-barbican-install.yml +++ b/playbooks/os-barbican-install.yml @@ -16,7 +16,6 @@ - name: Installation and setup of barbican hosts: barbican_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-ceilometer-install.yml b/playbooks/os-ceilometer-install.yml index 47dc8bcbcf..f9fcc92f63 100644 --- a/playbooks/os-ceilometer-install.yml +++ b/playbooks/os-ceilometer-install.yml @@ -16,7 +16,6 @@ - name: Install the ceilometer components hosts: ceilometer_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-cinder-install.yml b/playbooks/os-cinder-install.yml index b588e94767..23f83b8ec6 100644 --- a/playbooks/os-cinder-install.yml +++ b/playbooks/os-cinder-install.yml @@ -139,7 +139,6 @@ hosts: cinder_backup,cinder_volume,cinder_scheduler gather_facts: no serial: "{{ cinder_backend_serial | default(['1', '100%']) }}" - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: @@ -165,7 +164,6 @@ hosts: cinder_api gather_facts: no serial: "{{ cinder_api_serial | default(['1','100%']) }}" - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: diff --git a/playbooks/os-glance-install.yml b/playbooks/os-glance-install.yml index 257f6dc03c..fdd2b1c2a9 100644 --- a/playbooks/os-glance-install.yml +++ b/playbooks/os-glance-install.yml @@ -80,7 +80,6 @@ - name: Refresh local facts after all software changes are made hosts: glance_all gather_facts: no - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: diff --git a/playbooks/os-gnocchi-install.yml b/playbooks/os-gnocchi-install.yml index 5c222c63d2..bb70103477 100644 --- a/playbooks/os-gnocchi-install.yml +++ b/playbooks/os-gnocchi-install.yml @@ -15,7 +15,6 @@ - name: Install Gnocchi components hosts: gnocchi_all - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-heat-install.yml b/playbooks/os-heat-install.yml index c18cc35e5b..2f52f50736 100644 --- a/playbooks/os-heat-install.yml +++ b/playbooks/os-heat-install.yml @@ -16,7 +16,6 @@ - name: Install heat server hosts: heat_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-horizon-install.yml b/playbooks/os-horizon-install.yml index 4374742e64..07888fcfb9 100644 --- a/playbooks/os-horizon-install.yml +++ b/playbooks/os-horizon-install.yml @@ -16,7 +16,6 @@ - name: Install horizon server hosts: horizon_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-keystone-install.yml b/playbooks/os-keystone-install.yml index c0ecbc6de6..03e08e84d7 100644 --- a/playbooks/os-keystone-install.yml +++ b/playbooks/os-keystone-install.yml @@ -62,7 +62,6 @@ hosts: keystone_all serial: "{{ keystone_serial | default(['1', '100%']) }}" gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: @@ -154,7 +153,6 @@ - name: Finalise data migrations if required hosts: keystone_all gather_facts: no - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: diff --git a/playbooks/os-magnum-install.yml b/playbooks/os-magnum-install.yml index 982d614a4c..385742b7c7 100644 --- a/playbooks/os-magnum-install.yml +++ b/playbooks/os-magnum-install.yml @@ -18,7 +18,6 @@ - name: Install magnum server hosts: magnum_all - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-nova-install.yml b/playbooks/os-nova-install.yml index c2e674cf38..926c8f001d 100644 --- a/playbooks/os-nova-install.yml +++ b/playbooks/os-nova-install.yml @@ -125,7 +125,6 @@ - name: Refresh local facts after all software changes are made hosts: nova_all gather_facts: no - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: @@ -171,7 +170,6 @@ hosts: "nova_all:!nova_api_placement:!nova_console" gather_facts: no serial: "{{ nova_serial | default('100%') }}" - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: @@ -196,7 +194,6 @@ hosts: "nova_api_placement:nova_console" gather_facts: no serial: "{{ nova_api_serial | default(['1', '100%']) }}" - max_fail_percentage: 20 user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: diff --git a/playbooks/os-octavia-install.yml b/playbooks/os-octavia-install.yml index 1c97926b1b..317105955e 100644 --- a/playbooks/os-octavia-install.yml +++ b/playbooks/os-octavia-install.yml @@ -16,7 +16,6 @@ - name: Install octavia server hosts: octavia_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-swift-install.yml b/playbooks/os-swift-install.yml index 9a65b42491..e5c77b37b8 100644 --- a/playbooks/os-swift-install.yml +++ b/playbooks/os-swift-install.yml @@ -16,7 +16,6 @@ - name: Installation and setup of Swift hosts: swift_all:swift_remote_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" @@ -61,7 +60,6 @@ - name: Installation and setup of Swift hosts: swift_all - max_fail_percentage: 20 user: root roles: - role: "rsyslog_client" diff --git a/playbooks/os-swift-sync.yml b/playbooks/os-swift-sync.yml index 8af0bade67..7d9f7a2e4e 100644 --- a/playbooks/os-swift-sync.yml +++ b/playbooks/os-swift-sync.yml @@ -18,7 +18,6 @@ - name: Synchronisation of swift ring and ssh keys hosts: swift_all:swift_remote_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root roles: - role: "os_swift" diff --git a/playbooks/os-tacker-install.yml b/playbooks/os-tacker-install.yml index 4991321a35..c9a1d74ae6 100644 --- a/playbooks/os-tacker-install.yml +++ b/playbooks/os-tacker-install.yml @@ -16,7 +16,6 @@ - name: Install the tacker components hosts: tacker_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/os-tempest-install.yml b/playbooks/os-tempest-install.yml index c56d1b13a9..e8b4543451 100644 --- a/playbooks/os-tempest-install.yml +++ b/playbooks/os-tempest-install.yml @@ -16,7 +16,6 @@ - name: Installation and setup of Tempest hosts: utility_all[0] gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root roles: - role: "os_tempest" diff --git a/playbooks/rabbitmq-install.yml b/playbooks/rabbitmq-install.yml index 0a3556b02d..26f56e05eb 100644 --- a/playbooks/rabbitmq-install.yml +++ b/playbooks/rabbitmq-install.yml @@ -17,7 +17,6 @@ hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}" serial: 1 gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 0 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" @@ -40,7 +39,6 @@ # http://www.rabbitmq.com/clustering.html#upgrading - name: Stop RabbitMQ nodes that are not the upgrader hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}[1:]" - max_fail_percentage: 0 user: root tasks: - name: "Stop RabbitMQ" @@ -52,7 +50,6 @@ - name: Install rabbitmq server hosts: "{{ rabbitmq_host_group | default('rabbitmq_all') }}" serial: 20% - max_fail_percentage: 20 user: root roles: - role: "rabbitmq_server" diff --git a/playbooks/repo-server.yml b/playbooks/repo-server.yml index d60070305c..ddb70b310f 100644 --- a/playbooks/repo-server.yml +++ b/playbooks/repo-server.yml @@ -16,7 +16,6 @@ - name: Setup repo servers hosts: repo_all gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: diff --git a/playbooks/rsyslog-install.yml b/playbooks/rsyslog-install.yml index 381e1865ea..24a0b1c408 100644 --- a/playbooks/rsyslog-install.yml +++ b/playbooks/rsyslog-install.yml @@ -16,7 +16,6 @@ - name: Install rsyslog hosts: rsyslog gather_facts: "{{ osa_gather_facts | default(True) }}" - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/playbooks/utility-install.yml b/playbooks/utility-install.yml index 7a4ed4d9cb..019cddff75 100644 --- a/playbooks/utility-install.yml +++ b/playbooks/utility-install.yml @@ -15,7 +15,6 @@ - name: Setup the utility location(s) hosts: utility_all - max_fail_percentage: 20 user: root pre_tasks: - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" diff --git a/releasenotes/notes/remove-fax-fail-5038b4b6eb4951b3.yaml b/releasenotes/notes/remove-fax-fail-5038b4b6eb4951b3.yaml new file mode 100644 index 0000000000..4f54d7c293 --- /dev/null +++ b/releasenotes/notes/remove-fax-fail-5038b4b6eb4951b3.yaml @@ -0,0 +1,21 @@ +--- +other: + - The `max_fail_percentage` playbook option has been used with the default + playbooks since the first release of the playbooks back in **Icehouse**. + While the intention was to allow large-scale deployments to succeed in + cases where a single node fails due to transient issues, this option has + produced more problems that it solves. If a failure occurs that is transient + in nature but is under the set failure percentage the playbook will report a + success, which can cause silent failures depending on where the failure + happened. If a deployer finds themselves in this situation the problems are + are then compounded because the tools will report there are no known issues. + To ensure deployers have the best deployment experience and the most + accurate information a change has been made to remove the + `max_fail_percentage` option from all of the default playbooks. The removal + of this option has the side effect of requiring the deploy to skip specific + hosts should one need to be omitted from a run, but has the benefit of + eliminating silent, hard to track down, failures. To skip a failing host + for a given playbook run use the `--limit '!$HOSTNAME'` CLI switch for the + specific run. Once the issues have been resolved for the failing host rerun + the specific playbook without the `--limit` option to ensure everything + is in sync.