Merge "Extend the nova_scheduler healthcheck timeout."

ansible/roles/nova/defaults/main.yml

@@ -168,7 +168,11 @@ nova_metadata_healthcheck:
 
 nova_scheduler_enable_healthchecks: "{{ enable_container_healthchecks }}"
 nova_scheduler_healthcheck_interval: "{{ default_container_healthcheck_interval }}"
-nova_scheduler_healthcheck_retries: "{{ default_container_healthcheck_retries }}"
+
+# NOTE(mikal): this one is deliberately overridden because of the issues discussed
+# in releasenotes/notes/nova-scheduler-healthcheck-retries-12e681b16f6322aa.yaml
+nova_scheduler_healthcheck_retries: 8
+
 nova_scheduler_healthcheck_start_period: "{{ default_container_healthcheck_start_period }}"
 nova_scheduler_healthcheck_test: ["CMD-SHELL", "healthcheck_port nova-scheduler {{ om_rpc_port }}"]
 nova_scheduler_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}"

etc/kolla/globals.yml

@@ -209,6 +209,21 @@ workaround_ansible_issue_8743: true
 #default_container_healthcheck_timeout: 30
 #default_container_healthcheck_retries: 3
 #default_container_healthcheck_start_period: 5
+# Individual services accept per-service overrides of the options above,
+# named <service>_healthcheck_<option>. nova_scheduler ships with a wider
+# one: its healthcheck asserts an established AMQP connection, but every
+# deploy and every reconfigure signals the scheduler to restart its
+# workers, and they take [DEFAULT]/manager_shutdown_timeout in nova.conf
+# (160 seconds by default) to come back. The default 30 second interval
+# with 3 retries only covers 90 seconds of that.
+#nova_scheduler_healthcheck_retries: 8
+# The conductors wait for the same period, but are only ever stopped by
+# the container engine, which kills them after docker_graceful_timeout
+# and so stays inside what the default healthcheck covers. Raise these
+# to match if docker_graceful_timeout is raised towards
+# manager_shutdown_timeout.
+#nova_conductor_healthcheck_retries: 3
+#nova_super_conductor_healthcheck_retries: 3
 
 ##################
 # Firewall options

releasenotes/notes/nova-scheduler-healthcheck-retries-12e681b16f6322aa.yaml

@@ -0,0 +1,13 @@
+---
+fixes:
+  - |
+    The ``nova_scheduler`` container healthcheck asserts an established AMQP
+    connection, but every deploy and every reconfigure signals the scheduler
+    to restart its workers and they take
+    ``[DEFAULT]/manager_shutdown_timeout`` (160 seconds by default) to come
+    back. The default 30 second interval with 3 retries only covers 90
+    seconds of that, so the container could be reported unhealthy for the
+    rest of the restart even though the scheduler was fine.
+    ``nova_scheduler_healthcheck_retries`` now defaults to 8, which covers
+    the whole window, and is documented in the sample ``globals.yml``.
+    `LP#2162860 <https://bugs.launchpad.net/kolla/+bug/2162860>`__