Disable chrony container by default

Fixes an issue where multiple NTP daemons could be running on the
overcloud hosts, due to Kolla Ansible deploying a chrony container by
default starting with the Rocky release.

Kayobe now overrides this default, to ensure that chrony does not conflict
with the NTP daemon deployed on the host. To use the containerised chrony
daemon instead, set ``kolla_enable_chrony`` to ``true`` in
``${KAYOBE_CONFIG_PATH}/kolla.yml``. This will also disable the host NTP
daemon.

To ensure that chrony is not running, Kayobe removes the chrony container
if ``kolla_enable_chrony`` is ``false`` in the following commands:

* ``kayobe overcloud service deploy``
* ``kayobe overcloud service reconfigure``
* ``kayobe overcloud service upgrade``

The play in Kayobe is tagged with ``stop-chrony``.

Change-Id: I89a973c0b600abece79bddcba5a46cc28a4f1df9
Story: 2005272
Task: 30122

ansible/group_vars/all/kolla

@@ -339,6 +339,9 @@ kolla_enable_blazar: "no"
 kolla_enable_central_logging: "no"
 kolla_enable_ceph: "no"
 kolla_enable_ceilometer: "no"
+# The chrony container is disabled by default because we enable an NTP daemon
+# on the host. Setting this to true will disable NTP on the host.
+kolla_enable_chrony: "no"
 kolla_enable_cinder: "no"
 kolla_enable_collectd: "no"
 kolla_enable_designate: "no"

ansible/group_vars/all/ntp

@@ -10,6 +10,13 @@ timezone: "{{ ansible_date_time.tz }}"
 ###############################################################################
 # Network Time Protocol (NTP).
 
+# Whether to enable the NTP daemon on the host. Default is true unless
+# 'kolla_enable_chrony' has been set to true on overcloud hosts.
+ntp_service_enabled: "{{ 'overcloud' not in group_names or not kolla_enable_chrony | bool }}"
+
+ntp_package_state: "{{ 'present' if ntp_service_enabled | bool else 'absent' }}"
+ntp_service_state: "{{ 'started' if ntp_service_enabled | bool else 'stopped' }}"
+
 # List of names of NTP servers.
 #ntp_config_server:
 

ansible/kolla-ansible.yml

@@ -299,3 +299,6 @@
         kolla_inspector_dhcp_pool_start: "{{ inspection_net_name | net_inspection_allocation_pool_start }}"
         kolla_inspector_dhcp_pool_end: "{{ inspection_net_name | net_inspection_allocation_pool_end }}"
         kolla_inspector_default_gateway: "{{ inspection_net_name | net_inspection_gateway or inspection_net_name | net_gateway }}"
+        # While kayobe has its own support for installing an NTP daemon, the
+        # kolla-ansible baremetal role does a one-time sync which is useful.
+        kolla_enable_host_ntp: "{{ ntp_service_enabled }}"

ansible/roles/kolla-ansible/defaults/main.yml

@@ -303,3 +303,9 @@ kolla_tls_cert:
 
 # Desired SELinux state.
 kolla_selinux_state:
+
+###############################################################################
+# NTP
+
+# Whether to enable the NTP daemon.
+kolla_enable_host_ntp:

ansible/roles/kolla-ansible/templates/globals.yml.j2

@@ -384,6 +384,10 @@ selinux_state: {{ kolla_selinux_state }}
 
 install_epel: {{ kolla_ansible_install_epel | bool }}
 
+{% if kolla_enable_host_ntp is not none %}
+enable_host_ntp: {{ kolla_enable_host_ntp | bool }}
+{% endif %}
+
 {% if kolla_extra_globals %}
 #######################
 # Extra configuration

ansible/roles/kolla-ansible/tests/test-extras.yml

@@ -109,6 +109,7 @@
             kolla_enable_central_logging: True
             kolla_enable_ceph: True
             kolla_enable_ceph_rgw: True
+            kolla_enable_chrony: True
             kolla_enable_cinder: True
             kolla_enable_cinder_backend_hnas_iscsi: True
             kolla_enable_cinder_backend_hnas_nfs: True
@@ -243,6 +244,7 @@
               #enable_central_logging: True
               #enable_ceph: True
               #enable_ceph_rgw: True
+              #enable_chrony: True
               #enable_cinder: True
               #enable_cinder_backend_iscsi: True
               #enable_cinder_backend_hnas_iscsi: True

ansible/roles/kolla-ansible/vars/main.yml

@@ -80,6 +80,7 @@ kolla_feature_flags:
   - ceph_mds
   - ceph_nfs
   - ceph_rgw
+  - chrony
   - cinder
   - cinder_backend_hnas_iscsi
   - cinder_backend_hnas_nfs

ansible/stop-chrony.yml

@@ -0,0 +1,17 @@
+---
+# NOTE(mgoddard): In the Rocky release, Kolla Ansible enabled the chrony
+# container by default. Running this alongside the NTP daemon on the host is
+# likely to cause issues, so we should explicitly disable it. Kolla Ansible
+# won't automatically stop the container, so we do it here if it is disabled.
+# See https://storyboard.openstack.org/#!/story/2005272.
+
+- name: Stop the chrony container
+  hosts: overcloud
+  tags:
+    - stop-chrony
+  tasks:
+    - name: Stop the chrony container
+      docker_container:
+        name: chrony
+        state: absent
+      when: not kolla_enable_chrony | bool

doc/source/configuration/hosts.rst

@@ -355,10 +355,21 @@ The NTP service may be disabled as follows:
 .. code-block:: yaml
    :caption: ``ntp.yml``
 
-   ntp_package_state: absent
-   ntp_service_state: stopped
    ntp_service_enabled: false
 
+Chrony
+------
+
+Kolla Ansible can deploy a chrony container. This is disabled by default in
+Kayobe to avoid conflicting with the NTP daemon on the host.
+
+To use the containerised chrony daemon and disable the host NTP daemon, set the
+following in ``${KAYOBE_CONFIG_PATH}/kolla.yml``:
+
+.. code-block:: yaml
+
+   kolla_enable_chrony: true
+
 .. _configuration-hosts-mdadm:
 
 Software RAID

etc/kayobe/kolla.yml

@@ -171,6 +171,9 @@
 #kolla_enable_ceph_mds:
 #kolla_enable_ceph_nfs:
 #kolla_enable_ceph_rgw:
+# The chrony container is disabled by default because we enable an NTP daemon
+# on the host. Setting this to true will disable NTP on the host.
+#kolla_enable_chrony:
 #kolla_enable_cinder:
 #kolla_enable_cinder_backend_hnas_iscsi:
 #kolla_enable_cinder_backend_hnas_nfs:

etc/kayobe/ntp.yml

@@ -10,6 +10,10 @@
 ###############################################################################
 # Network Time Protocol (NTP).
 
+# Whether to enable the NTP daemon on the host. Default is true unless
+# 'kolla_enable_chrony' has been set to true on overcloud hosts.
+#ntp_service_enabled:
+
 # List of names of NTP servers.
 #ntp_config_server:
 

kayobe/cli/commands.py

@@ -621,6 +621,7 @@ class SeedServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
 
         playbooks = _build_playbook_list("kolla-bifrost")
         self.run_kayobe_playbooks(parsed_args, playbooks)
+
         self.run_kolla_ansible_seed(parsed_args, "deploy-bifrost")
         playbooks = _build_playbook_list(
             "overcloud-host-image-workaround-resolv",
@@ -656,6 +657,7 @@ class SeedServiceUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
             "kolla-bifrost",
             "seed-service-upgrade-prep")
         self.run_kayobe_playbooks(parsed_args, playbooks)
+
         self.run_kolla_ansible_seed(parsed_args, "deploy-bifrost")
         playbooks = _build_playbook_list(
             "overcloud-host-image-workaround-resolv",
@@ -1170,6 +1172,11 @@ class OvercloudServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin,
         if not parsed_args.skip_prechecks:
             self.run_kolla_ansible_overcloud(parsed_args, "prechecks")
 
+        # Workaround: Stop the chrony container.
+        # TODO(mgoddard): Remove in the Train cycle.
+        playbooks = _build_playbook_list("stop-chrony")
+        self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
+
         # Perform the kolla-ansible deployment.
         self.run_kolla_ansible_overcloud(parsed_args, "deploy")
 
@@ -1228,6 +1235,11 @@ class OvercloudServiceReconfigure(KollaAnsibleMixin, KayobeAnsibleMixin,
         if not parsed_args.skip_prechecks:
             self.run_kolla_ansible_overcloud(parsed_args, "prechecks")
 
+        # Workaround: Stop the chrony container.
+        # TODO(mgoddard): Remove in the Train cycle.
+        playbooks = _build_playbook_list("stop-chrony")
+        self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
+
         # Perform the kolla-ansible reconfiguration.
         self.run_kolla_ansible_overcloud(parsed_args, "reconfigure")
 
@@ -1281,6 +1293,11 @@ class OvercloudServiceUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin,
         if not parsed_args.skip_prechecks:
             self.run_kolla_ansible_overcloud(parsed_args, "prechecks")
 
+        # Workaround: Stop the chrony container.
+        # TODO(mgoddard): Remove in the Train cycle.
+        playbooks = _build_playbook_list("stop-chrony")
+        self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud")
+
         # Perform the kolla-ansible upgrade.
         self.run_kolla_ansible_overcloud(parsed_args, "upgrade")
 

kayobe/tests/unit/cli/test_commands.py

@@ -1422,6 +1422,202 @@ class TestCase(unittest.TestCase):
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
 
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_service_deploy(self, mock_kolla_run, mock_run):
+        command = commands.OvercloudServiceDeploy(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [utils.get_data_files_path("ansible", "kolla-ansible.yml")],
+                tags="config",
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible",
+                                              "kolla-openstack.yml"),
+                ],
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible", "stop-chrony.yml"),
+                ],
+                limit="overcloud",
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible",
+                                              "overcloud-extras.yml"),
+                ],
+                limit="overcloud",
+                extra_vars={
+                    "kayobe_action": "deploy",
+                },
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible", "public-openrc.yml"),
+                ],
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "prechecks",
+            ),
+            mock.call(
+                mock.ANY,
+                "deploy",
+            ),
+            mock.call(
+                mock.ANY,
+                "post-deploy",
+                extra_vars={
+                    "node_config_directory": "/etc/kolla",
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_service_reconfigure(self, mock_kolla_run, mock_run):
+        command = commands.OvercloudServiceReconfigure(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [utils.get_data_files_path("ansible", "kolla-ansible.yml")],
+                tags="config",
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible",
+                                              "kolla-openstack.yml"),
+                ],
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible", "stop-chrony.yml"),
+                ],
+                limit="overcloud",
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible",
+                                              "overcloud-extras.yml"),
+                ],
+                limit="overcloud",
+                extra_vars={
+                    "kayobe_action": "reconfigure",
+                },
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible", "public-openrc.yml"),
+                ],
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "prechecks",
+            ),
+            mock.call(
+                mock.ANY,
+                "reconfigure",
+            ),
+            mock.call(
+                mock.ANY,
+                "post-deploy",
+                extra_vars={
+                    "node_config_directory": "/etc/kolla",
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    @mock.patch.object(commands.KollaAnsibleMixin,
+                       "run_kolla_ansible_overcloud")
+    def test_overcloud_service_upgrade(self, mock_kolla_run, mock_run):
+        command = commands.OvercloudServiceUpgrade(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible", "kolla-ansible.yml"),
+                    utils.get_data_files_path("ansible",
+                                              "kolla-openstack.yml"),
+                ]
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible", "stop-chrony.yml"),
+                ],
+                limit="overcloud"
+            ),
+            mock.call(
+                mock.ANY,
+                [
+                    utils.get_data_files_path("ansible",
+                                              "overcloud-extras.yml"),
+                ],
+                limit="overcloud",
+                extra_vars={
+                    "kayobe_action": "upgrade",
+                }
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)
+
+        expected_calls = [
+            mock.call(
+                mock.ANY,
+                "prechecks"
+            ),
+            mock.call(
+                mock.ANY,
+                "upgrade"
+            ),
+        ]
+        self.assertEqual(expected_calls, mock_kolla_run.call_args_list)
+
     @mock.patch.object(commands.KayobeAnsibleMixin,
                        "run_kayobe_playbooks")
     def test_overcloud_service_configuration_save_args(self, mock_run):

releasenotes/notes/disable-chrony-5f1e4f9db509c42d.yaml

@@ -0,0 +1,24 @@
+---
+fixes:
+  - |
+    Fixes an issue where multiple NTP daemons could be running on the overcloud
+    hosts, due to Kolla Ansible deploying a chrony container by default
+    starting with the Rocky release.
+
+    Kayobe now overrides this default, to ensure that chrony does not conflict
+    with the NTP daemon deployed on the host. To use the containerised chrony
+    daemon instead, set ``kolla_enable_chrony`` to ``true`` in
+    ``${KAYOBE_CONFIG_PATH}/kolla.yml``. This will also disable the host NTP
+    daemon.
+
+    To ensure that chrony is not running, Kayobe removes the chrony container
+    if ``kolla_enable_chrony`` is ``false`` in the following commands:
+
+    * ``kayobe overcloud service deploy``
+    * ``kayobe overcloud service reconfigure``
+    * ``kayobe overcloud service upgrade``
+
+    The play in Kayobe is tagged with ``stop-chrony``.
+
+    See `story 2005272 <https://storyboard.openstack.org/#!/story/2005272>`__
+    for details.