From 8dd05487d4ef872e7d3467e74fd6ab5141361d68 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 6 Feb 2025 18:56:32 +0100 Subject: [PATCH] Replace pause with chronyc waitsync in ntp sync `chronyc waitsync 60 0.01` will wait up to about 10 minutes (60 times 10 seconds) for chronyd to synchronise to a source and the remaining correction to be less than 10 milliseconds [1]. Also flush handlers to restart chronyd before synchronising the clock. Force system clock synchronisation in CI jobs. This allows us to run test_ntp_clock_synchronized on Ubuntu again. [1] https://chrony-project.org/doc/4.6.1/chronyc.html Change-Id: I73e7f2a3258e5427ec16b01e7482093d7bfb33f8 --- ansible/inventory/group_vars/all/time | 14 ++++++++++++++ ansible/roles/ntp/tasks/sync.yml | 8 ++++++-- etc/kayobe/time.yml | 14 ++++++++++++++ playbooks/kayobe-infra-vm-base/overrides.yml.j2 | 3 +++ playbooks/kayobe-overcloud-base/overrides.yml.j2 | 3 +++ .../overrides.yml.j2 | 3 +++ .../tests/test_overcloud_host_configure.py | 3 --- .../kayobe-overcloud-upgrade-base/overrides.yml.j2 | 3 +++ playbooks/kayobe-seed-base/overrides.yml.j2 | 3 +++ .../kayobe-seed-upgrade-base/overrides.yml.j2 | 3 +++ playbooks/kayobe-seed-vm-base/overrides.yml.j2 | 3 +++ 11 files changed, 55 insertions(+), 5 deletions(-) diff --git a/ansible/inventory/group_vars/all/time b/ansible/inventory/group_vars/all/time index 6a60b7c15..4f84ce60d 100644 --- a/ansible/inventory/group_vars/all/time +++ b/ansible/inventory/group_vars/all/time @@ -38,3 +38,17 @@ chrony_ntp_servers: # Synchronise hardware clock with system time. Default is true. chrony_rtcsync_enabled: true + +# Force synchronisation from NTP sources. This methods may jump the clock by +# large values which can cause issues with some software. Disabled by default. +ntp_force_sync: false + +# Maximum number of tries used by the `chronyc waitsync` command. Only used +# when ntp_force_sync is true. Default is 60 which waits for a maximum of 10 +# minutes (60 times 10 seconds). +chrony_waitsync_max_tries: 60 + +# Maximum correction used by the `chronyc waitsync` command. Only used when +# ntp_force_sync is true. Default is 0.01 which waits for the remaining +# correction to be less than 10 milliseconds. +chrony_waitsync_max_correction: 0.01 diff --git a/ansible/roles/ntp/tasks/sync.yml b/ansible/roles/ntp/tasks/sync.yml index 6200c1006..39d7536c8 100644 --- a/ansible/roles/ntp/tasks/sync.yml +++ b/ansible/roles/ntp/tasks/sync.yml @@ -1,4 +1,9 @@ --- +# NOTE(priteau): We need to run handlers to restart chronyd before forcing +# clock synchronisation, or the system clock could appear as not synchronised +# during CI jobs because the tests run too quickly after the chronyd restart. +- meta: flush_handlers + - name: Force time synchronisation become: True block: @@ -9,8 +14,7 @@ command: chronyc makestep - name: Wait before hardware clock synchronisation - pause: - seconds: 5 + command: "chronyc waitsync {{ chrony_waitsync_max_tries }} {{ chrony_waitsync_max_correction }}" - name: Ensure hwclock is installed package: diff --git a/etc/kayobe/time.yml b/etc/kayobe/time.yml index 8bfbd3391..652844c8d 100644 --- a/etc/kayobe/time.yml +++ b/etc/kayobe/time.yml @@ -33,6 +33,20 @@ # Synchronise hardware clock with system time. Default is true. #chrony_rtcsync_enabled: +# Force synchronisation from NTP sources. This methods may jump the clock by +# large values which can cause issues with some software. Disabled by default. +#ntp_force_sync: + +# Maximum number of tries used by the `chronyc waitsync` command. Only used +# when ntp_force_sync is true. Default is 60 which waits for a maximum of 10 +# minutes (60 times 10 seconds). +#chrony_waitsync_max_tries: + +# Maximum correction used by the `chronyc waitsync` command. Only used when +# ntp_force_sync is true. Default is 0.01 which waits for the remaining +# correction to be less than 10 milliseconds. +#chrony_waitsync_max_correction: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/playbooks/kayobe-infra-vm-base/overrides.yml.j2 b/playbooks/kayobe-infra-vm-base/overrides.yml.j2 index 3493adab5..0c3865b48 100644 --- a/playbooks/kayobe-infra-vm-base/overrides.yml.j2 +++ b/playbooks/kayobe-infra-vm-base/overrides.yml.j2 @@ -15,6 +15,9 @@ pip_upper_constraints_file: "/tmp/upper-constraints.txt" # QEMU. libvirt_vm_engine: "qemu" +# Force system clock synchronisation +ntp_force_sync: True + # Use the CI infra's PyPI mirror. pip_local_mirror: true pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2 index 35efa6777..c13f16f1a 100644 --- a/playbooks/kayobe-overcloud-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2 @@ -15,6 +15,9 @@ kolla_ansible_requirements_yml: "/tmp/kolla-ansible-requirements.yml" kolla_openstack_logging_debug: True pip_upper_constraints_file: "/tmp/upper-constraints.txt" +# Force system clock synchronisation +ntp_force_sync: True + # Use the CI infra's PyPI mirror. pip_local_mirror: true pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" diff --git a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 index 5e52d2f5e..91bb0a0db 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 @@ -196,6 +196,9 @@ chrony_ntp_servers: - option: maxsources val: 2 +# Force system clock synchronisation +ntp_force_sync: True + # Enable firewalld controller_firewalld_enabled: true controller_firewalld_zones: diff --git a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py index 1f404e40a..6c10068fe 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py +++ b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py @@ -204,9 +204,6 @@ def test_ntp_non_default_time_server(host): assert "time.cloudflare.com" in chrony_config.content_string -# TODO(priteau): Remove once we force time sync -@pytest.mark.skipif(_is_ubuntu_noble(), - reason="Clock often fails to synchronize on Ubuntu Noble") def test_ntp_clock_synchronized(host): # Tests that the clock is synchronized status_output = host.check_output("timedatectl status") diff --git a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 index 2f0f90c8f..4ac75a54c 100644 --- a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 @@ -19,6 +19,9 @@ kolla_ansible_requirements_yml: "/tmp/previous-kolla-ansible-requirements.yml" {% endif %} kolla_openstack_logging_debug: true +# Force system clock synchronisation +ntp_force_sync: True + # Use the CI infra's PyPI mirror. pip_local_mirror: true pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" diff --git a/playbooks/kayobe-seed-base/overrides.yml.j2 b/playbooks/kayobe-seed-base/overrides.yml.j2 index 9abbd3355..da643ece9 100644 --- a/playbooks/kayobe-seed-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-base/overrides.yml.j2 @@ -17,6 +17,9 @@ kolla_ansible_requirements_yml: "/tmp/kolla-ansible-requirements.yml" kolla_openstack_logging_debug: True pip_upper_constraints_file: "/tmp/upper-constraints.txt" +# Force system clock synchronisation +ntp_force_sync: True + # Use the CI infra's PyPI mirror. pip_local_mirror: true pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" diff --git a/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 index 185d42420..683833025 100644 --- a/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 @@ -19,6 +19,9 @@ kolla_ansible_requirements_yml: "/tmp/previous-kolla-ansible-requirements.yml" {% endif %} kolla_openstack_logging_debug: True +# Force system clock synchronisation +ntp_force_sync: True + # Use the CI infra's PyPI mirror. pip_local_mirror: true pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" diff --git a/playbooks/kayobe-seed-vm-base/overrides.yml.j2 b/playbooks/kayobe-seed-vm-base/overrides.yml.j2 index c7284ce5e..2413fcdf6 100644 --- a/playbooks/kayobe-seed-vm-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-vm-base/overrides.yml.j2 @@ -16,6 +16,9 @@ pip_upper_constraints_file: "/tmp/upper-constraints.txt" # QEMU. libvirt_vm_engine: "qemu" +# Force system clock synchronisation +ntp_force_sync: True + # Use the CI infra's PyPI mirror. pip_local_mirror: true pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"