From 67e4f9d8110014d7318b5ddbf64bc68587a3dbd9 Mon Sep 17 00:00:00 2001 From: Anton Studenov Date: Fri, 27 May 2016 13:06:30 +0300 Subject: [PATCH] Support for chrony NTP - added NtpChronyd - BaseNtp and its sub-classes were refactored - simplified AbstractNtp API - updated unit tests Closes-Bug: #1583190 Change-Id: I10fcd0660a0fa20695b7e99f748e743f681bc21d --- devops/helpers/ntp.py | 386 ++++++++++++------------------- devops/tests/helpers/test_ntp.py | 371 +++++++++++++++++++---------- 2 files changed, 394 insertions(+), 363 deletions(-) diff --git a/devops/helpers/ntp.py b/devops/helpers/ntp.py index 3caf7b95..6e8ff79c 100644 --- a/devops/helpers/ntp.py +++ b/devops/helpers/ntp.py @@ -13,12 +13,10 @@ # under the License. import abc -import time from six import add_metaclass -from devops.error import TimeoutError -from devops.helpers.helpers import get_admin_ip +from devops.error import DevopsError from devops.helpers.helpers import get_admin_remote from devops.helpers.helpers import get_node_remote from devops.helpers.helpers import wait @@ -48,13 +46,36 @@ def sync_time(env, node_names, skip_sync=False): g_ntp.do_sync_time(g_ntp.other_ntps) all_ntps = g_ntp.admin_ntps + g_ntp.pacemaker_ntps + g_ntp.other_ntps - results = {ntp.node_name: ntp.date[0].rstrip() for ntp in all_ntps} + results = {ntp.node_name: ntp.date for ntp in all_ntps} return results @add_metaclass(abc.ABCMeta) class AbstractNtp(object): + + def __init__(self, remote, node_name): + self._remote = remote + self._node_name = node_name + + def __repr__(self): + return "{0}(remote={1}, node_name={2!r})".format( + self.__class__.__name__, self.remote, self.node_name) + + @property + def remote(self): + """remote object""" + return self._remote + + @property + def node_name(self): + """node name""" + return self._node_name + + @property + def date(self): + return self.remote.execute("date")['stdout'][0].rstrip() + @abc.abstractmethod def start(self): """Start ntp daemon""" @@ -63,10 +84,6 @@ class AbstractNtp(object): def stop(self): """Stop ntp daemon""" - @abc.abstractmethod - def get_peers(self): - """Get connected clients""" - @abc.abstractmethod def set_actual_time(self, timeout=600): """enforce time sync""" @@ -75,254 +92,184 @@ class AbstractNtp(object): def wait_peer(self, interval=8, timeout=600): """Wait for connection""" - @abc.abstractproperty - def date(self): - """get date command output""" - - @abc.abstractproperty - def remote(self): - """remote object""" - - @abc.abstractproperty - def admin_ip(self): - """admin node ip""" - - is_connected = abc.abstractproperty( - fget=lambda: None, fset=lambda status: None, doc="connectivity status") - - is_synchronized = abc.abstractproperty( - fget=lambda: None, fset=lambda status: None, doc="sync status") - - @abc.abstractproperty - def node_name(self): - """node name""" - - @abc.abstractproperty - def peers(self): - """peers""" - - @abc.abstractproperty - def is_pacemaker(self): - """how NTPD is managed - by init script or by pacemaker""" - - @abc.abstractproperty - def server(self): - """IP of a server from which the time will be synchronized.""" - # pylint: disable=abstract-method # noinspection PyAbstractClass class BaseNtp(AbstractNtp): - def __init__(self, remote, node_name='node', admin_ip=None): - self.__remote = remote - self.__node_name = node_name - self.__admin_ip = admin_ip - self.__is_synchronized = False - self.__is_connected = False - # Get IP of a server from which the time will be synchronized. - cmd = "awk '/^server/ && $2 !~ /127.*/ {print $2}' /etc/ntp.conf" - self.__server = remote.execute(cmd)['stdout'][0] + """Base class for ntpd based services - @property - def server(self): - return self.__server - - @property - def remote(self): - return self.__remote - - @property - def is_connected(self): - return self.__is_connected - - @is_connected.setter - def is_connected(self, status): - self.__is_connected = status - - @property - def is_synchronized(self): - return self.__is_synchronized - - @is_synchronized.setter - def is_synchronized(self, status): - self.__is_synchronized = status - - @property - def node_name(self): - return self.__node_name - - @property - def admin_ip(self): - return self.__admin_ip - - @property - def peers(self): - return self.get_peers()[2:] - - @property - def date(self): - return self.remote.execute("date")['stdout'] + Provides common methods: + - set_actual_time + - wait_peer + """ def set_actual_time(self, timeout=600): - # Waiting for parent server until it starts providing the time - cmd = "ntpdate -p 4 -t 0.2 -bu {0}".format(self.server) - self.is_synchronized = False - try: - wait(lambda: not self.remote.execute(cmd)['exit_code'], timeout) - self.remote.execute('hwclock -w') - self.is_synchronized = True - except TimeoutError as e: - logger.debug('Time sync failed with {}'.format(e)) + # Get IP of a server from which the time will be synchronized. + srv_cmd = "awk '/^server/ && $2 !~ /127.*/ {print $2}' /etc/ntp.conf" + server = self.remote.execute(srv_cmd)['stdout'][0] - return self.is_synchronized + # Waiting for parent server until it starts providing the time + set_date_cmd = "ntpdate -p 4 -t 0.2 -bu {0}".format(server) + wait(lambda: not self.remote.execute(set_date_cmd)['exit_code'], + timeout=timeout, + timeout_msg='Failed to set actual time on node {!r}'.format( + self._node_name)) + + self.remote.check_call('hwclock -w') + + def _get_ntpq(self): + return self.remote.execute('ntpq -pn 127.0.0.1')['stdout'][2:] + + def _get_sync_complete(self): + peers = self._get_ntpq() + + logger.debug("Node: {0}, ntpd peers: {1}".format( + self.node_name, peers)) + + for peer in peers: + p = peer.split() + remote = str(p[0]) + reach = int(p[6], 8) # From octal to int + offset = float(p[8]) + jitter = float(p[9]) + + # 1. offset and jitter should not be higher than 500 + # Otherwise, time should be re-set. + if (abs(offset) > 500) or (abs(jitter) > 500): + continue + + # 2. remote should be marked with tally '*' + if remote[0] != '*': + continue + + # 3. reachability bit array should have '1' at least in + # two lower bits as the last two successful checks + if reach & 3 != 3: + continue + + return True + return False def wait_peer(self, interval=8, timeout=600): - self.is_connected = False + wait(self._get_sync_complete, + interval=interval, + timeout=timeout, + timeout_msg='Failed to wait peer on node {!r}'.format( + self._node_name)) - start_time = time.time() - while start_time + timeout > time.time(): - # peer = `ntpq -pn 127.0.0.1` - logger.debug( - "Node: {0}, ntpd peers: {1}".format(self.node_name, self.peers) - ) - - for peer in self.peers: - p = peer.split() - remote = str(p[0]) - reach = int(p[6], 8) # From octal to int - offset = float(p[8]) - jitter = float(p[9]) - - # 1. offset and jitter should not be higher than 500 - # Otherwise, time should be re-set. - if (abs(offset) > 500) or (abs(jitter) > 500): - return self.is_connected - - # 2. remote should be marked with tally '*' - if remote[0] != '*': - continue - - # 3. reachability bit array should have '1' at least in - # two lower bits as the last two successful checks - if reach & 3 == 3: - self.is_connected = True - return self.is_connected - - time.sleep(interval) - return self.is_connected # pylint: enable=abstract-method class NtpInitscript(BaseNtp): """NtpInitscript.""" # TODO(ddmitriev) documentation - def __init__(self, remote, node_name='node', admin_ip=None): - super(NtpInitscript, self).__init__( - remote, node_name, admin_ip) - cmd = "find /etc/init.d/ -regex '/etc/init.d/ntp.?'" - self.__service = remote.execute(cmd)['stdout'][0].strip() + def __init__(self, remote, node_name): + super(NtpInitscript, self).__init__(remote, node_name) + get_ntp_cmd = \ + "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable" + result = remote.execute(get_ntp_cmd) + self._service = result['stdout'][0].strip() def start(self): - self.is_connected = False - self.remote.execute("{0} start".format(self.__service)) + self.remote.check_call("{0} start".format(self._service)) def stop(self): - self.is_connected = False - self.remote.execute("{0} stop".format(self.__service)) - - def get_peers(self): - return self.remote.execute('ntpq -pn 127.0.0.1')['stdout'] - - @property - def is_pacemaker(self): - return False - - def __repr__(self): - return "{0}(remote={1}, node_name={2}, admin_ip={3})".format( - self.__class__.__name__, self.remote, self.node_name, self.admin_ip - ) + self.remote.check_call("{0} stop".format(self._service)) class NtpPacemaker(BaseNtp): """NtpPacemaker.""" # TODO(ddmitriev) documentation def start(self): - self.is_connected = False - # Temporary workaround of the LP bug #1441121 self.remote.execute('ip netns exec vrouter ip l set dev lo up') self.remote.execute('crm resource start p_ntp') def stop(self): - self.is_connected = False self.remote.execute('crm resource stop p_ntp; killall ntpd') - def get_peers(self): + def _get_ntpq(self): return self.remote.execute( - 'ip netns exec vrouter ntpq -pn 127.0.0.1')['stdout'] - - @property - def is_pacemaker(self): - return True - - def __repr__(self): - return "{0}(remote={1}, node_name={2}, admin_ip={3})".format( - self.__class__.__name__, self.remote, self.node_name, self.admin_ip - ) + 'ip netns exec vrouter ntpq -pn 127.0.0.1')['stdout'][2:] class NtpSystemd(BaseNtp): """NtpSystemd.""" # TODO(ddmitriev) documentation def start(self): - self.is_connected = False - self.remote.execute('systemctl start ntpd') + self.remote.check_call('systemctl start ntpd') def stop(self): - self.is_connected = False - self.remote.execute('systemctl stop ntpd') + self.remote.check_call('systemctl stop ntpd') - def get_peers(self): - return self.remote.execute('ntpq -pn 127.0.0.1')['stdout'] - @property - def is_pacemaker(self): - return False +class NtpChronyd(AbstractNtp): + """Implements communication with chrony service - def __repr__(self): - return "{0}(remote={1}, node_name={2}, admin_ip={3})".format( - self.__class__.__name__, self.remote, self.node_name, self.admin_ip - ) + Reference: http://chrony.tuxfamily.org/ + """ + + def start(self): + # No need to stop/start chronyd + # client can't work without daemon + pass + + def stop(self): + # No need to stop/start chronyd + # client can't work without daemon + pass + + def _get_burst_complete(self): + result = self._remote.check_call('chronyc -a activity') + stdout = result['stdout'] + burst_line = stdout[4] + return burst_line == '0 sources doing burst (return to online)\n' + + def set_actual_time(self, timeout=600): + # sync time + # 3 - good measurements + # 5 - max measurements + self._remote.check_call('chronyc -a burst 3/5') + + # wait burst complete + wait(self._get_burst_complete, timeout=timeout, + timeout_msg='Failed to set actual time on node {!r}'.format( + self._node_name)) + + # set system clock + self._remote.check_call('chronyc -a makestep') + + def wait_peer(self, interval=8, timeout=600): + # wait for synchronization + # 10 - number of tries + # 0.01 - maximum allowed remaining correction + self._remote.check_call('chronyc -a waitsync 10 0.01') class GroupNtpSync(object): """Synchronize a group of nodes.""" + @staticmethod - def get_ntp(remote, node_name='node', admin_ip=None): + def get_ntp(remote, node_name): # Detect how NTPD is managed - by init script or by pacemaker. pcs_cmd = "ps -C pacemakerd && crm_resource --resource p_ntp --locate" systemd_cmd = "systemctl list-unit-files| grep ntpd" + chronyd_cmd = "systemctl is-active chronyd" + initd_cmd = "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable" - # pylint: disable=redefined-variable-type if remote.execute(pcs_cmd)['exit_code'] == 0: # Pacemaker service found - ntp = NtpPacemaker(remote, node_name, admin_ip) + return NtpPacemaker(remote, node_name) elif remote.execute(systemd_cmd)['exit_code'] == 0: - ntp = NtpSystemd(remote, node_name, admin_ip) + return NtpSystemd(remote, node_name) + elif remote.execute(chronyd_cmd)['exit_code'] == 0: + return NtpChronyd(remote, node_name) + elif len(remote.execute(initd_cmd)['stdout']): + return NtpInitscript(remote, node_name) else: - # Pacemaker not found, using native ntpd - ntp = NtpInitscript(remote, node_name, admin_ip) - # pylint: enable=redefined-variable-type - - # Speedup time synchronization for slaves that use admin node as a peer - if admin_ip: - cmd = ( - "sed -i 's/^server {0} .*/server {0} minpoll 3 maxpoll 5 " - "iburst/' /etc/ntp.conf".format(admin_ip)) - remote.execute(cmd) - - return ntp + raise DevopsError('No suitable NTP service found on node {!r}' + ''.format(node_name)) def __init__(self, env, node_names): """Context manager for synchronize time on nodes @@ -330,26 +277,22 @@ class GroupNtpSync(object): param: env - environment object param: node_names - list of devops node names """ - if not env: - raise Exception("'env' is not set, failed to initialize" - " connections to {0}".format(node_names)) self.admin_ntps = [] self.pacemaker_ntps = [] self.other_ntps = [] - admin_ip = get_admin_ip(env) - for node_name in node_names: if node_name == 'admin': # 1. Add a 'Ntp' instance with connection to Fuel admin node - self.admin_ntps.append( - self.get_ntp(get_admin_remote(env), 'admin')) + admin_remote = get_admin_remote(env) + admin_ntp = self.get_ntp(admin_remote, 'admin') + self.admin_ntps.append(admin_ntp) logger.debug("Added node '{0}' to self.admin_ntps" .format(node_name)) continue - ntp = self.get_ntp( - get_node_remote(env, node_name), node_name, admin_ip) - if ntp.is_pacemaker: + remote = get_node_remote(env, node_name) + ntp = self.get_ntp(remote, node_name) + if isinstance(ntp, NtpPacemaker): # 2. Create a list of 'Ntp' connections to the controller nodes self.pacemaker_ntps.append(ntp) logger.debug("Added node '{0}' to self.pacemaker_ntps" @@ -371,34 +314,11 @@ class GroupNtpSync(object): for ntp in self.other_ntps: ntp.remote.clear() - @staticmethod - def is_synchronized(ntps): - return all([ntp.is_synchronized for ntp in ntps]) - - @staticmethod - def is_connected(ntps): - return all([ntp.is_connected for ntp in ntps]) - - @staticmethod - def report_not_synchronized(ntps): - return [(ntp.node_name, ntp.date) - for ntp in ntps if not ntp.is_synchronized] - - @staticmethod - def report_not_connected(ntps): - return [(ntp.node_name, ntp.peers) - for ntp in ntps if not ntp.is_connected] - @staticmethod def report_node_names(ntps): return [ntp.node_name for ntp in ntps] def do_sync_time(self, ntps): - # 0. 'ntps' can be filled by __init__() or outside the class - if not ntps: - raise ValueError("No servers were provided to synchronize " - "the time in self.ntps") - # 1. Stop NTPD service on nodes logger.debug("Stop NTPD service on nodes {0}" .format(self.report_node_names(ntps))) @@ -411,10 +331,6 @@ class GroupNtpSync(object): for ntp in ntps: ntp.set_actual_time() - if not self.is_synchronized(ntps): - raise TimeoutError("Time on nodes was not set with 'ntpdate':\n{0}" - .format(self.report_not_synchronized(ntps))) - # 3. Start NTPD service on nodes logger.debug("Start NTPD service on nodes {0}" .format(self.report_node_names(ntps))) @@ -427,7 +343,3 @@ class GroupNtpSync(object): for ntp in ntps: ntp.wait_peer() - - if not self.is_connected(ntps): - raise TimeoutError("NTPD on nodes was not synchronized:\n" - "{0}".format(self.report_not_connected(ntps))) diff --git a/devops/tests/helpers/test_ntp.py b/devops/tests/helpers/test_ntp.py index 7715d253..545bebb3 100644 --- a/devops/tests/helpers/test_ntp.py +++ b/devops/tests/helpers/test_ntp.py @@ -20,166 +20,285 @@ import unittest import mock -from devops import error from devops.helpers import ntp +from devops.helpers import ssh_client -return_value = {'stdout': [' 0 2 4 ', '1', '2', '3']} +class NtpTestCase(unittest.TestCase): + + def patch(self, *args, **kwargs): + patcher = mock.patch(*args, **kwargs) + m = patcher.start() + self.addCleanup(patcher.stop) + return m + + def setUp(self): + self.remote_mock = mock.Mock(spec=ssh_client.SSHClient) + self.remote_mock.__repr__ = mock.Mock(return_value='') + + self.wait_mock = self.patch('devops.helpers.ntp.wait') + + @staticmethod + def make_exec_result(stdout, exit_code=0): + return { + 'exit_code': exit_code, + 'stderr': [], + 'stdout': stdout.splitlines(True), + } -class Remote(object): - def __init__(self): - self.execute = mock.Mock(return_value=return_value) +class TestNtpInitscript(NtpTestCase): - def reset_mock(self): - self.execute.reset_mock() - self.execute.return_value = return_value + def setUp(self): + super(TestNtpInitscript, self).setUp() - def __repr__(self): - return self.__class__.__name__ + self.remote_mock.execute.return_value = self.make_exec_result( + '/etc/init.d/ntp') + def test_init(self): + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') + assert ntp_init.remote is self.remote_mock + assert ntp_init.node_name == 'node' + assert repr(ntp_init) == \ + "NtpInitscript(remote=, node_name='node')" + self.remote_mock.execute.assert_called_once_with( + "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable") -class TestNtp(unittest.TestCase): - @mock.patch('time.time', return_value=1, autospec=True) - @mock.patch('devops.helpers.ntp.wait') - @mock.patch('devops.helpers.ntp.logger', autospec=True) - def test_ntp_common(self, logger, wait, time): - remote = Remote() - ntp_init = ntp.NtpInitscript(remote) - - remote.reset_mock() - - result = ntp_init.set_actual_time() - self.assertTrue(result) - self.assertTrue(ntp_init.is_synchronized) - - wait.assert_called_once() - remote.execute.assert_called_once_with("hwclock -w") - - wait.reset_mock() - logger.reset_mock() - debug = mock.Mock() - logger.attach_mock(debug, 'debug') - - wait.side_effect = error.TimeoutError('E') - result = ntp_init.set_actual_time() - self.assertFalse(result) - self.assertFalse(ntp_init.is_synchronized) - debug.assert_called_once_with('Time sync failed with E') - - result = ntp_init.wait_peer(timeout=-1) - self.assertFalse(result) - self.assertFalse(ntp_init.is_connected) - time.assert_has_calls((mock.call(), mock.call())) - - def check_shared(self, ntp_obj, remote, pacemaker): - self.assertEqual(ntp_obj.remote, remote) - self.assertEqual(ntp_obj.node_name, 'node') - self.assertIsNone(ntp_obj.admin_ip) - self.assertEqual(ntp_obj.is_pacemaker, pacemaker) - self.assertFalse(ntp_obj.is_synchronized) - self.assertFalse(ntp_obj.is_connected) - self.assertEqual(ntp_obj.server, ' 0 2 4 ') - - def test_ntp_init(self): - remote = Remote() - ntp_init = ntp.NtpInitscript(remote) - self.check_shared(ntp_obj=ntp_init, remote=remote, pacemaker=False) - - remote.execute.assert_has_calls(( - mock.call( - "awk '/^server/ && $2 !~ /127.*/ {print $2}' /etc/ntp.conf"), - mock.call("find /etc/init.d/ -regex '/etc/init.d/ntp.?'") - )) - self.assertEqual( - str(ntp_init), - 'NtpInitscript(remote=Remote, node_name=node, admin_ip=None)') - - remote.reset_mock() - - peers = ntp_init.peers - self.assertEqual(peers, ['2', '3']) - remote.execute.assert_called_once_with('ntpq -pn 127.0.0.1') - - remote.reset_mock() - - date = ntp_init.date - self.assertEqual(date, return_value['stdout']) - remote.execute.assert_called_once_with('date') - - remote.reset_mock() + def test_start(self): + self.remote_mock.check_call.return_value = self.make_exec_result('') + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') ntp_init.start() - self.assertFalse(ntp_init.is_connected) - remote.execute.assert_called_once_with('0 2 4 start') - remote.reset_mock() + self.remote_mock.check_call.assert_called_once_with( + '/etc/init.d/ntp start') + def test_stop(self): + self.remote_mock.check_call.return_value = self.make_exec_result('') + + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') ntp_init.stop() - self.assertFalse(ntp_init.is_connected) - remote.execute.assert_called_once_with('0 2 4 stop') - def test_ntp_pacemaker(self): - remote = Remote() - ntp_pcm = ntp.NtpPacemaker(remote) + self.remote_mock.check_call.assert_called_once_with( + '/etc/init.d/ntp stop') - self.check_shared(ntp_obj=ntp_pcm, remote=remote, pacemaker=True) + def test_get_ntpq(self): + self.remote_mock.execute.side_effect = ( + self.make_exec_result('/etc/init.d/ntp'), + self.make_exec_result('Line1\nLine2\nLine3\nLine4\n'), + ) - remote.execute.assert_called_once_with( - "awk '/^server/ && $2 !~ /127.*/ {print $2}' /etc/ntp.conf") - self.assertEqual( - str(ntp_pcm), - 'NtpPacemaker(remote=Remote, node_name=node, admin_ip=None)') + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') + peers = ntp_init._get_ntpq() - remote.reset_mock() + self.remote_mock.execute.assert_has_calls(( + mock.call( + "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable"), + mock.call('ntpq -pn 127.0.0.1'), + )) + assert peers == ['Line3\n', 'Line4\n'] - ntp_pcm.start() - self.assertFalse(ntp_pcm.is_connected) - remote.execute.assert_has_calls(( - mock.call('ip netns exec vrouter ip l set dev lo up'), - mock.call('crm resource start p_ntp') + def test_date(self): + self.remote_mock.execute.side_effect = ( + self.make_exec_result('/etc/init.d/ntp'), + self.make_exec_result('Thu May 26 13:35:43 MSK 2016'), + ) + + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') + date = ntp_init.date + + self.remote_mock.execute.assert_has_calls(( + mock.call( + "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable"), + mock.call('date'), + )) + assert date == 'Thu May 26 13:35:43 MSK 2016' + + def test_set_actual_time(self): + self.remote_mock.execute.side_effect = ( + self.make_exec_result('/etc/init.d/ntp'), + self.make_exec_result('server1.com'), + self.make_exec_result(''), + ) + + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') + ntp_init.set_actual_time() + + self.wait_mock.assert_called_once_with( + mock.ANY, timeout=600, + timeout_msg="Failed to set actual time on node 'node'") + + waiter = self.wait_mock.call_args[0][0] + assert waiter() is True + self.remote_mock.execute.assert_has_calls(( + mock.call( + "find /etc/init.d/ -regex '/etc/init.d/ntp.?' -executable"), + mock.call("awk '/^server/ && $2 !~ /127.*/ {print $2}' " + "/etc/ntp.conf"), + mock.call('ntpdate -p 4 -t 0.2 -bu server1.com'), )) - remote.reset_mock() + self.remote_mock.check_call.assert_called_once_with('hwclock -w') + def test_get_sync_complete(self): + self.remote_mock.execute.side_effect = ( + self.make_exec_result('/etc/init.d/ntp'), + self.make_exec_result("""\ + remote refid st t when poll reach delay offset jitter +============================================================================== +-95.213.132.250 195.210.189.106 2 u 8 64 377 40.263 -1.379 15.326 +*87.229.205.75 212.51.144.44 2 u 16 64 377 31.288 -1.919 9.969 ++31.131.249.26 46.46.152.214 2 u 34 64 377 40.522 -0.988 7.747 +-217.65.8.75 195.3.254.2 3 u 26 64 377 28.758 -4.249 44.240 ++91.189.94.4 138.96.64.10 2 u 24 64 377 83.284 -1.810 14.550 +""")) + + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') + assert ntp_init._get_sync_complete() is True + + def test_get_sync_complete_false(self): + self.remote_mock.execute.side_effect = ( + self.make_exec_result('/etc/init.d/ntp'), + self.make_exec_result("""\ + remote refid st t when poll reach delay offset jitter +============================================================================== ++95.213.132.250 195.210.189.106 2 u 8 64 377 40.263 -1.379 532.46 +-87.229.205.75 212.51.144.44 2 u 16 64 377 31.288 -1.919 9.969 +*31.131.249.26 46.46.152.214 2 u 34 64 1 40.522 -0.988 7.747 +-217.65.8.75 195.3.254.2 3 u 26 64 377 28.758 -4.249 44.240 ++91.189.94.4 138.96.64.10 2 u 24 64 377 83.284 -1.810 14.550 +""")) + + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') + assert ntp_init._get_sync_complete() is False + + def test_wait_peer(self): + ntp_init = ntp.NtpInitscript(self.remote_mock, 'node') + ntp_init.wait_peer() + + self.wait_mock.assert_called_once_with( + ntp_init._get_sync_complete, interval=8, timeout=600, + timeout_msg="Failed to wait peer on node 'node'") + + +class TestNtpPacemaker(NtpTestCase): + + def test_init(self): + ntp_pcm = ntp.NtpPacemaker(self.remote_mock, 'node') + assert ntp_pcm.remote is self.remote_mock + assert ntp_pcm.node_name == 'node' + assert repr(ntp_pcm) == \ + "NtpPacemaker(remote=, node_name='node')" + + def test_start(self): + ntp_pcm = ntp.NtpPacemaker(self.remote_mock, 'node') + ntp_pcm.start() + + self.remote_mock.execute.assert_has_calls(( + mock.call('ip netns exec vrouter ip l set dev lo up'), + mock.call('crm resource start p_ntp'), + )) + + def test_stop(self): + ntp_pcm = ntp.NtpPacemaker(self.remote_mock, 'node') ntp_pcm.stop() - self.assertFalse(ntp_pcm.is_connected) - remote.execute.assert_called_once_with( + + self.remote_mock.execute.assert_called_once_with( 'crm resource stop p_ntp; killall ntpd') - remote.reset_mock() + def test_get_ntpq(self): + self.remote_mock.execute.return_value = self.make_exec_result( + 'Line1\nLine2\nLine3\nLine4\n') - result = ntp_pcm.get_peers() - self.assertEqual(result, return_value['stdout']) - remote.execute.assert_called_once_with( + ntp_pcm = ntp.NtpPacemaker(self.remote_mock, 'node') + peers = ntp_pcm._get_ntpq() + + self.remote_mock.execute.assert_called_once_with( 'ip netns exec vrouter ntpq -pn 127.0.0.1') + assert peers == ['Line3\n', 'Line4\n'] - def test_ntp_systemd(self): - remote = Remote() - ntp_sysd = ntp.NtpSystemd(remote) - self.check_shared(ntp_obj=ntp_sysd, remote=remote, pacemaker=False) +class TestNtpSystemd(NtpTestCase): - remote.execute.assert_called_once_with( - "awk '/^server/ && $2 !~ /127.*/ {print $2}' /etc/ntp.conf") - self.assertEqual( - str(ntp_sysd), - 'NtpSystemd(remote=Remote, node_name=node, admin_ip=None)') - - remote.reset_mock() + def test_init(self): + ntp_sysd = ntp.NtpSystemd(self.remote_mock, 'node') + assert ntp_sysd.remote is self.remote_mock + assert ntp_sysd.node_name == 'node' + assert repr(ntp_sysd) == \ + "NtpSystemd(remote=, node_name='node')" + def test_start(self): + ntp_sysd = ntp.NtpSystemd(self.remote_mock, 'node') ntp_sysd.start() - self.assertFalse(ntp_sysd.is_connected) - remote.execute.assert_called_once_with('systemctl start ntpd') - remote.reset_mock() + self.remote_mock.check_call.assert_called_once_with( + 'systemctl start ntpd') + def test_stop(self): + ntp_sysd = ntp.NtpSystemd(self.remote_mock, 'node') ntp_sysd.stop() - self.assertFalse(ntp_sysd.is_connected) - remote.execute.assert_called_once_with('systemctl stop ntpd') - remote.reset_mock() + self.remote_mock.check_call.assert_called_once_with( + 'systemctl stop ntpd') - result = ntp_sysd.get_peers() - self.assertEqual(result, return_value['stdout']) - remote.execute.assert_called_once_with('ntpq -pn 127.0.0.1') + +class TestNtpChronyd(NtpTestCase): + + def test_init(self): + ntp_chrony = ntp.NtpChronyd(self.remote_mock, 'node') + assert ntp_chrony.remote is self.remote_mock + assert ntp_chrony.node_name == 'node' + assert repr(ntp_chrony) == \ + "NtpChronyd(remote=, node_name='node')" + + ntp_chrony.start() + ntp_chrony.stop() + + def test_get_burst_complete(self): + self.remote_mock.check_call.return_value = \ + self.make_exec_result("""200 OK +200 OK +4 sources online +0 sources offline +0 sources doing burst (return to online) +0 sources doing burst (return to offline) +0 sources with unknown address""") + + ntp_chrony = ntp.NtpChronyd(self.remote_mock, 'node') + r = ntp_chrony._get_burst_complete() + self.remote_mock.check_call.assert_called_once_with( + 'chronyc -a activity') + assert r is True + + def test_get_burst_complete_false(self): + self.remote_mock.check_call.return_value = \ + self.make_exec_result("""200 OK +200 OK +3 sources online +0 sources offline +1 sources doing burst (return to online) +0 sources doing burst (return to offline) +0 sources with unknown address""") + + ntp_chrony = ntp.NtpChronyd(self.remote_mock, 'node') + r = ntp_chrony._get_burst_complete() + self.remote_mock.check_call.assert_called_once_with( + 'chronyc -a activity') + assert r is False + + def test_set_actual_time(self): + ntp_chrony = ntp.NtpChronyd(self.remote_mock, 'node') + ntp_chrony.set_actual_time() + self.remote_mock.check_call.assert_has_calls(( + mock.call('chronyc -a burst 3/5'), + mock.call('chronyc -a makestep'), + )) + self.wait_mock.assert_called_once_with( + ntp_chrony._get_burst_complete, timeout=600, + timeout_msg="Failed to set actual time on node 'node'") + + def test_wait_peer(self): + ntp_chrony = ntp.NtpChronyd(self.remote_mock, 'node') + ntp_chrony.wait_peer() + self.remote_mock.check_call.assert_called_once_with( + 'chronyc -a waitsync 10 0.01')