diff --git a/swift/cli/recon.py b/swift/cli/recon.py index d370cc65e0..4eb49f0c34 100644 --- a/swift/cli/recon.py +++ b/swift/cli/recon.py @@ -867,14 +867,16 @@ class SwiftRecon(object): host = urlparse(url).netloc.split(':')[0] print('%.02f%% %s' % (used, '%-15s %s' % (host, device))) - def time_check(self, hosts): + def time_check(self, hosts, jitter=0.0): """ Check a time synchronization of hosts with current time :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) + :param jitter: Maximal allowed time jitter """ + jitter = abs(jitter) matches = 0 errors = 0 recon = Scout("time", self.verbose, self.suppress_errors, @@ -885,13 +887,13 @@ class SwiftRecon(object): if status != 200: errors = errors + 1 continue - if (ts_remote < ts_start or ts_remote > ts_end): + if (ts_remote + jitter < ts_start or ts_remote - jitter > ts_end): diff = abs(ts_end - ts_remote) ts_end_f = self._ptime(ts_end) ts_remote_f = self._ptime(ts_remote) print("!! %s current time is %s, but remote is %s, " - "differs by %.2f sec" % ( + "differs by %.4f sec" % ( url, ts_end_f, ts_remote_f, @@ -978,6 +980,8 @@ class SwiftRecon(object): help="Get drive audit error stats") args.add_option('--time', '-T', action="store_true", help="Check time synchronization") + args.add_option('--jitter', type="float", default=0.0, + help="Maximal allowed time jitter") args.add_option('--top', type='int', metavar='COUNT', default=0, help='Also show the top COUNT entries in rank order.') args.add_option('--lowest', type='int', metavar='COUNT', default=0, @@ -1058,7 +1062,7 @@ class SwiftRecon(object): self.socket_usage(hosts) self.server_type_check(hosts) self.driveaudit_check(hosts) - self.time_check(hosts) + self.time_check(hosts, options.jitter) else: if options.async: if self.server_type == 'object': @@ -1107,7 +1111,7 @@ class SwiftRecon(object): if options.driveaudit: self.driveaudit_check(hosts) if options.time: - self.time_check(hosts) + self.time_check(hosts, options.jitter) def main(): diff --git a/test/unit/cli/test_recon.py b/test/unit/cli/test_recon.py index f1e3c13664..3cb3d4f8e9 100644 --- a/test/unit/cli/test_recon.py +++ b/test/unit/cli/test_recon.py @@ -1000,12 +1000,12 @@ class TestReconCommands(unittest.TestCase): def dummy_request(*args, **kwargs): return [ - ('http://127.0.0.1:6010/recon/load', + ('http://127.0.0.1:6010/recon/time', now, 200, now - 0.5, now + 0.5), - ('http://127.0.0.1:6020/recon/load', + ('http://127.0.0.1:6020/recon/time', now, 200, now, @@ -1050,7 +1050,7 @@ class TestReconCommands(unittest.TestCase): default_calls = [ mock.call("!! http://127.0.0.1:6010/recon/time current time is " "2015-04-25 22:13:21, but remote is " - "2015-04-25 22:13:20, differs by 1.30 sec"), + "2015-04-25 22:13:20, differs by 1.3000 sec"), mock.call('1/2 hosts matched, 0 error[s] while checking hosts.'), ] @@ -1060,6 +1060,77 @@ class TestReconCommands(unittest.TestCase): # that is returned from the recon middleware, thus can't rely on it mock_print.assert_has_calls(default_calls, any_order=True) + @mock.patch('six.moves.builtins.print') + @mock.patch('time.time') + def test_time_check_jitter(self, mock_now, mock_print): + now = 1430000000.0 + mock_now.return_value = now + + def dummy_request(*args, **kwargs): + return [ + ('http://127.0.0.1:6010/recon/time', + now - 2, + 200, + now, + now + 3), + ('http://127.0.0.1:6020/recon/time', + now + 2, + 200, + now - 3, + now), + ] + + cli = recon.SwiftRecon() + cli.pool.imap = dummy_request + + default_calls = [ + mock.call('2/2 hosts matched, 0 error[s] while checking hosts.') + ] + + cli.time_check([('127.0.0.1', 6010), ('127.0.0.1', 6020)], 3) + # We need any_order=True because the order of calls depends on the dict + # that is returned from the recon middleware, thus can't rely on it + mock_print.assert_has_calls(default_calls, any_order=True) + + @mock.patch('six.moves.builtins.print') + @mock.patch('time.time') + def test_time_check_jitter_mismatch(self, mock_now, mock_print): + now = 1430000000.0 + mock_now.return_value = now + + def dummy_request(*args, **kwargs): + return [ + ('http://127.0.0.1:6010/recon/time', + now - 4, + 200, + now, + now + 2), + ('http://127.0.0.1:6020/recon/time', + now + 4, + 200, + now - 2, + now), + ] + + cli = recon.SwiftRecon() + cli.pool.imap = dummy_request + + default_calls = [ + mock.call("!! http://127.0.0.1:6010/recon/time current time is " + "2015-04-25 22:13:22, but remote is " + "2015-04-25 22:13:16, differs by 6.0000 sec"), + mock.call("!! http://127.0.0.1:6020/recon/time current time is " + "2015-04-25 22:13:20, but remote is " + "2015-04-25 22:13:24, differs by 4.0000 sec"), + mock.call('0/2 hosts matched, 0 error[s] while checking hosts.'), + ] + + cli.time_check([('127.0.0.1', 6010), ('127.0.0.1', 6020)], 3) + + # We need any_order=True because the order of calls depends on the dict + # that is returned from the recon middleware, thus can't rely on it + mock_print.assert_has_calls(default_calls, any_order=True) + @mock.patch('six.moves.builtins.print') @mock.patch('swift.cli.recon.SwiftRecon.get_hosts') def test_multiple_server_types(self, mock_get_hosts, mock_print):