Merge "make statsd_client more explicit"

swift/common/statsd_client.py

@@ -15,12 +15,12 @@
 
 """ Statsd Client """
 
-import socket
 import time
 import warnings
 from contextlib import closing
 from random import random
 
+from eventlet.green import socket
 import six
 
 

test/unit/common/test_statsd_client.py

@@ -29,10 +29,10 @@ from mock import patch
 
 
 from swift.common import utils
-from swift.common.statsd_client import StatsdClient
+from swift.common import statsd_client
 
 from test.debug_logger import debug_logger
-from test.unit.common.test_utils import MockUdpSocket, reset_logger_state
+from test.unit.common.test_utils import MockUdpSocket
 from swift.common.swob import Response
 
 
@@ -53,9 +53,9 @@ class TestStatsdLogging(unittest.TestCase):
                      '',
                      ('127.0.0.1', port))]
 
-        self.real_getaddrinfo = utils.socket.getaddrinfo
+        self.real_getaddrinfo = statsd_client.socket.getaddrinfo
         self.getaddrinfo_patcher = mock.patch.object(
-            utils.socket, 'getaddrinfo', fake_getaddrinfo)
+            statsd_client.socket, 'getaddrinfo', fake_getaddrinfo)
         self.mock_getaddrinfo = self.getaddrinfo_patcher.start()
         self.addCleanup(self.getaddrinfo_patcher.stop)
 
@@ -68,7 +68,8 @@ class TestStatsdLogging(unittest.TestCase):
         logger = utils.get_logger({'log_statsd_host': 'some.host.com'},
                                   'some-name', log_route='some-route')
         # white-box construction validation
-        self.assertIsInstance(logger.logger.statsd_client, StatsdClient)
+        self.assertIsInstance(logger.logger.statsd_client,
+                              statsd_client.StatsdClient)
         self.assertEqual(logger.logger.statsd_client._host, 'some.host.com')
         self.assertEqual(logger.logger.statsd_client._port, 8125)
         self.assertEqual(logger.logger.statsd_client._prefix, 'some-name.')
@@ -181,18 +182,18 @@ class TestStatsdLogging(unittest.TestCase):
                 return [(socket.AF_INET6, 'blah', 'blah', 'blah',
                         ('::1', int(port), 0, 0))]
 
-        with mock.patch.object(utils.socket, 'getaddrinfo',
+        with mock.patch.object(statsd_client.socket, 'getaddrinfo',
                                new=stub_getaddrinfo_both_ipv4_and_ipv6):
             logger = utils.get_logger({
                 'log_statsd_host': 'localhost',
                 'log_statsd_port': '9876',
             }, 'some-name', log_route='some-route')
-        statsd_client = logger.logger.statsd_client
+        client = logger.logger.statsd_client
 
-        self.assertEqual(statsd_client._sock_family, socket.AF_INET)
+        self.assertEqual(client._sock_family, socket.AF_INET)
-        self.assertEqual(statsd_client._target, ('localhost', 9876))
+        self.assertEqual(client._target, ('localhost', 9876))
 
-        got_sock = statsd_client._open_socket()
+        got_sock = client._open_socket()
         self.assertEqual(got_sock.family, socket.AF_INET)
 
     def test_ipv4_instantiation_and_socket_creation(self):
@@ -234,34 +235,35 @@ class TestStatsdLogging(unittest.TestCase):
                      '',
                      ('::1', port, 0, 0))]
 
-        with mock.patch.object(utils.logs.socket,
+        with mock.patch.object(statsd_client.socket,
                                'getaddrinfo', fake_getaddrinfo):
             logger = utils.get_logger({
                 'log_statsd_host': '::1',
                 'log_statsd_port': '9876',
             }, 'some-name', log_route='some-route')
-        statsd_client = logger.logger.statsd_client
+        client = logger.logger.statsd_client
         self.assertEqual([socket.AF_INET, socket.AF_INET6], calls)
-        self.assertEqual(statsd_client._sock_family, socket.AF_INET6)
+        self.assertEqual(client._sock_family, socket.AF_INET6)
-        self.assertEqual(statsd_client._target, ('::1', 9876, 0, 0))
+        self.assertEqual(client._target, ('::1', 9876, 0, 0))
 
-        got_sock = statsd_client._open_socket()
+        got_sock = client._open_socket()
         self.assertEqual(got_sock.family, socket.AF_INET6)
 
     def test_bad_hostname_instantiation(self):
-        with mock.patch.object(utils.socket, 'getaddrinfo',
+        stub_err = statsd_client.socket.gaierror('whoops')
-                               side_effect=utils.socket.gaierror("whoops")):
+        with mock.patch.object(statsd_client.socket, 'getaddrinfo',
+                               side_effect=stub_err):
             logger = utils.get_logger({
                 'log_statsd_host': 'i-am-not-a-hostname-or-ip',
                 'log_statsd_port': '9876',
             }, 'some-name', log_route='some-route')
-        statsd_client = logger.logger.statsd_client
+        client = logger.logger.statsd_client
 
-        self.assertEqual(statsd_client._sock_family, socket.AF_INET)
+        self.assertEqual(client._sock_family, socket.AF_INET)
-        self.assertEqual(statsd_client._target,
+        self.assertEqual(client._target,
                          ('i-am-not-a-hostname-or-ip', 9876))
 
-        got_sock = statsd_client._open_socket()
+        got_sock = client._open_socket()
         self.assertEqual(got_sock.family, socket.AF_INET)
         # Maybe the DNS server gets fixed in a bit and it starts working... or
         # maybe the DNS record hadn't propagated yet.  In any case, failed
@@ -282,18 +284,19 @@ class TestStatsdLogging(unittest.TestCase):
                      '',
                      ('::1', port, 0, 0))]
 
-        with mock.patch.object(utils.socket, 'getaddrinfo', fake_getaddrinfo):
+        with mock.patch.object(statsd_client.socket, 'getaddrinfo',
+                               fake_getaddrinfo):
             logger = utils.get_logger({
                 'log_statsd_host': '::1',
                 'log_statsd_port': '9876',
             }, 'some-name', log_route='some-route')
-        statsd_client = logger.logger.statsd_client
+        client = logger.logger.statsd_client
 
         fl = debug_logger()
-        statsd_client.logger = fl
+        client.logger = fl
         mock_socket = MockUdpSocket()
 
-        statsd_client._open_socket = lambda *_: mock_socket
+        client._open_socket = lambda *_: mock_socket
         logger.increment('tunafish')
         self.assertEqual(fl.get_lines_for_level('warning'), [])
         self.assertEqual(mock_socket.sent,
@@ -301,11 +304,11 @@ class TestStatsdLogging(unittest.TestCase):
 
     def test_no_exception_when_cant_send_udp_packet(self):
         logger = utils.get_logger({'log_statsd_host': 'some.host.com'})
-        statsd_client = logger.logger.statsd_client
+        client = logger.logger.statsd_client
         fl = debug_logger()
-        statsd_client.logger = fl
+        client.logger = fl
         mock_socket = MockUdpSocket(sendto_errno=errno.EPERM)
-        statsd_client._open_socket = lambda *_: mock_socket
+        client._open_socket = lambda *_: mock_socket
         logger.increment('tunafish')
         expected = ["Error sending UDP message to ('some.host.com', 8125): "
                     "[Errno 1] test errno 1"]
@@ -316,16 +319,16 @@ class TestStatsdLogging(unittest.TestCase):
 
         mock_socket = MockUdpSocket()
         # encapsulation? what's that?
-        statsd_client = logger.logger.statsd_client
+        client = logger.logger.statsd_client
-        self.assertTrue(statsd_client.random is random.random)
+        self.assertTrue(client.random is random.random)
 
-        statsd_client._open_socket = lambda *_: mock_socket
+        client._open_socket = lambda *_: mock_socket
-        statsd_client.random = lambda: 0.50001
+        client.random = lambda: 0.50001
 
         logger.increment('tribbles', sample_rate=0.5)
         self.assertEqual(len(mock_socket.sent), 0)
 
-        statsd_client.random = lambda: 0.49999
+        client.random = lambda: 0.49999
         logger.increment('tribbles', sample_rate=0.5)
         self.assertEqual(len(mock_socket.sent), 1)
 
@@ -342,16 +345,16 @@ class TestStatsdLogging(unittest.TestCase):
 
         mock_socket = MockUdpSocket()
         # encapsulation? what's that?
-        statsd_client = logger.logger.statsd_client
+        client = logger.logger.statsd_client
-        self.assertTrue(statsd_client.random is random.random)
+        self.assertTrue(client.random is random.random)
 
-        statsd_client._open_socket = lambda *_: mock_socket
+        client._open_socket = lambda *_: mock_socket
-        statsd_client.random = lambda: effective_sample_rate + 0.001
+        client.random = lambda: effective_sample_rate + 0.001
 
         logger.increment('tribbles')
         self.assertEqual(len(mock_socket.sent), 0)
 
-        statsd_client.random = lambda: effective_sample_rate - 0.001
+        client.random = lambda: effective_sample_rate - 0.001
         logger.increment('tribbles')
         self.assertEqual(len(mock_socket.sent), 1)
 
@@ -362,7 +365,7 @@ class TestStatsdLogging(unittest.TestCase):
         self.assertTrue(payload.endswith(suffix), payload)
 
         effective_sample_rate = 0.587 * 0.91
-        statsd_client.random = lambda: effective_sample_rate - 0.001
+        client.random = lambda: effective_sample_rate - 0.001
         logger.increment('tribbles', sample_rate=0.587)
         self.assertEqual(len(mock_socket.sent), 2)
 
@@ -707,86 +710,3 @@ class TestStatsdLoggingDelegation(unittest.TestCase):
         self.assertStat('alpha.beta.another.counter:3|c|@0.9912',
                         self.logger.update_stats, 'another.counter', 3,
                         sample_rate=0.9912)
-
-    @reset_logger_state
-    def test_thread_locals(self):
-        logger = utils.get_logger(None)
-        # test the setter
-        logger.thread_locals = ('id', 'ip')
-        self.assertEqual(logger.thread_locals, ('id', 'ip'))
-        # reset
-        logger.thread_locals = (None, None)
-        self.assertEqual(logger.thread_locals, (None, None))
-        logger.txn_id = '1234'
-        logger.client_ip = '1.2.3.4'
-        self.assertEqual(logger.thread_locals, ('1234', '1.2.3.4'))
-        logger.txn_id = '5678'
-        logger.client_ip = '5.6.7.8'
-        self.assertEqual(logger.thread_locals, ('5678', '5.6.7.8'))
-
-    def test_no_fdatasync(self):
-        called = []
-
-        class NoFdatasync(object):
-            pass
-
-        def fsync(fd):
-            called.append(fd)
-
-        with patch('swift.common.utils.os', NoFdatasync()):
-            with patch('swift.common.utils.fsync', fsync):
-                utils.fdatasync(12345)
-                self.assertEqual(called, [12345])
-
-    def test_yes_fdatasync(self):
-        called = []
-
-        class YesFdatasync(object):
-
-            def fdatasync(self, fd):
-                called.append(fd)
-
-        with patch('swift.common.utils.os', YesFdatasync()):
-            utils.fdatasync(12345)
-            self.assertEqual(called, [12345])
-
-    def test_fsync_bad_fullsync(self):
-
-        class FCNTL(object):
-
-            F_FULLSYNC = 123
-
-            def fcntl(self, fd, op):
-                raise IOError(18)
-
-        with patch('swift.common.utils.fcntl', FCNTL()):
-            self.assertRaises(OSError, lambda: utils.fsync(12345))
-
-    def test_fsync_f_fullsync(self):
-        called = []
-
-        class FCNTL(object):
-
-            F_FULLSYNC = 123
-
-            def fcntl(self, fd, op):
-                called[:] = [fd, op]
-                return 0
-
-        with patch('swift.common.utils.fcntl', FCNTL()):
-            utils.fsync(12345)
-            self.assertEqual(called, [12345, 123])
-
-    def test_fsync_no_fullsync(self):
-        called = []
-
-        class FCNTL(object):
-            pass
-
-        def fsync(fd):
-            called.append(fd)
-
-        with patch('swift.common.utils.fcntl', FCNTL()):
-            with patch('os.fsync', fsync):
-                utils.fsync(12345)
-                self.assertEqual(called, [12345])

test/unit/common/test_utils.py

@@ -5057,6 +5057,22 @@ class TestSwiftLoggerAdapter(unittest.TestCase):
         self.assertEqual(logger.thread_locals, locals2)
         logger.thread_locals = (None, None)
 
+    @reset_logger_state
+    def test_thread_locals_more(self):
+        logger = utils.get_logger(None)
+        # test the setter
+        logger.thread_locals = ('id', 'ip')
+        self.assertEqual(logger.thread_locals, ('id', 'ip'))
+        # reset
+        logger.thread_locals = (None, None)
+        self.assertEqual(logger.thread_locals, (None, None))
+        logger.txn_id = '1234'
+        logger.client_ip = '1.2.3.4'
+        self.assertEqual(logger.thread_locals, ('1234', '1.2.3.4'))
+        logger.txn_id = '5678'
+        logger.client_ip = '5.6.7.8'
+        self.assertEqual(logger.thread_locals, ('5678', '5.6.7.8'))
+
     def test_exception(self):
         # verify that the adapter routes exception calls to utils.LogAdapter
         # for special case handling
@@ -8276,6 +8292,76 @@ class TestShardRangeList(unittest.TestCase):
                          do_test([utils.ShardRange.ACTIVE]))
 
 
+class TestFsync(unittest.TestCase):
+
+    def test_no_fdatasync(self):
+        called = []
+
+        class NoFdatasync(object):
+            pass
+
+        def fsync(fd):
+            called.append(fd)
+
+        with patch('swift.common.utils.os', NoFdatasync()):
+            with patch('swift.common.utils.fsync', fsync):
+                utils.fdatasync(12345)
+                self.assertEqual(called, [12345])
+
+    def test_yes_fdatasync(self):
+        called = []
+
+        class YesFdatasync(object):
+
+            def fdatasync(self, fd):
+                called.append(fd)
+
+        with patch('swift.common.utils.os', YesFdatasync()):
+            utils.fdatasync(12345)
+            self.assertEqual(called, [12345])
+
+    def test_fsync_bad_fullsync(self):
+
+        class FCNTL(object):
+
+            F_FULLSYNC = 123
+
+            def fcntl(self, fd, op):
+                raise IOError(18)
+
+        with patch('swift.common.utils.fcntl', FCNTL()):
+            self.assertRaises(OSError, lambda: utils.fsync(12345))
+
+    def test_fsync_f_fullsync(self):
+        called = []
+
+        class FCNTL(object):
+
+            F_FULLSYNC = 123
+
+            def fcntl(self, fd, op):
+                called[:] = [fd, op]
+                return 0
+
+        with patch('swift.common.utils.fcntl', FCNTL()):
+            utils.fsync(12345)
+            self.assertEqual(called, [12345, 123])
+
+    def test_fsync_no_fullsync(self):
+        called = []
+
+        class FCNTL(object):
+            pass
+
+        def fsync(fd):
+            called.append(fd)
+
+        with patch('swift.common.utils.fcntl', FCNTL()):
+            with patch('os.fsync', fsync):
+                utils.fsync(12345)
+                self.assertEqual(called, [12345])
+
+
 @patch('ctypes.get_errno')
 @patch.object(utils, '_sys_posix_fallocate')
 @patch.object(utils, '_sys_fallocate')