Browse Source

Merge "Refactor the L3 agent batch notifier" into stable/stein

changes/85/681585/1
Zuul 1 week ago
parent
commit
a18213987d

+ 0
- 3
neutron/agent/l3/ha.py View File

@@ -182,9 +182,6 @@ class AgentMixin(object):
182 182
             ri.disable_radvd()
183 183
 
184 184
     def notify_server(self, batched_events):
185
-        eventlet.spawn_n(self._notify_server, batched_events)
186
-
187
-    def _notify_server(self, batched_events):
188 185
         translated_states = dict((router_id, TRANSLATION_MAP[state]) for
189 186
                                  router_id, state in batched_events)
190 187
         LOG.debug('Updating server with HA routers states %s',

+ 22
- 19
neutron/notifiers/batch_notifier.py View File

@@ -10,17 +10,17 @@
10 10
 #    License for the specific language governing permissions and limitations
11 11
 #    under the License.
12 12
 
13
+import threading
14
+
13 15
 import eventlet
14
-from neutron_lib.utils import runtime
15
-from oslo_utils import uuidutils
16 16
 
17 17
 
18 18
 class BatchNotifier(object):
19 19
     def __init__(self, batch_interval, callback):
20
-        self.pending_events = []
20
+        self._pending_events = eventlet.Queue()
21 21
         self.callback = callback
22 22
         self.batch_interval = batch_interval
23
-        self._lock_identifier = 'notifier-%s' % uuidutils.generate_uuid()
23
+        self._mutex = threading.Lock()
24 24
 
25 25
     def queue_event(self, event):
26 26
         """Called to queue sending an event with the next batch of events.
@@ -33,32 +33,35 @@ class BatchNotifier(object):
33 33
 
34 34
         This replaces the loopingcall with a mechanism that creates a
35 35
         short-lived thread on demand whenever an event is queued. That thread
36
-        will wait for a lock, send all queued events and then sleep for
37
-        'batch_interval' seconds to allow other events to queue up.
36
+        will check if the lock is released, send all queued events and then
37
+        sleep for 'batch_interval' seconds. If at the end of this sleep time,
38
+        other threads have added new events to the event queue, the same thread
39
+        will process them.
38 40
 
39
-        This effectively acts as a rate limiter to only allow 1 batch per
40
-        'batch_interval' seconds.
41
+        At the same time, other threads will be able to add new events to the
42
+        queue and will spawn new "synced_send" threads to process them. But if
43
+        the mutex is locked, the spawned thread will end immediately.
41 44
 
42 45
         :param event: the event that occurred.
43 46
         """
44 47
         if not event:
45 48
             return
46 49
 
47
-        self.pending_events.append(event)
50
+        self._pending_events.put(event)
48 51
 
49
-        @runtime.synchronized(self._lock_identifier)
50 52
         def synced_send():
51
-            self._notify()
52
-            # sleeping after send while holding the lock allows subsequent
53
-            # events to batch up
54
-            eventlet.sleep(self.batch_interval)
53
+            if not self._mutex.locked():
54
+                with self._mutex:
55
+                    while not self._pending_events.empty():
56
+                        self._notify()
57
+                        # sleeping after send while holding the lock allows
58
+                        # subsequent events to batch up
59
+                        eventlet.sleep(self.batch_interval)
55 60
 
56 61
         eventlet.spawn_n(synced_send)
57 62
 
58 63
     def _notify(self):
59
-        if not self.pending_events:
60
-            return
61
-
62
-        batched_events = self.pending_events
63
-        self.pending_events = []
64
+        batched_events = []
65
+        while not self._pending_events.empty():
66
+            batched_events.append(self._pending_events.get())
64 67
         self.callback(batched_events)

+ 43
- 29
neutron/tests/unit/notifiers/test_batch_notifier.py View File

@@ -16,6 +16,7 @@
16 16
 import eventlet
17 17
 import mock
18 18
 
19
+from neutron.common import utils
19 20
 from neutron.notifiers import batch_notifier
20 21
 from neutron.tests import base
21 22
 
@@ -23,41 +24,54 @@ from neutron.tests import base
23 24
 class TestBatchNotifier(base.BaseTestCase):
24 25
     def setUp(self):
25 26
         super(TestBatchNotifier, self).setUp()
26
-        self.notifier = batch_notifier.BatchNotifier(0.1, lambda x: x)
27
-        self.spawn_n_p = mock.patch('eventlet.spawn_n')
28
-        self.spawn_n = self.spawn_n_p.start()
27
+        self._received_events = eventlet.Queue()
28
+        self.notifier = batch_notifier.BatchNotifier(2, self._queue_events)
29
+        self.spawn_n_p = mock.patch.object(eventlet, 'spawn_n')
30
+
31
+    def _queue_events(self, events):
32
+        for event in events:
33
+            self._received_events.put(event)
29 34
 
30 35
     def test_queue_event_no_event(self):
36
+        spawn_n = self.spawn_n_p.start()
31 37
         self.notifier.queue_event(None)
32
-        self.assertEqual(0, len(self.notifier.pending_events))
33
-        self.assertEqual(0, self.spawn_n.call_count)
38
+        self.assertEqual(0, len(self.notifier._pending_events.queue))
39
+        self.assertEqual(0, spawn_n.call_count)
34 40
 
35 41
     def test_queue_event_first_event(self):
42
+        spawn_n = self.spawn_n_p.start()
36 43
         self.notifier.queue_event(mock.Mock())
37
-        self.assertEqual(1, len(self.notifier.pending_events))
38
-        self.assertEqual(1, self.spawn_n.call_count)
39
-
40
-    def test_queue_event_multiple_events(self):
41
-        self.spawn_n_p.stop()
42
-        c_mock = mock.patch.object(self.notifier, 'callback').start()
43
-        events = 6
44
-        for i in range(0, events):
45
-            self.notifier.queue_event(mock.Mock())
44
+        self.assertEqual(1, len(self.notifier._pending_events.queue))
45
+        self.assertEqual(1, spawn_n.call_count)
46
+
47
+    def test_queue_event_multiple_events_notify_method(self):
48
+        def _batch_notifier_dequeue():
49
+            while not self.notifier._pending_events.empty():
50
+                self.notifier._pending_events.get()
51
+
52
+        c_mock = mock.patch.object(self.notifier, '_notify',
53
+                                   side_effect=_batch_notifier_dequeue).start()
54
+        events = 20
55
+        for i in range(events):
56
+            self.notifier.queue_event('Event %s' % i)
46 57
             eventlet.sleep(0)  # yield to let coro execute
47 58
 
48
-        while self.notifier.pending_events:
49
-            # wait for coroutines to finish
50
-            eventlet.sleep(0.1)
59
+        utils.wait_until_true(self.notifier._pending_events.empty,
60
+                              timeout=5)
61
+        # Called twice: when the first thread calls "synced_send" and then,
62
+        # in the same loop, when self._pending_events is not empty(). All
63
+        # self.notifier.queue_event calls are done in just one
64
+        # "batch_interval" (2 secs).
51 65
         self.assertEqual(2, c_mock.call_count)
52
-        self.assertEqual(6, sum(len(c[0][0]) for c in c_mock.call_args_list))
53
-        self.assertEqual(0, len(self.notifier.pending_events))
54
-
55
-    def test_queue_event_call_send_events(self):
56
-        with mock.patch.object(self.notifier,
57
-                               'callback') as send_events:
58
-            self.spawn_n.side_effect = lambda func: func()
59
-            self.notifier.queue_event(mock.Mock())
60
-            while self.notifier.pending_events:
61
-                # wait for coroutines to finish
62
-                eventlet.sleep(0.1)
63
-            self.assertTrue(send_events.called)
66
+
67
+    def test_queue_event_multiple_events_callback_method(self):
68
+        events = 20
69
+        for i in range(events):
70
+            self.notifier.queue_event('Event %s' % i)
71
+            eventlet.sleep(0)  # yield to let coro execute
72
+
73
+        utils.wait_until_true(self.notifier._pending_events.empty,
74
+                              timeout=5)
75
+        expected = ['Event %s' % i for i in range(events)]
76
+        # Check the events have been handled in the same input order.
77
+        self.assertEqual(expected, list(self._received_events.queue))

+ 8
- 6
neutron/tests/unit/notifiers/test_nova.py View File

@@ -294,7 +294,7 @@ class TestNovaNotify(base.BaseTestCase):
294 294
         self.nova_notifier.send_network_change(
295 295
             'update_floatingip', original_obj, returned_obj)
296 296
         self.assertEqual(
297
-            2, len(self.nova_notifier.batch_notifier.pending_events))
297
+            2, len(self.nova_notifier.batch_notifier._pending_events.queue))
298 298
 
299 299
         returned_obj_non = {'floatingip': {'port_id': None}}
300 300
         event_dis = self.nova_notifier.create_port_changed_event(
@@ -302,9 +302,10 @@ class TestNovaNotify(base.BaseTestCase):
302 302
         event_assoc = self.nova_notifier.create_port_changed_event(
303 303
             'update_floatingip', original_obj, returned_obj)
304 304
         self.assertEqual(
305
-            self.nova_notifier.batch_notifier.pending_events[0], event_dis)
305
+            self.nova_notifier.batch_notifier._pending_events.get(), event_dis)
306 306
         self.assertEqual(
307
-            self.nova_notifier.batch_notifier.pending_events[1], event_assoc)
307
+            self.nova_notifier.batch_notifier._pending_events.get(),
308
+            event_assoc)
308 309
 
309 310
     def test_delete_port_notify(self):
310 311
         device_id = '32102d7b-1cf4-404d-b50a-97aae1f55f87'
@@ -365,6 +366,7 @@ class TestNovaNotify(base.BaseTestCase):
365 366
         self.nova_notifier.notify_port_active_direct(port)
366 367
 
367 368
         self.assertEqual(
368
-            1, len(self.nova_notifier.batch_notifier.pending_events))
369
-        self.assertEqual(expected_event,
370
-                         self.nova_notifier.batch_notifier.pending_events[0])
369
+            1, len(self.nova_notifier.batch_notifier._pending_events.queue))
370
+        self.assertEqual(
371
+            expected_event,
372
+            self.nova_notifier.batch_notifier._pending_events.get())

Loading…
Cancel
Save