From 2626baf3d603fcda9fc6be8cc63265530954a370 Mon Sep 17 00:00:00 2001 From: Sebastian Lohff Date: Wed, 15 May 2019 13:39:55 +0200 Subject: [PATCH] Limit max ports per rpc for dhcp_ready_on_ports() The Neutron dhcp agents reports all ready ports to the Neutron server via the dhcp_ready_on_ports() rpc call. When the dhcp agent gets ports ready faster than the server can process them the amount of ports per rpc call can grow so high (e.g. 10000 Ports) that the neutron server never has a chance of processing the request before the rpc timeout kills the request, leading to the dhcp agent sending the request again, resulting in an endless loop of dhcp_ready_on_ports() calls. This happens especially on agent startup. To mitigate this problems we now limit the number of ports sent per dhcp_ready_on_ports() call. Closes-bug: #1834257 Change-Id: I407e126e760ebf6aca4c31b9c3ff58dcfa55107f (cherry picked from commit 76ccdb35d4b09106aa8adc97528b02f0fd8acbcc) --- neutron/agent/dhcp/agent.py | 9 +++++-- neutron/tests/unit/agent/dhcp/test_agent.py | 26 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/neutron/agent/dhcp/agent.py b/neutron/agent/dhcp/agent.py index 8dc9be4c40d..ebc9f653a72 100644 --- a/neutron/agent/dhcp/agent.py +++ b/neutron/agent/dhcp/agent.py @@ -51,6 +51,8 @@ DEFAULT_PRIORITY = 255 DHCP_PROCESS_GREENLET_MAX = 32 DHCP_PROCESS_GREENLET_MIN = 8 +DHCP_READY_PORTS_SYNC_MAX = 64 + def _sync_lock(f): """Decorator to block all operations for a global sync call.""" @@ -243,8 +245,11 @@ class DhcpAgent(manager.Manager): # this is just watching a set so we can do it really frequently eventlet.sleep(0.1) if self.dhcp_ready_ports: - ports_to_send = self.dhcp_ready_ports - self.dhcp_ready_ports = set() + ports_to_send = set() + for port_count in range(min(len(self.dhcp_ready_ports), + DHCP_READY_PORTS_SYNC_MAX)): + ports_to_send.add(self.dhcp_ready_ports.pop()) + try: self.plugin_rpc.dhcp_ready_on_ports(ports_to_send) continue diff --git a/neutron/tests/unit/agent/dhcp/test_agent.py b/neutron/tests/unit/agent/dhcp/test_agent.py index 0a5f9c9f86c..460d218896e 100644 --- a/neutron/tests/unit/agent/dhcp/test_agent.py +++ b/neutron/tests/unit/agent/dhcp/test_agent.py @@ -482,6 +482,32 @@ class TestDhcpAgent(base.BaseTestCase): # should have been called with all ports again after the failure ready.assert_has_calls([mock.call(set(range(4)))] * 2) + def test_dhcp_ready_ports_loop_with_limit_ports_per_call(self): + dhcp = dhcp_agent.DhcpAgent(HOSTNAME) + sync_max = dhcp_agent.DHCP_READY_PORTS_SYNC_MAX + port_count = sync_max + 1 + dhcp.dhcp_ready_ports = set(range(port_count)) + + with mock.patch.object(dhcp.plugin_rpc, + 'dhcp_ready_on_ports') as ready: + # exit after 2 iterations + with mock.patch.object(dhcp_agent.eventlet, 'sleep', + side_effect=[0, 0, RuntimeError]): + with testtools.ExpectedException(RuntimeError): + dhcp._dhcp_ready_ports_loop() + + # all ports should have been processed + self.assertEqual(set(), dhcp.dhcp_ready_ports) + # two calls are expected, one with DHCP_READY_PORTS_SYNC_MAX ports, + # second one with one port + self.assertEqual(2, ready.call_count) + self.assertEqual(sync_max, len(ready.call_args_list[0][0][0])) + self.assertEqual(1, len(ready.call_args_list[1][0][0])) + # all ports need to be ready + ports_ready = (ready.call_args_list[0][0][0] | + ready.call_args_list[1][0][0]) + self.assertEqual(set(range(port_count)), ports_ready) + def test_dhcp_ready_ports_updates_after_enable_dhcp(self): dhcp = dhcp_agent.DhcpAgent(HOSTNAME) self.assertEqual(set(), dhcp.dhcp_ready_ports)