Limit max ports per rpc for dhcp_ready_on_ports()
The Neutron dhcp agents reports all ready ports to the Neutron
server via the dhcp_ready_on_ports() rpc call. When the dhcp agent
gets ports ready faster than the server can process them the amount
of ports per rpc call can grow so high (e.g. 10000 Ports) that the
neutron server never has a chance of processing the request before
the rpc timeout kills the request, leading to the dhcp agent
sending the request again, resulting in an endless loop of
dhcp_ready_on_ports() calls. This happens especially on agent startup.
To mitigate this problems we now limit the number of ports sent
per dhcp_ready_on_ports() call.
Closes-bug: #1834257
Change-Id: I407e126e760ebf6aca4c31b9c3ff58dcfa55107f
(cherry picked from commit 76ccdb35d4
)
This commit is contained in:
parent
55a503b4c9
commit
f9f6ae9c98
|
@ -47,6 +47,8 @@ _SYNC_STATE_LOCK = lockutils.ReaderWriterLock()
|
|||
|
||||
DEFAULT_PRIORITY = 255
|
||||
|
||||
DHCP_READY_PORTS_SYNC_MAX = 64
|
||||
|
||||
|
||||
def _sync_lock(f):
|
||||
"""Decorator to block all operations for a global sync call."""
|
||||
|
@ -222,8 +224,11 @@ class DhcpAgent(manager.Manager):
|
|||
# this is just watching a set so we can do it really frequently
|
||||
eventlet.sleep(0.1)
|
||||
if self.dhcp_ready_ports:
|
||||
ports_to_send = self.dhcp_ready_ports
|
||||
self.dhcp_ready_ports = set()
|
||||
ports_to_send = set()
|
||||
for port_count in range(min(len(self.dhcp_ready_ports),
|
||||
DHCP_READY_PORTS_SYNC_MAX)):
|
||||
ports_to_send.add(self.dhcp_ready_ports.pop())
|
||||
|
||||
try:
|
||||
self.plugin_rpc.dhcp_ready_on_ports(ports_to_send)
|
||||
continue
|
||||
|
|
|
@ -459,6 +459,32 @@ class TestDhcpAgent(base.BaseTestCase):
|
|||
# should have been called with all ports again after the failure
|
||||
ready.assert_has_calls([mock.call(set(range(4)))] * 2)
|
||||
|
||||
def test_dhcp_ready_ports_loop_with_limit_ports_per_call(self):
|
||||
dhcp = dhcp_agent.DhcpAgent(HOSTNAME)
|
||||
sync_max = dhcp_agent.DHCP_READY_PORTS_SYNC_MAX
|
||||
port_count = sync_max + 1
|
||||
dhcp.dhcp_ready_ports = set(range(port_count))
|
||||
|
||||
with mock.patch.object(dhcp.plugin_rpc,
|
||||
'dhcp_ready_on_ports') as ready:
|
||||
# exit after 2 iterations
|
||||
with mock.patch.object(dhcp_agent.eventlet, 'sleep',
|
||||
side_effect=[0, 0, RuntimeError]):
|
||||
with testtools.ExpectedException(RuntimeError):
|
||||
dhcp._dhcp_ready_ports_loop()
|
||||
|
||||
# all ports should have been processed
|
||||
self.assertEqual(set(), dhcp.dhcp_ready_ports)
|
||||
# two calls are expected, one with DHCP_READY_PORTS_SYNC_MAX ports,
|
||||
# second one with one port
|
||||
self.assertEqual(2, ready.call_count)
|
||||
self.assertEqual(sync_max, len(ready.call_args_list[0][0][0]))
|
||||
self.assertEqual(1, len(ready.call_args_list[1][0][0]))
|
||||
# all ports need to be ready
|
||||
ports_ready = (ready.call_args_list[0][0][0] |
|
||||
ready.call_args_list[1][0][0])
|
||||
self.assertEqual(set(range(port_count)), ports_ready)
|
||||
|
||||
def test_dhcp_ready_ports_updates_after_enable_dhcp(self):
|
||||
dhcp = dhcp_agent.DhcpAgent(HOSTNAME)
|
||||
self.assertEqual(set(), dhcp.dhcp_ready_ports)
|
||||
|
|
Loading…
Reference in New Issue