openstack
/
neutron
Code Issues Proposed changes

Limit max ports per rpc for dhcp_ready_on_ports() 

The Neutron dhcp agents reports all ready ports to the Neutron
server via the dhcp_ready_on_ports() rpc call. When the dhcp agent
gets ports ready faster than the server can process them the amount
of ports per rpc call can grow so high (e.g. 10000 Ports) that the
neutron server never has a chance of processing the request before
the rpc timeout kills the request, leading to the dhcp agent
sending the request again, resulting in an endless loop of
dhcp_ready_on_ports() calls. This happens especially on agent startup.

To mitigate this problems we now limit the number of ports sent
per dhcp_ready_on_ports() call.

Closes-bug: #1834257
Change-Id: I407e126e760ebf6aca4c31b9c3ff58dcfa55107f
This commit is contained in:
Sebastian Lohff 2019-05-15 13:39:55 +02:00
parent db119dfe6c
commit 76ccdb35d4
2 changed files with 33 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
neutron/agent/dhcp/agent.py
View File

@ -50,6 +50,8 @@ DEFAULT_PRIORITY = 255
DHCP_PROCESS_GREENLET_MAX = 32
DHCP_PROCESS_GREENLET_MIN = 8
DHCP_READY_PORTS_SYNC_MAX = 64
def _sync_lock(f):
"""Decorator to block all operations for a global sync call."""
@ -241,8 +243,11 @@ class DhcpAgent(manager.Manager):
# this is just watching a set so we can do it really frequently
eventlet.sleep(0.1)
if self.dhcp_ready_ports:
ports_to_send = self.dhcp_ready_ports
self.dhcp_ready_ports = set()
ports_to_send = set()
for port_count in range(min(len(self.dhcp_ready_ports),
DHCP_READY_PORTS_SYNC_MAX)):
ports_to_send.add(self.dhcp_ready_ports.pop())
try:
self.plugin_rpc.dhcp_ready_on_ports(ports_to_send)
continue

26
neutron/tests/unit/agent/dhcp/test_agent.py
View File

@ -482,6 +482,32 @@ class TestDhcpAgent(base.BaseTestCase):
# should have been called with all ports again after the failure
ready.assert_has_calls([mock.call(set(range(4)))] * 2)
def test_dhcp_ready_ports_loop_with_limit_ports_per_call(self):
dhcp = dhcp_agent.DhcpAgent(HOSTNAME)
sync_max = dhcp_agent.DHCP_READY_PORTS_SYNC_MAX
port_count = sync_max + 1
dhcp.dhcp_ready_ports = set(range(port_count))
with mock.patch.object(dhcp.plugin_rpc,
'dhcp_ready_on_ports') as ready:
# exit after 2 iterations
with mock.patch.object(dhcp_agent.eventlet, 'sleep',
side_effect=[0, 0, RuntimeError]):
with testtools.ExpectedException(RuntimeError):
dhcp._dhcp_ready_ports_loop()
# all ports should have been processed
self.assertEqual(set(), dhcp.dhcp_ready_ports)
# two calls are expected, one with DHCP_READY_PORTS_SYNC_MAX ports,
# second one with one port
self.assertEqual(2, ready.call_count)
self.assertEqual(sync_max, len(ready.call_args_list[0][0][0]))
self.assertEqual(1, len(ready.call_args_list[1][0][0]))
# all ports need to be ready
ports_ready = (ready.call_args_list[0][0][0] |
ready.call_args_list[1][0][0])
self.assertEqual(set(range(port_count)), ports_ready)
def test_dhcp_ready_ports_updates_after_enable_dhcp(self):
dhcp = dhcp_agent.DhcpAgent(HOSTNAME)
self.assertEqual(set(), dhcp.dhcp_ready_ports)