Browse Source

Dynamically increase DHCP process queue green pool size

As done for the l3-agent in 837c9283ab,
dynamically resize the DHCP process queue green pool.

This patch adds a new measurement based on the network quantity to
indicate the DHCP process queue green pool size. The pool size
will be limited from 8 (original value) to 32, because we do not want
to increase the DHCP agent processing cost on the node.

Change-Id: Ic0e7bc15f138273c7a6ad41f228c9f315e6c7a91
Related-Bug: #1813787
changes/90/641490/9
Brian Haley 3 years ago committed by Slawek Kaplonski
parent
commit
7369b69e2e
  1. 24
      neutron/agent/dhcp/agent.py
  2. 40
      neutron/tests/functional/agent/test_dhcp_agent.py
  3. 3
      neutron/tests/unit/agent/dhcp/test_agent.py
  4. 10
      releasenotes/notes/dynamically-resize-agent-greenthreads-c163ab37d36fcafe.yaml

24
neutron/agent/dhcp/agent.py

@ -48,6 +48,9 @@ _SYNC_STATE_LOCK = lockutils.ReaderWriterLock()
DEFAULT_PRIORITY = 255
DHCP_PROCESS_GREENLET_MAX = 32
DHCP_PROCESS_GREENLET_MIN = 8
def _sync_lock(f):
"""Decorator to block all operations for a global sync call."""
@ -106,6 +109,8 @@ class DhcpAgent(manager.Manager):
self._process_monitor = external_process.ProcessMonitor(
config=self.conf,
resource_type='dhcp')
self._pool_size = DHCP_PROCESS_GREENLET_MIN
self._pool = eventlet.GreenPool(size=self._pool_size)
self._queue = queue.ResourceProcessingQueue()
def init_host(self):
@ -330,6 +335,8 @@ class DhcpAgent(manager.Manager):
self.dhcp_ready_ports |= {p.id for p in network.ports}
break
self._resize_process_pool()
def disable_dhcp_helper(self, network_id):
"""Disable DHCP for a network known to the agent."""
network = self.cache.get_network_by_id(network_id)
@ -345,6 +352,8 @@ class DhcpAgent(manager.Manager):
if self.call_driver('disable', network):
self.cache.remove(network)
self._resize_process_pool()
def refresh_dhcp_helper(self, network_id):
"""Refresh or disable DHCP for a network depending on the current state
of the network.
@ -482,12 +491,23 @@ class DhcpAgent(manager.Manager):
return
self.refresh_dhcp_helper(network.id)
@lockutils.synchronized('resize_greenpool')
def _resize_process_pool(self):
num_nets = len(self.cache.get_network_ids())
pool_size = max([DHCP_PROCESS_GREENLET_MIN,
min([DHCP_PROCESS_GREENLET_MAX, num_nets])])
if pool_size == self._pool_size:
return
LOG.info("Resizing dhcp processing queue green pool size to: %d",
pool_size)
self._pool.resize(pool_size)
self._pool_size = pool_size
def _process_loop(self):
LOG.debug("Starting _process_loop")
pool = eventlet.GreenPool(size=8)
while True:
pool.spawn_n(self._process_resource_update)
self._pool.spawn_n(self._process_resource_update)
def _process_resource_update(self):
for tmp, update in self._queue.each_update_to_next_resource():

40
neutron/tests/functional/agent/test_dhcp_agent.py

@ -399,3 +399,43 @@ class DHCPAgentOVSTestCase(DHCPAgentOVSTestFramework):
exception=RuntimeError("'dhcp_ready_on_ports' not be called"))
self.mock_plugin_api.dhcp_ready_on_ports.assert_called_with(
ports_to_send)
def test_dhcp_processing_pool_size(self):
mock.patch.object(self.agent, 'call_driver').start().return_value = (
True)
self.agent.update_isolated_metadata_proxy = mock.Mock()
self.agent.disable_isolated_metadata_proxy = mock.Mock()
network_info_1 = self.network_dict_for_dhcp()
self.configure_dhcp_for_network(network=network_info_1)
self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN,
self.agent._pool.size)
network_info_2 = self.network_dict_for_dhcp()
self.configure_dhcp_for_network(network=network_info_2)
self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN,
self.agent._pool.size)
network_info_list = [network_info_1, network_info_2]
for _i in range(agent.DHCP_PROCESS_GREENLET_MAX + 1):
ni = self.network_dict_for_dhcp()
self.configure_dhcp_for_network(network=ni)
network_info_list.append(ni)
self.assertEqual(agent.DHCP_PROCESS_GREENLET_MAX,
self.agent._pool.size)
for network in network_info_list:
self.agent.disable_dhcp_helper(network.id)
agent_network_info_len = len(self.agent.cache.get_network_ids())
if agent_network_info_len < agent.DHCP_PROCESS_GREENLET_MIN:
self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN,
self.agent._pool.size)
elif (agent.DHCP_PROCESS_GREENLET_MIN <= agent_network_info_len <=
agent.DHCP_PROCESS_GREENLET_MAX):
self.assertEqual(agent_network_info_len,
self.agent._pool.size)
else:
self.assertEqual(agent.DHCP_PROCESS_GREENLET_MAX,
self.agent._pool.size)

3
neutron/tests/unit/agent/dhcp/test_agent.py

@ -685,6 +685,9 @@ class TestDhcpAgentEventHandler(base.BaseTestCase):
'neutron.agent.linux.external_process.ProcessManager'
)
self.external_process = self.external_process_p.start()
self.mock_resize_p = mock.patch('neutron.agent.dhcp.agent.'
'DhcpAgent._resize_process_pool')
self.mock_resize = self.mock_resize_p.start()
def _process_manager_constructor_call(self, ns=FAKE_NETWORK_DHCP_NS):
return mock.call(conf=cfg.CONF,

10
releasenotes/notes/dynamically-resize-agent-greenthreads-c163ab37d36fcafe.yaml

@ -0,0 +1,10 @@
---
features:
- |
The Neutron L3 and DHCP agents now dynamically tune the number of
processing greenthreads they run based on the number of objects they
are managing, with the current values for this range being between eight
and thirty-two threads, which is an increase over the previous static value
of eight threads. This should help address some of the scaling problems
in the agents. For more information see bug
`1813787 <https://bugs.launchpad.net/neutron/+bug/1813787>`_.
Loading…
Cancel
Save