From 7369b69e2ef5b1b3c30b237885c2648c63f1dffb Mon Sep 17 00:00:00 2001 From: Brian Haley Date: Wed, 6 Mar 2019 16:47:27 -0500 Subject: [PATCH] Dynamically increase DHCP process queue green pool size As done for the l3-agent in 837c9283abd4ccb56d5b4ad0eb1ca435cd2fdf3b, dynamically resize the DHCP process queue green pool. This patch adds a new measurement based on the network quantity to indicate the DHCP process queue green pool size. The pool size will be limited from 8 (original value) to 32, because we do not want to increase the DHCP agent processing cost on the node. Change-Id: Ic0e7bc15f138273c7a6ad41f228c9f315e6c7a91 Related-Bug: #1813787 --- neutron/agent/dhcp/agent.py | 24 ++++++++++- .../tests/functional/agent/test_dhcp_agent.py | 40 +++++++++++++++++++ neutron/tests/unit/agent/dhcp/test_agent.py | 3 ++ ...e-agent-greenthreads-c163ab37d36fcafe.yaml | 10 +++++ 4 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/dynamically-resize-agent-greenthreads-c163ab37d36fcafe.yaml diff --git a/neutron/agent/dhcp/agent.py b/neutron/agent/dhcp/agent.py index a32d45aab5f..f588dd8bb5c 100644 --- a/neutron/agent/dhcp/agent.py +++ b/neutron/agent/dhcp/agent.py @@ -48,6 +48,9 @@ _SYNC_STATE_LOCK = lockutils.ReaderWriterLock() DEFAULT_PRIORITY = 255 +DHCP_PROCESS_GREENLET_MAX = 32 +DHCP_PROCESS_GREENLET_MIN = 8 + def _sync_lock(f): """Decorator to block all operations for a global sync call.""" @@ -106,6 +109,8 @@ class DhcpAgent(manager.Manager): self._process_monitor = external_process.ProcessMonitor( config=self.conf, resource_type='dhcp') + self._pool_size = DHCP_PROCESS_GREENLET_MIN + self._pool = eventlet.GreenPool(size=self._pool_size) self._queue = queue.ResourceProcessingQueue() def init_host(self): @@ -330,6 +335,8 @@ class DhcpAgent(manager.Manager): self.dhcp_ready_ports |= {p.id for p in network.ports} break + self._resize_process_pool() + def disable_dhcp_helper(self, network_id): """Disable DHCP for a network known to the agent.""" network = self.cache.get_network_by_id(network_id) @@ -345,6 +352,8 @@ class DhcpAgent(manager.Manager): if self.call_driver('disable', network): self.cache.remove(network) + self._resize_process_pool() + def refresh_dhcp_helper(self, network_id): """Refresh or disable DHCP for a network depending on the current state of the network. @@ -482,12 +491,23 @@ class DhcpAgent(manager.Manager): return self.refresh_dhcp_helper(network.id) + @lockutils.synchronized('resize_greenpool') + def _resize_process_pool(self): + num_nets = len(self.cache.get_network_ids()) + pool_size = max([DHCP_PROCESS_GREENLET_MIN, + min([DHCP_PROCESS_GREENLET_MAX, num_nets])]) + if pool_size == self._pool_size: + return + LOG.info("Resizing dhcp processing queue green pool size to: %d", + pool_size) + self._pool.resize(pool_size) + self._pool_size = pool_size + def _process_loop(self): LOG.debug("Starting _process_loop") - pool = eventlet.GreenPool(size=8) while True: - pool.spawn_n(self._process_resource_update) + self._pool.spawn_n(self._process_resource_update) def _process_resource_update(self): for tmp, update in self._queue.each_update_to_next_resource(): diff --git a/neutron/tests/functional/agent/test_dhcp_agent.py b/neutron/tests/functional/agent/test_dhcp_agent.py index 43eda5fa1ec..98b5ae3df51 100644 --- a/neutron/tests/functional/agent/test_dhcp_agent.py +++ b/neutron/tests/functional/agent/test_dhcp_agent.py @@ -399,3 +399,43 @@ class DHCPAgentOVSTestCase(DHCPAgentOVSTestFramework): exception=RuntimeError("'dhcp_ready_on_ports' not be called")) self.mock_plugin_api.dhcp_ready_on_ports.assert_called_with( ports_to_send) + + def test_dhcp_processing_pool_size(self): + mock.patch.object(self.agent, 'call_driver').start().return_value = ( + True) + self.agent.update_isolated_metadata_proxy = mock.Mock() + self.agent.disable_isolated_metadata_proxy = mock.Mock() + + network_info_1 = self.network_dict_for_dhcp() + self.configure_dhcp_for_network(network=network_info_1) + self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN, + self.agent._pool.size) + + network_info_2 = self.network_dict_for_dhcp() + self.configure_dhcp_for_network(network=network_info_2) + self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN, + self.agent._pool.size) + + network_info_list = [network_info_1, network_info_2] + for _i in range(agent.DHCP_PROCESS_GREENLET_MAX + 1): + ni = self.network_dict_for_dhcp() + self.configure_dhcp_for_network(network=ni) + network_info_list.append(ni) + + self.assertEqual(agent.DHCP_PROCESS_GREENLET_MAX, + self.agent._pool.size) + + for network in network_info_list: + self.agent.disable_dhcp_helper(network.id) + + agent_network_info_len = len(self.agent.cache.get_network_ids()) + if agent_network_info_len < agent.DHCP_PROCESS_GREENLET_MIN: + self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN, + self.agent._pool.size) + elif (agent.DHCP_PROCESS_GREENLET_MIN <= agent_network_info_len <= + agent.DHCP_PROCESS_GREENLET_MAX): + self.assertEqual(agent_network_info_len, + self.agent._pool.size) + else: + self.assertEqual(agent.DHCP_PROCESS_GREENLET_MAX, + self.agent._pool.size) diff --git a/neutron/tests/unit/agent/dhcp/test_agent.py b/neutron/tests/unit/agent/dhcp/test_agent.py index a7f5132e8e1..ee27cbeab0a 100644 --- a/neutron/tests/unit/agent/dhcp/test_agent.py +++ b/neutron/tests/unit/agent/dhcp/test_agent.py @@ -685,6 +685,9 @@ class TestDhcpAgentEventHandler(base.BaseTestCase): 'neutron.agent.linux.external_process.ProcessManager' ) self.external_process = self.external_process_p.start() + self.mock_resize_p = mock.patch('neutron.agent.dhcp.agent.' + 'DhcpAgent._resize_process_pool') + self.mock_resize = self.mock_resize_p.start() def _process_manager_constructor_call(self, ns=FAKE_NETWORK_DHCP_NS): return mock.call(conf=cfg.CONF, diff --git a/releasenotes/notes/dynamically-resize-agent-greenthreads-c163ab37d36fcafe.yaml b/releasenotes/notes/dynamically-resize-agent-greenthreads-c163ab37d36fcafe.yaml new file mode 100644 index 00000000000..6f7fcddfb43 --- /dev/null +++ b/releasenotes/notes/dynamically-resize-agent-greenthreads-c163ab37d36fcafe.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The Neutron L3 and DHCP agents now dynamically tune the number of + processing greenthreads they run based on the number of objects they + are managing, with the current values for this range being between eight + and thirty-two threads, which is an increase over the previous static value + of eight threads. This should help address some of the scaling problems + in the agents. For more information see bug + `1813787 `_.