Dynamically increase DHCP process queue green pool size

As done for the l3-agent in 837c9283ab,
dynamically resize the DHCP process queue green pool.

This patch adds a new measurement based on the network quantity to
indicate the DHCP process queue green pool size. The pool size
will be limited from 8 (original value) to 32, because we do not want
to increase the DHCP agent processing cost on the node.

Change-Id: Ic0e7bc15f138273c7a6ad41f228c9f315e6c7a91
Related-Bug: #1813787

neutron/agent/dhcp/agent.py

@@ -48,6 +48,9 @@ _SYNC_STATE_LOCK = lockutils.ReaderWriterLock()
 
 DEFAULT_PRIORITY = 255
 
+DHCP_PROCESS_GREENLET_MAX = 32
+DHCP_PROCESS_GREENLET_MIN = 8
+
 
 def _sync_lock(f):
     """Decorator to block all operations for a global sync call."""
@@ -106,6 +109,8 @@ class DhcpAgent(manager.Manager):
         self._process_monitor = external_process.ProcessMonitor(
             config=self.conf,
             resource_type='dhcp')
+        self._pool_size = DHCP_PROCESS_GREENLET_MIN
+        self._pool = eventlet.GreenPool(size=self._pool_size)
         self._queue = queue.ResourceProcessingQueue()
 
     def init_host(self):
@@ -330,6 +335,8 @@ class DhcpAgent(manager.Manager):
                     self.dhcp_ready_ports |= {p.id for p in network.ports}
                 break
 
+        self._resize_process_pool()
+
     def disable_dhcp_helper(self, network_id):
         """Disable DHCP for a network known to the agent."""
         network = self.cache.get_network_by_id(network_id)
@@ -345,6 +352,8 @@ class DhcpAgent(manager.Manager):
             if self.call_driver('disable', network):
                 self.cache.remove(network)
 
+        self._resize_process_pool()
+
     def refresh_dhcp_helper(self, network_id):
         """Refresh or disable DHCP for a network depending on the current state
         of the network.
@@ -482,12 +491,23 @@ class DhcpAgent(manager.Manager):
             return
         self.refresh_dhcp_helper(network.id)
 
+    @lockutils.synchronized('resize_greenpool')
+    def _resize_process_pool(self):
+        num_nets = len(self.cache.get_network_ids())
+        pool_size = max([DHCP_PROCESS_GREENLET_MIN,
+                         min([DHCP_PROCESS_GREENLET_MAX, num_nets])])
+        if pool_size == self._pool_size:
+            return
+        LOG.info("Resizing dhcp processing queue green pool size to: %d",
+                 pool_size)
+        self._pool.resize(pool_size)
+        self._pool_size = pool_size
+
     def _process_loop(self):
         LOG.debug("Starting _process_loop")
 
-        pool = eventlet.GreenPool(size=8)
         while True:
-            pool.spawn_n(self._process_resource_update)
+            self._pool.spawn_n(self._process_resource_update)
 
     def _process_resource_update(self):
         for tmp, update in self._queue.each_update_to_next_resource():

neutron/tests/functional/agent/test_dhcp_agent.py

@@ -399,3 +399,43 @@ class DHCPAgentOVSTestCase(DHCPAgentOVSTestFramework):
             exception=RuntimeError("'dhcp_ready_on_ports' not be called"))
         self.mock_plugin_api.dhcp_ready_on_ports.assert_called_with(
             ports_to_send)
+
+    def test_dhcp_processing_pool_size(self):
+        mock.patch.object(self.agent, 'call_driver').start().return_value = (
+            True)
+        self.agent.update_isolated_metadata_proxy = mock.Mock()
+        self.agent.disable_isolated_metadata_proxy = mock.Mock()
+
+        network_info_1 = self.network_dict_for_dhcp()
+        self.configure_dhcp_for_network(network=network_info_1)
+        self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN,
+                         self.agent._pool.size)
+
+        network_info_2 = self.network_dict_for_dhcp()
+        self.configure_dhcp_for_network(network=network_info_2)
+        self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN,
+                         self.agent._pool.size)
+
+        network_info_list = [network_info_1, network_info_2]
+        for _i in range(agent.DHCP_PROCESS_GREENLET_MAX + 1):
+            ni = self.network_dict_for_dhcp()
+            self.configure_dhcp_for_network(network=ni)
+            network_info_list.append(ni)
+
+        self.assertEqual(agent.DHCP_PROCESS_GREENLET_MAX,
+                         self.agent._pool.size)
+
+        for network in network_info_list:
+            self.agent.disable_dhcp_helper(network.id)
+
+        agent_network_info_len = len(self.agent.cache.get_network_ids())
+        if agent_network_info_len < agent.DHCP_PROCESS_GREENLET_MIN:
+            self.assertEqual(agent.DHCP_PROCESS_GREENLET_MIN,
+                             self.agent._pool.size)
+        elif (agent.DHCP_PROCESS_GREENLET_MIN <= agent_network_info_len <=
+              agent.DHCP_PROCESS_GREENLET_MAX):
+            self.assertEqual(agent_network_info_len,
+                             self.agent._pool.size)
+        else:
+            self.assertEqual(agent.DHCP_PROCESS_GREENLET_MAX,
+                             self.agent._pool.size)

neutron/tests/unit/agent/dhcp/test_agent.py

@@ -685,6 +685,9 @@ class TestDhcpAgentEventHandler(base.BaseTestCase):
             'neutron.agent.linux.external_process.ProcessManager'
         )
         self.external_process = self.external_process_p.start()
+        self.mock_resize_p = mock.patch('neutron.agent.dhcp.agent.'
+                                        'DhcpAgent._resize_process_pool')
+        self.mock_resize = self.mock_resize_p.start()
 
     def _process_manager_constructor_call(self, ns=FAKE_NETWORK_DHCP_NS):
         return mock.call(conf=cfg.CONF,

releasenotes/notes/dynamically-resize-agent-greenthreads-c163ab37d36fcafe.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    The Neutron L3 and DHCP agents now dynamically tune the number of
+    processing greenthreads they run based on the number of objects they
+    are managing, with the current values for this range being between eight
+    and thirty-two threads, which is an increase over the previous static value
+    of eight threads. This should help address some of the scaling problems
+    in the agents. For more information see bug
+    `1813787 <https://bugs.launchpad.net/neutron/+bug/1813787>`_.