Merge "Parallelize per-node resource updates"

doc/source/admin/configuration/hypervisor-ironic.rst

@@ -56,10 +56,19 @@ Scaling and performance issues
   a conductor group within your deployment.
 - The ``update_available_resource`` periodic task reports all the resources
   managed by Ironic. Depending the number of nodes, it can take a lot of time.
-  The nova-compute will not perform any other operations when this task is
-  running. You can use conductor groups to help shard your deployment
-  between multiple nova-compute processes by setting
-  :oslo.config:option:`ironic.conductor_group`.
+  You can use Ironic shards to help shard your deployment between multiple
+  nova-compute processes by setting :oslo.config:option:`ironic.shard` on a
+  per nova-compute basis. Setting the shard key restricts a nova-compute
+  instance to only manage a subset of Ironic nodes.
+
+  Large Ironic deployments can also tune
+  :oslo.config:option:`compute.update_resources_max_workers` to update
+  multiple nodes concurrently within a single nova-compute service. The
+  default value of ``1`` preserves the previous sequential behavior. Increasing
+  the value can reduce nova-compute startup and periodic resource-reporting
+  latency, but also increases nova-compute CPU usage and concurrent Placement
+  API traffic. Increase it gradually and monitor nova-compute and Placement
+  service load.
 - The nova-compute process using the Ironic driver can be moved between
   different physical servers using active/passive failover. But when doing
   this failover, you must ensure :oslo.config:option:`host` is the same

nova/compute/manager.py

@@ -11205,9 +11205,40 @@ class ComputeManager(manager.Manager):
                             "Failed to delete compute node resource provider "
                             "for compute node %s: %s", cn.uuid, str(e))
 
-        for nodename in nodenames:
-            self._update_available_resource_for_node(context, nodename,
-                                                     startup=startup)
+        # Run per-node updates in parallel.  Each node's update touches
+        # only its own ResourceTracker state and its own ProviderTree
+        # subtree (protected by the report client's lock), so concurrent
+        # execution is safe.
+        futures = [
+            (
+                nodename,
+                thread_pool_factory.spawn_on(
+                    thread_pool_factory.ExecutorType.UPDATE_RESOURCES,
+                    self._update_available_resource_for_node,
+                    context, nodename, startup=startup),
+            )
+            for nodename in nodenames
+        ]
+        # Drain all futures so errors are re-raised in this thread rather
+        # than being silently lost. Keep draining after the first failure
+        # so all scheduled per-node updates finish before we preserve the
+        # sequential path's exception propagation semantics.
+        first_exc: Exception | None = None
+        failed_nodes = []
+        for nodename, fut in futures:
+            try:
+                fut.result()
+            except Exception as exc:
+                failed_nodes.append(nodename)
+                if first_exc is None:
+                    first_exc = exc
+
+        if first_exc is not None:
+            LOG.error(
+                "update_available_resource failed for %(count)d nodes; "
+                "re-raising the first failure from node %(node)s.",
+                {'count': len(failed_nodes), 'node': failed_nodes[0]})
+            raise first_exc
 
     def _get_compute_nodes_in_db(self, context, nodenames, use_slave=False,
                                  startup=False):

nova/conf/compute.py

@@ -1078,6 +1078,28 @@ Possible values:
 * ``True``: Packing VM's NUMA cell on most used host NUMA cell.
 * ``False``: Spreading VM's NUMA cell on host's NUMA cells with more resources
   available.
+"""),
+    cfg.IntOpt(
+        'update_resources_max_workers',
+        default=1,
+        min=1,
+        help="""
+Maximum number of worker threads used to run per-node
+update_available_resource() calls in parallel during the
+update_available_resource periodic task.
+
+For drivers that manage a single compute node (most hypervisors) the
+default of 1 is optimal.  For the Ironic driver, which manages hundreds
+or thousands of baremetal nodes under a single nova-compute service,
+increasing this value allows node updates to proceed concurrently and
+dramatically reduces startup and periodic-task latency.
+
+Setting this to 1 (the default) preserves the original sequential
+behaviour.
+
+Related options:
+
+* update_resources_interval
 """),
 ]
 

nova/tests/unit/compute/test_compute_mgr.py

@@ -536,6 +536,65 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase,
         self.assertEqual(1, mock_rt.remove_node.call_count)
         mock_rt.clean_compute_node_cache.assert_called_once_with(db_nodes)
 
+    @mock.patch('nova.thread_pool_factory.spawn_on')
+    @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
+    @mock.patch.object(manager.ComputeManager, '_get_compute_nodes_in_db')
+    def test_update_available_resource_parallel(
+            self, get_db_nodes, get_avail_nodes, mock_spawn_on):
+        """When [compute] update_resources_max_workers > 1, all nodes must
+        still be scheduled and the parallel path must complete without error.
+        """
+        get_db_nodes.return_value = []
+        avail_nodes = set(['node1', 'node2', 'node3'])
+        get_avail_nodes.return_value = avail_nodes
+        futures = [mock.Mock() for _node in avail_nodes]
+        mock_spawn_on.side_effect = futures
+
+        self.flags(update_resources_max_workers=4, group='compute')
+
+        self.compute.update_available_resource(self.context, startup=True)
+
+        self.assertEqual(len(avail_nodes), mock_spawn_on.call_count)
+        for nodename in avail_nodes:
+            mock_spawn_on.assert_any_call(
+                thread_pool_factory.ExecutorType.UPDATE_RESOURCES,
+                self.compute._update_available_resource_for_node,
+                self.context, nodename, startup=True)
+        for fut in futures:
+            fut.result.assert_called_once_with()
+
+    @mock.patch('nova.thread_pool_factory.spawn_on')
+    @mock.patch.object(fake_driver.FakeDriver, 'get_available_nodes')
+    @mock.patch.object(manager.ComputeManager, '_get_compute_nodes_in_db')
+    def test_update_available_resource_parallel_reraises_first_failure(
+            self, get_db_nodes, get_avail_nodes, mock_spawn_on):
+        get_db_nodes.return_value = []
+        get_avail_nodes.return_value = set(['node1', 'node2', 'node3'])
+        first_failure = exception.InvalidConfiguration(reason='broken')
+        second_failure = test.TestingException()
+        futures = [mock.Mock() for _node in get_avail_nodes.return_value]
+        futures[0].result.side_effect = first_failure
+        futures[2].result.side_effect = second_failure
+        mock_spawn_on.side_effect = futures
+
+        self.flags(update_resources_max_workers=4, group='compute')
+
+        with mock.patch.object(manager.LOG, 'error') as mock_log_error:
+            ex = self.assertRaises(
+                exception.InvalidConfiguration,
+                self.compute.update_available_resource,
+                self.context,
+                startup=True)
+
+        self.assertIs(first_failure, ex)
+        for fut in futures:
+            fut.result.assert_called_once_with()
+        first_failed_node = mock_spawn_on.call_args_list[0].args[3]
+        mock_log_error.assert_called_once_with(
+            "update_available_resource failed for %(count)d nodes; "
+            "re-raising the first failure from node %(node)s.",
+            {'count': 2, 'node': first_failed_node})
+
     @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
                 'delete_resource_provider')
     @mock.patch.object(manager.ComputeManager,

nova/thread_pool_factory.py

@@ -115,6 +115,7 @@ class ExecutorType(enum.Enum):
     LONG_TASK = "long_task"
     SYNC_POWER = "sync_power_state"
     LIVE_MIGRATION = "live_migration"
+    UPDATE_RESOURCES = "update_resources"
 
 
 class ExecutorsPoolSize:
@@ -165,6 +166,10 @@ class ExecutorsPoolSize:
             return max_builds + max_snapshots
         return max_tasks
 
+    @staticmethod
+    def _update_resources_pool_size():
+        return CONF.compute.update_resources_max_workers
+
     _EXECUTOR_POOL_SIZE = {
         ExecutorType.DEFAULT: _default_pool_size,
         ExecutorType.SCATTER_GATHER: _scatter_gather_pool_size,
@@ -172,6 +177,7 @@ class ExecutorsPoolSize:
         ExecutorType.LONG_TASK: _long_task_pool_size,
         ExecutorType.SYNC_POWER: _sync_power_pool_size,
         ExecutorType.LIVE_MIGRATION: get_max_concurrent_live_migrations,
+        ExecutorType.UPDATE_RESOURCES: _update_resources_pool_size,
     }
 
     @classmethod

releasenotes/notes/bug-ironic-compute-startup-performance-bf5e0a686f8f4c6b.yaml

@@ -0,0 +1,15 @@
+---
+fixes:
+  - |
+    Improved nova-compute startup and resource reporting performance for large
+    Ironic deployments. The compute service now avoids repeated per-instance
+    metadata lookups during startup, limits vTPM startup validation to the
+    libvirt driver, avoids deep-copying the full Placement provider tree for
+    each Ironic node, and can update per-node resources concurrently using the
+    new ``[compute]update_resources_max_workers`` option.
+
+    The new option defaults to ``1``, preserving the previous sequential
+    behavior. Operators of large Ironic deployments can increase it to reduce
+    ``update_available_resource`` startup and periodic-task latency, trading
+    lower wall-clock time for additional nova-compute CPU usage and Placement
+    API concurrency.