Merge "Add lock to host-state consumption"

nova/scheduler/host_manager.py

@@ -159,6 +159,10 @@ class HostState(object):
 
         @utils.synchronized(self._lock_name)
         def _locked_update(self, compute, service, aggregates, inst_dict):
+            # Scheduler API is inherently multi-threaded as every incoming RPC
+            # message will be dispatched in it's own green thread. So the
+            # shared host state should be updated in a consistent way to make
+            # sure its data is valid under concurrent write operations.
             if compute is not None:
                 LOG.debug("Update host state from compute node: %s", compute)
                 self._update_from_compute_node(compute)
@@ -238,9 +242,21 @@ class HostState(object):
         self.cpu_allocation_ratio = compute.cpu_allocation_ratio
         self.ram_allocation_ratio = compute.ram_allocation_ratio
 
-    @set_update_time_on_success
     def consume_from_request(self, spec_obj):
         """Incrementally update host state from an RequestSpec object."""
+
+        @utils.synchronized(self._lock_name)
+        @set_update_time_on_success
+        def _locked(self, spec_obj):
+            # Scheduler API is inherently multi-threaded as every incoming RPC
+            # message will be dispatched in it's own green thread. So the
+            # shared host state should be consumed in a consistent way to make
+            # sure its data is valid under concurrent write operations.
+            self._locked_consume_from_request(spec_obj)
+
+        return _locked(self, spec_obj)
+
+    def _locked_consume_from_request(self, spec_obj):
         disk_mb = (spec_obj.root_gb +
                    spec_obj.ephemeral_gb) * 1024
         ram_mb = spec_obj.memory_mb

nova/scheduler/ironic_host_manager.py

@@ -62,8 +62,7 @@ class IronicNodeState(host_manager.HostState):
 
         self.updated = compute.updated_at
 
-    @host_manager.set_update_time_on_success
-    def consume_from_request(self, spec_obj):
+    def _locked_consume_from_request(self, spec_obj):
         """Consume nodes entire resources regardless of instance request."""
         self.free_ram_mb = 0
         self.free_disk_mb = 0

nova/tests/unit/scheduler/test_host_manager.py

@@ -918,6 +918,8 @@ class HostStateTestCase(test.NoDBTestCase):
         self.assertEqual([], host.pci_stats.pools)
         self.assertEqual(hyper_ver_int, host.hypervisor_version)
 
+    @mock.patch('nova.utils.synchronized',
+                side_effect=lambda a: lambda f: lambda *args: f(*args))
     @mock.patch('nova.virt.hardware.get_host_numa_usage_from_instance')
     @mock.patch('nova.objects.Instance')
     @mock.patch('nova.virt.hardware.numa_fit_instance_to_host')
@@ -925,7 +927,8 @@ class HostStateTestCase(test.NoDBTestCase):
     def test_stat_consumption_from_instance(self, host_topo_mock,
                                             numa_fit_mock,
                                             instance_init_mock,
-                                            numa_usage_mock):
+                                            numa_usage_mock,
+                                            sync_mock):
         fake_numa_topology = objects.InstanceNUMATopology(
             cells=[objects.InstanceNUMACell()])
         fake_host_numa_topology = mock.Mock()
@@ -949,6 +952,7 @@ class HostStateTestCase(test.NoDBTestCase):
                                               limits=None, pci_requests=None,
                                               pci_stats=None)
         numa_usage_mock.assert_called_once_with(host, fake_instance)
+        sync_mock.assert_called_once_with(("fakehost", "fakenode"))
         self.assertEqual(fake_host_numa_topology, host.numa_topology)
         self.assertIsNotNone(host.updated)