metadata: export the vCPU IDs that are pinning on the host CPUs

Add the 'dedicated_cpus' section in metadata API service to tell
the instance CPUs that are set as 'dedicated' CPUs if instance
has any dedicated CPUs. If instance is using a 'shared' CPU
allocation policy, it reports 'None' for no dedicated CPU.

Part of blueprint use-pcpu-and-vcpu-in-one-instance

Change-Id: I3d19195ddefb856c10fa6756dd98850119a4dfcb
Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com>

nova/api/metadata/base.py

@@ -16,6 +16,7 @@
 
 """Instance Metadata information."""
 
+import itertools
 import os
 import posixpath
 
@@ -70,6 +71,7 @@ NEWTON_ONE = '2016-06-30'
 NEWTON_TWO = '2016-10-06'
 OCATA = '2017-02-22'
 ROCKY = '2018-08-27'
+VICTORIA = '2020-10-14'
 
 OPENSTACK_VERSIONS = [
     FOLSOM,
@@ -80,6 +82,7 @@ OPENSTACK_VERSIONS = [
     NEWTON_TWO,
     OCATA,
     ROCKY,
+    VICTORIA,
 ]
 
 VERSION = "version"
@@ -130,6 +133,7 @@ class InstanceMetadata(object):
         instance.ec2_ids
         instance.keypairs
         instance.device_metadata
+        instance.numa_topology
         instance = objects.Instance.obj_from_primitive(
             instance.obj_to_primitive())
 
@@ -358,6 +362,9 @@ class InstanceMetadata(object):
         if self._check_os_version(NEWTON_ONE, version):
             metadata['devices'] = self._get_device_metadata(version)
 
+        if self._check_os_version(VICTORIA, version):
+            metadata['dedicated_cpus'] = self._get_instance_dedicated_cpus()
+
         self.set_mimetype(MIME_TYPE_APPLICATION_JSON)
         return jsonutils.dump_as_bytes(metadata)
 
@@ -435,6 +442,15 @@ class InstanceMetadata(object):
                 device_metadata_list.append(device_metadata)
         return device_metadata_list
 
+    def _get_instance_dedicated_cpus(self):
+        dedicated_cpus = []
+        if self.instance.numa_topology:
+            dedicated_cpus = sorted(list(itertools.chain.from_iterable([
+                cell.pcpuset for cell in self.instance.numa_topology.cells
+            ])))
+
+        return dedicated_cpus
+
     def _handle_content(self, path_tokens):
         if len(path_tokens) == 1:
             raise KeyError("no listing for %s" % "/".join(path_tokens))
@@ -658,7 +674,7 @@ def get_metadata_by_instance_id(instance_id, address, ctxt=None):
     attrs = ['ec2_ids', 'flavor', 'info_cache',
              'metadata', 'system_metadata',
              'security_groups', 'keypairs',
-             'device_metadata']
+             'device_metadata', 'numa_topology']
 
     if CONF.api.local_metadata_per_cell:
         instance = objects.Instance.get_by_uuid(ctxt, instance_id,

nova/tests/unit/test_metadata.py

@@ -49,6 +49,7 @@ from nova import exception
 from nova.network import model as network_model
 from nova.network import neutron as neutronapi
 from nova import objects
+from nova.objects import instance_numa as numa
 from nova.objects import virt_device_metadata as metadata_obj
 from nova import test
 from nova.tests.unit.api.openstack import fakes
@@ -419,6 +420,8 @@ class MetadataTestCase(test.TestCase):
                 inst.device_metadata = original_device_meta
             elif attrname == 'ec2_ids':
                 inst.ec2_ids = objects.EC2Ids()
+            elif attrname == 'numa_topology':
+                inst.numa_topology = None
             else:
                 self.fail('Unexpected instance lazy-load: %s' % attrname)
 
@@ -428,9 +431,11 @@ class MetadataTestCase(test.TestCase):
                 side_effect=fake_obj_load_attr) as mock_obj_load_attr:
             md = fake_InstanceMetadata(self, inst)
         self.assertFalse(hasattr(md.instance, '_will_not_pass'))
-        self.assertEqual(2, mock_obj_load_attr.call_count)
+        self.assertEqual(3, mock_obj_load_attr.call_count)
         mock_obj_load_attr.assert_has_calls(
-            [mock.call('device_metadata'), mock.call('ec2_ids')],
+            [mock.call('device_metadata'),
+             mock.call('ec2_ids'),
+             mock.call('numa_topology')],
             any_order=True)
         self.assertIs(original_device_meta, inst.device_metadata)
 
@@ -502,6 +507,11 @@ class MetadataTestCase(test.TestCase):
             'openstack/2018-08-27/vendor_data.json',
             'openstack/2018-08-27/network_data.json',
             'openstack/2018-08-27/vendor_data2.json',
+            'openstack/2020-10-14/meta_data.json',
+            'openstack/2020-10-14/user_data',
+            'openstack/2020-10-14/vendor_data.json',
+            'openstack/2020-10-14/network_data.json',
+            'openstack/2020-10-14/vendor_data2.json',
             'openstack/latest/meta_data.json',
             'openstack/latest/user_data',
             'openstack/latest/vendor_data.json',
@@ -588,6 +598,8 @@ class MetadataTestCase(test.TestCase):
             expose_trusted = md._check_os_version(base.ROCKY, os_version)
             expected_metadata['devices'] = fake_metadata_dicts(
                 True, expose_trusted)
+        if md._check_os_version(base.VICTORIA, os_version):
+            expected_metadata['dedicated_cpus'] = []
         md._metadata_as_json(os_version, 'non useless path parameter')
         self.assertEqual(md.md_mimetype, base.MIME_TYPE_APPLICATION_JSON)
         mock_json_dump_as_bytes.assert_called_once_with(expected_metadata)
@@ -611,6 +623,30 @@ class MetadataTestCase(test.TestCase):
         self.assertNotIn('keys', meta)
         self.assertNotIn('public_keys', meta)
 
+    @mock.patch.object(objects.Instance, 'get_by_uuid')
+    def test_metadata_as_json_numatopology(self, mock_inst_get_by_uuid):
+        """Ensure instance dedicated CPUs is properly listed."""
+        fake_topo = numa.InstanceNUMATopology(cells=[
+            numa.InstanceNUMACell(id=0, memory=1024, pagesize=4,
+                                  cpuset=set([2]), pcpuset=set([0, 1])),
+            numa.InstanceNUMACell(id=1, memory=2048, pagesize=4,
+                                  cpuset=set([5]), pcpuset=set([3, 4])),
+        ])
+        instance = self.instance.obj_clone()
+
+        mock_inst_get_by_uuid.return_value = instance
+        md = fake_InstanceMetadata(self, instance)
+        meta = md._metadata_as_json(base.OPENSTACK_VERSIONS[-1], path=None)
+        meta = jsonutils.loads(meta)
+        self.assertEqual([], meta['dedicated_cpus'])
+
+        instance.numa_topology = fake_topo
+        mock_inst_get_by_uuid.return_value = instance
+        md = fake_InstanceMetadata(self, instance)
+        meta = md._metadata_as_json(base.OPENSTACK_VERSIONS[-1], path=None)
+        meta = jsonutils.loads(meta)
+        self.assertEqual([0, 1, 3, 4], meta['dedicated_cpus'])
+
 
 class OpenStackMetadataTestCase(test.TestCase):
     def setUp(self):
@@ -1702,7 +1738,7 @@ class MetadataHandlerTestCase(test.TestCase):
         mock_uuid.assert_called_once_with(ctxt, 'foo',
             expected_attrs=['ec2_ids', 'flavor', 'info_cache', 'metadata',
                             'system_metadata', 'security_groups', 'keypairs',
-                            'device_metadata'])
+                            'device_metadata', 'numa_topology'])
         imd.assert_called_once_with(inst, 'bar')
 
     @mock.patch.object(context, 'get_admin_context')
@@ -1720,7 +1756,7 @@ class MetadataHandlerTestCase(test.TestCase):
         mock_uuid.assert_called_once_with(mock_context.return_value, 'foo',
             expected_attrs=['ec2_ids', 'flavor', 'info_cache', 'metadata',
                             'system_metadata', 'security_groups', 'keypairs',
-                            'device_metadata'])
+                            'device_metadata', 'numa_topology'])
         imd.assert_called_once_with(inst, 'bar')
 
     @mock.patch.object(objects.Instance, 'get_by_uuid')