From 3978b99d8f37643a3b32f6b7ead9274c454095a1 Mon Sep 17 00:00:00 2001 From: LuyaoZhong Date: Wed, 11 Sep 2019 03:50:21 +0000 Subject: [PATCH] libvirt: Enable driver discovering PMEM namespaces Add related functions to enable libvirt driver to get PMEM namespaces(VPMEMs) info from host and configuration. note: PMEM namespaces configuration is not supported yet, so libvirt driver will not get any PMEM namespaces from host. Change-Id: Idd49b0c70caedfcd42420ffa2ac926a6087d406e Partially-Implements: blueprint virtual-persistent-memory Co-Authored-By: He Jie Xu --- nova/exception.py | 9 ++ nova/objects/resource.py | 25 +++++ nova/privsep/libvirt.py | 7 ++ nova/tests/unit/objects/test_objects.py | 4 +- nova/tests/unit/objects/test_resource.py | 32 ++++++ nova/tests/unit/virt/libvirt/test_driver.py | 105 ++++++++++++++++++++ nova/virt/libvirt/driver.py | 90 +++++++++++++++++ 7 files changed, 271 insertions(+), 1 deletion(-) diff --git a/nova/exception.py b/nova/exception.py index 6728176601cb..422e5f983b12 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -2523,3 +2523,12 @@ class UnableToRollbackPortUpdates(HealPortAllocationException): class AssignedResourceNotFound(NovaException): msg_fmt = _("Assigned resources not found: %(reason)s") + + +class PMEMNamespaceConfigInvalid(NovaException): + msg_fmt = _("The pmem_namespaces configuration is invalid: %(reason)s, " + "please check your conf file. ") + + +class GetPMEMNamespaceFailed(NovaException): + msg_fmt = _("Get PMEM namespaces on host failed: %(reason)s.") diff --git a/nova/objects/resource.py b/nova/objects/resource.py index bd1deaa1c1a6..6ac97a57855e 100644 --- a/nova/objects/resource.py +++ b/nova/objects/resource.py @@ -83,3 +83,28 @@ class ResourceList(base.ObjectListBase, base.NovaObject): primitive = jsonutils.loads(db_extra['resources']) resources = cls.obj_from_primitive(primitive) return resources + + +@base.NovaObjectRegistry.register +class LibvirtVPMEMDevice(ResourceMetadata): + # Version 1.0: Initial version + VERSION = "1.0" + + fields = { + # This is configured in file, used to generate resource class name + # CUSTOM_PMEM_NAMESPACE_$LABEL + 'label': fields.StringField(), + # Backend pmem namespace's name + 'name': fields.StringField(), + # Backend pmem namespace's size + 'size': fields.IntegerField(), + # Backend device path + 'devpath': fields.StringField(), + # Backend pmem namespace's alignment + 'align': fields.IntegerField(), + } + + def __hash__(self): + # Be sure all fields are set before using hash method + return hash((self.label, self.name, self.size, + self.devpath, self.align)) diff --git a/nova/privsep/libvirt.py b/nova/privsep/libvirt.py index b186bef6f428..f4e59acd6cd9 100644 --- a/nova/privsep/libvirt.py +++ b/nova/privsep/libvirt.py @@ -246,3 +246,10 @@ def unprivileged_umount(mnt_base): """Unmount volume""" umnt_cmd = ['umount', mnt_base] return processutils.execute(*umnt_cmd) + + +@nova.privsep.sys_admin_pctxt.entrypoint +def get_pmem_namespaces(): + ndctl_cmd = ['ndctl', 'list', '-X'] + nss_info = processutils.execute(*ndctl_cmd)[0] + return nss_info diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 4e473459f142..c02cfa512b59 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1149,7 +1149,9 @@ object_data = { 'VirtualInterfaceList': '1.0-9750e2074437b3077e46359102779fc6', 'VolumeUsage': '1.0-6c8190c46ce1469bb3286a1f21c2e475', 'XenDeviceBus': '1.0-272a4f899b24e31e42b2b9a7ed7e9194', - 'XenapiLiveMigrateData': '1.4-7dc9417e921b2953faa6751f18785f3f' + 'XenapiLiveMigrateData': '1.4-7dc9417e921b2953faa6751f18785f3f', + # TODO(efried): re-alphabetize this + 'LibvirtVPMEMDevice': '1.0-17ffaf47585199eeb9a2b83d6bde069f', } diff --git a/nova/tests/unit/objects/test_resource.py b/nova/tests/unit/objects/test_resource.py index 7eba07fcfab3..966d9c6d207c 100644 --- a/nova/tests/unit/objects/test_resource.py +++ b/nova/tests/unit/objects/test_resource.py @@ -25,12 +25,28 @@ fake_resources = resource.ResourceList(objects=[ resource.Resource(provider_uuid=uuids.rp, resource_class='CUSTOM_RESOURCE', identifier='bar')]) +fake_vpmems = [ + resource.LibvirtVPMEMDevice( + label='4GB', name='ns_0', devpath='/dev/dax0.0', + size=4292870144, align=2097152), + resource.LibvirtVPMEMDevice( + label='4GB', name='ns_1', devpath='/dev/dax0.0', + size=4292870144, align=2097152)] + fake_instance_extras = { 'resources': jsonutils.dumps(fake_resources.obj_to_primitive()) } class TestResourceObject(test_objects._LocalTest): + def _create_resource(self, metadata=None): + fake_resource = resource.Resource(provider_uuid=uuids.rp, + resource_class='bar', + identifier='foo') + if metadata: + fake_resource.metadata = metadata + return fake_resource + def _test_set_malformed_resource_class(self, rc): try: resource.Resource(provider_uuid=uuids.rp, @@ -69,6 +85,22 @@ class TestResourceObject(test_objects._LocalTest): def test_not_equal_without_matadata(self): self.assertNotEqual(fake_resources[0], fake_resources[1]) + def test_equal_with_vpmem_metadata(self): + resource_0 = self._create_resource(metadata=fake_vpmems[0]) + resource_1 = self._create_resource(metadata=fake_vpmems[0]) + self.assertEqual(resource_0, resource_1) + + def test_not_equal_with_vpmem_metadata(self): + resource_0 = self._create_resource(metadata=fake_vpmems[0]) + resource_1 = self._create_resource(metadata=fake_vpmems[1]) + self.assertNotEqual(resource_0, resource_1) + + def test_not_equal_with_and_without_metadata(self): + # one resource has metadata, another one has not metadata + resource_0 = self._create_resource(metadata=fake_vpmems[0]) + resource_1 = self._create_resource() + self.assertNotEqual(resource_0, resource_1) + class _TestResourceListObject(object): @mock.patch('nova.db.api.instance_extra_get_by_instance_uuid') diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index a7c4584ef4b3..3b058c90b163 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -24257,3 +24257,108 @@ class TestLibvirtSEVSupported(TestLibvirtSEV): self.flags(num_memory_encrypted_guests=0, group='libvirt') self.driver._host._set_amd_sev_support() self.assertEqual(0, self.driver._get_memory_encrypted_slots()) + + +class LibvirtPMEMNamespaceTests(test.NoDBTestCase): + + def setUp(self): + super(LibvirtPMEMNamespaceTests, self).setUp() + self.useFixture(fakelibvirt.FakeLibvirtFixture()) + self.vpmem_0 = objects.LibvirtVPMEMDevice( + label='4GB', + name='ns_0', devpath='/dev/dax0.0', + size=4292870144, align=2097152) + self.vpmem_1 = objects.LibvirtVPMEMDevice( + label='SMALL', + name='ns_1', devpath='/dev/dax0.1', + size=17177772032, align=2097152) + self.vpmem_2 = objects.LibvirtVPMEMDevice( + label='SMALL', + name='ns_2', devpath='/dev/dax0.2', + size=17177772032, align=2097152) + + self.pmem_namespaces = ''' + [{"dev":"namespace0.0", + "mode":"devdax", + "map":"mem", + "size":4292870144, + "uuid":"24ffd5e4-2b39-4f28-88b3-d6dc1ec44863", + "daxregion":{"id": 0, "size": 4292870144,"align": 2097152, + "devices":[{"chardev":"dax0.0", + "size":4292870144}]}, + "name":"ns_0", + "numa_node":0}, + {"dev":"namespace0.1", + "mode":"devdax", + "map":"mem", + "size":17177772032, + "uuid":"ac64fe52-de38-465b-b32b-947a6773ac66", + "daxregion":{"id": 0, "size": 17177772032,"align": 2097152, + "devices":[{"chardev":"dax0.1", + "size":17177772032}]}, + "name":"ns_1", + "numa_node":0}, + {"dev":"namespace0.2", + "mode":"devdax", + "map":"mem", + "size":17177772032, + "uuid":"48189df5-2599-4001-8696-c260f7460381", + "daxregion":{"id": 0, "size": 17177772032,"align": 2097152, + "devices":[{"chardev":"dax0.2", + "size":17177772032}]}, + "name":"ns_2", + "numa_node":0}]''' + + @mock.patch('nova.virt.libvirt.host.Host.has_min_version', + new=mock.Mock(return_value=True)) + @mock.patch('nova.privsep.libvirt.get_pmem_namespaces') + def test_discover_vpmems(self, mock_get): + mock_get.return_value = self.pmem_namespaces + vpmem_conf = ["4GB:ns_0", "SMALL:ns_1|ns_2"] + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + vpmems_by_name, vpmems_by_rc = drvr._discover_vpmems( + vpmem_conf=vpmem_conf) + expected_vpmems_by_name = { + 'ns_0': self.vpmem_0, + 'ns_1': self.vpmem_1, + 'ns_2': self.vpmem_2} + expected_vpmems_by_rc = { + 'CUSTOM_PMEM_NAMESPACE_4GB': [self.vpmem_0], + 'CUSTOM_PMEM_NAMESPACE_SMALL': [self.vpmem_1, self.vpmem_2] + } + for name, vpmem in expected_vpmems_by_name.items(): + self.assertEqual(vpmem.devpath, vpmems_by_name[name].devpath) + self.assertEqual(vpmem.size, vpmems_by_name[name].size) + for rc in expected_vpmems_by_rc.keys(): + self.assertEqual(len(expected_vpmems_by_rc[rc]), + len(vpmems_by_rc[rc])) + + @mock.patch('nova.virt.libvirt.host.Host.has_min_version', + new=mock.Mock(return_value=True)) + @mock.patch('nova.privsep.libvirt.get_pmem_namespaces') + def test_vpmems_not_on_host(self, mock_get): + mock_get.return_value = self.pmem_namespaces + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + vpmem_conf = ["4GB:ns_3"] + self.assertRaises(exception.PMEMNamespaceConfigInvalid, + drvr._discover_vpmems, vpmem_conf) + + @mock.patch('nova.virt.libvirt.host.Host.has_min_version', + new=mock.Mock(return_value=True)) + @mock.patch('nova.privsep.libvirt.get_pmem_namespaces') + def test_vpmems_invalid_format(self, mock_get): + mock_get.return_value = self.pmem_namespaces + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + vpmem_conf = ["ns_0", "ns_1", "ns_2"] + self.assertRaises(exception.PMEMNamespaceConfigInvalid, + drvr._discover_vpmems, vpmem_conf) + + @mock.patch('nova.virt.libvirt.host.Host.has_min_version', + new=mock.Mock(return_value=True)) + @mock.patch('nova.privsep.libvirt.get_pmem_namespaces') + def test_vpmems_duplicated_config(self, mock_get): + mock_get.return_value = self.pmem_namespaces + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + vpmem_conf = ["4GB:ns_0", "SMALL:ns_0"] + self.assertRaises(exception.PMEMNamespaceConfigInvalid, + drvr._discover_vpmems, vpmem_conf) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 9647dcdc87c4..ec3f38c4433d 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -295,6 +295,10 @@ MIN_LIBVIRT_VIDEO_MODEL_VERSIONS = { fields.VideoModel.NONE: (4, 6, 0), } +# Persistent Memory (PMEM/NVDIMM) Device Support +MIN_LIBVIRT_PMEM_SUPPORT = (5, 0, 0) +MIN_QEMU_PMEM_SUPPORT = (3, 1, 0) + class LibvirtDriver(driver.ComputeDriver): def __init__(self, virtapi, read_only=False): @@ -430,6 +434,92 @@ class LibvirtDriver(driver.ComputeDriver): self.cpu_models_mapping = {} self.cpu_model_flag_mapping = {} + self._vpmems_by_name, self._vpmems_by_rc = self._discover_vpmems() + + def _discover_vpmems(self, vpmem_conf=None): + """Discover vpmems on host and configuration. + + :param vpmem_conf: pmem namespaces configuration from CONF + :returns: a dict of vpmem keyed by name, and + a dict of vpmem list keyed by resource class + :raises: exception.InvalidConfiguration if Libvirt or QEMU version + does not meet requirement. + """ + if not vpmem_conf: + return {}, {} + + if not self._host.has_min_version(lv_ver=MIN_LIBVIRT_PMEM_SUPPORT, + hv_ver=MIN_QEMU_PMEM_SUPPORT): + raise exception.InvalidConfiguration( + _('Nova requires QEMU version %(qemu)s or greater ' + 'and Libvirt version %(libvirt)s or greater ' + 'for NVDIMM (Persistent Memory) support.') % { + 'qemu': libvirt_utils.version_to_string( + MIN_QEMU_PMEM_SUPPORT), + 'libvirt': libvirt_utils.version_to_string( + MIN_LIBVIRT_PMEM_SUPPORT)}) + + # vpmem keyed by name {name: objects.LibvirtVPMEMDevice,...} + vpmems_by_name = {} + # vpmem list keyed by resource class + # {'RC_0': [objects.LibvirtVPMEMDevice, ...], 'RC_1': [...]} + vpmems_by_rc = collections.defaultdict(list) + + vpmems_host = self._get_vpmems_on_host() + for ns_conf in vpmem_conf: + try: + ns_label, ns_names = ns_conf.split(":", 1) + except ValueError: + reason = _("The configuration doesn't follow the format") + raise exception.PMEMNamespaceConfigInvalid( + reason=reason) + ns_names = ns_names.split("|") + for ns_name in ns_names: + if ns_name not in vpmems_host: + reason = _("The PMEM namespace %s isn't on host") % ns_name + raise exception.PMEMNamespaceConfigInvalid( + reason=reason) + if ns_name in vpmems_by_name: + reason = (_("Duplicated PMEM namespace %s configured") % + ns_name) + raise exception.PMEMNamespaceConfigInvalid( + reason=reason) + pmem_ns_updated = vpmems_host[ns_name] + pmem_ns_updated.label = ns_label + vpmems_by_name[ns_name] = pmem_ns_updated + rc = orc.normalize_name( + "PMEM_NAMESPACE_%s" % ns_label) + vpmems_by_rc[rc].append(pmem_ns_updated) + + return vpmems_by_name, vpmems_by_rc + + def _get_vpmems_on_host(self): + """Get PMEM namespaces on host using ndctl utility.""" + try: + output = nova.privsep.libvirt.get_pmem_namespaces() + except Exception as e: + reason = _("Get PMEM namespaces by ndctl utility, " + "please ensure ndctl is installed: %s") % e + raise exception.GetPMEMNamespacesFailed(reason=reason) + + if not output: + return {} + namespaces = jsonutils.loads(output) + vpmems_host = {} # keyed by namespace name + for ns in namespaces: + # store namespace info parsed from ndctl utility return + if not ns.get('name'): + # The name is used to identify namespaces, it's optional + # config when creating namespace. If an namespace don't have + # name, it can not be used by Nova, we will skip it. + continue + vpmems_host[ns['name']] = objects.LibvirtVPMEMDevice( + name=ns['name'], + devpath= '/dev/' + ns['daxregion']['devices'][0]['chardev'], + size=ns['size'], + align=ns['daxregion']['align']) + return vpmems_host + def _get_volume_drivers(self): driver_registry = dict()