From 0e8fdcdf63c9fc8687a9c468a7b380f8aceead7e Mon Sep 17 00:00:00 2001 From: Sridhar Venkat Date: Sun, 17 Jul 2016 23:14:18 -0400 Subject: [PATCH] VIF driver implementation for SR-IOV PvmVnicSriovVifDriver, a VIF driver for vNICs backed by SR-IOV VFs. The main plug method intercepts incoming vif requests and branches off to this new vif driver if the vif type is 'pvm_sriov'. The driver "plugs" a new vNIC into the instance by: - Building a new pypowervm.wrappers.iocard.VNIC object. - Assigning backing devices from the pool of physical ports labeled according to the physical network name. - Creating the VNIC on the PowerVM REST server as a child of the LPAR corresponding to the instance. The unplug method locates the appropriate VNIC according to its MAC address and deletes it from the PowerVM REST server. Implements: bp/powervm-sriov-nova Change-Id: I9b707e8c12c2124001089ac1a0f2ef28e5cc7478 --- README.rst | 4 +- doc/source/devref/usage.rst | 10 +- .../tests/virt/powervm/test_driver.py | 14 +- nova_powervm/tests/virt/powervm/test_host.py | 23 +++- nova_powervm/tests/virt/powervm/test_vif.py | 129 +++++++++++++++++- nova_powervm/virt/powervm/driver.py | 7 +- nova_powervm/virt/powervm/host.py | 22 +++ nova_powervm/virt/powervm/vif.py | 76 ++++++++++- 8 files changed, 256 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index 96534e82..ba27bf32 100644 --- a/README.rst +++ b/README.rst @@ -197,8 +197,8 @@ Shared Storage Pool (a PowerVM clustered file system) is supported. For volume attachments, the driver supports Cinder-based attachments via protocols supported by the hypervisor (e.g. Fibre Channel). -For networking, the networking-powervm project provides a Neutron ML2 Agent. -The agent provides the necessary configuration on the Virtual I/O Server for +For networking, the networking-powervm project provides Neutron ML2 Agents. +The agents provide the necessary configuration on the Virtual I/O Server for networking. The PowerVM Nova driver code creates the VIF for the client VM, but the Neutron agent creates the VIF for VLANs. diff --git a/doc/source/devref/usage.rst b/doc/source/devref/usage.rst index b9b07065..a9f35847 100644 --- a/doc/source/devref/usage.rst +++ b/doc/source/devref/usage.rst @@ -34,8 +34,14 @@ configured ahead of time. Configuration File Options -------------------------- -The standard nova configuration options are supported. Additionally, a -``[powervm]`` section is used to provide additional customization to the driver. +The standard nova configuration options are supported. In particular, to use +PowerVM SR-IOV vNIC for networking, the ``pci_passthrough_whitelist`` option +must be set. See the `networking-powervm usage devref`_ for details. + +.. _`networking-powervm usage devref`: http://networking-powervm.readthedocs.io/en/latest/devref/usage.html + +Additionally, a ``[powervm]`` section is used to provide additional +customization to the driver. By default, no additional inputs are needed. The base configuration allows for a Nova driver to support ephemeral disks to a local volume group (only diff --git a/nova_powervm/tests/virt/powervm/test_driver.py b/nova_powervm/tests/virt/powervm/test_driver.py index 047a11d4..aca4596a 100644 --- a/nova_powervm/tests/virt/powervm/test_driver.py +++ b/nova_powervm/tests/virt/powervm/test_driver.py @@ -1501,13 +1501,13 @@ class TestPowerVMDriver(test.TestCase): 'op': 'fake_op'} mock_log.info.assert_called_with(entry, msg_dict) - def test_host_resources(self): - # Mock methods not currently under test - with mock.patch.object(self.apt, 'read') as mock_read: - mock_read.return_value = None - # Run the actual test - stats = self.drv.get_available_resource('nodename') - self.assertIsNotNone(stats) + @mock.patch('pypowervm.wrappers.managed_system.System.get', + new=mock.Mock(return_value=[mock.Mock()])) + @mock.patch('nova_powervm.virt.powervm.host.build_host_resource_from_ms') + def test_host_resources(self, mock_bhrfm): + mock_bhrfm.return_value = {} + stats = self.drv.get_available_resource('nodename') + self.assertIsNotNone(stats) # Check for the presence of fields added to host stats fields = ('local_gb', 'local_gb_used') diff --git a/nova_powervm/tests/virt/powervm/test_host.py b/nova_powervm/tests/virt/powervm/test_host.py index 71269917..e096cc57 100644 --- a/nova_powervm/tests/virt/powervm/test_host.py +++ b/nova_powervm/tests/virt/powervm/test_host.py @@ -19,6 +19,7 @@ import mock import logging from nova import test +from oslo_serialization import jsonutils import pypowervm.tests.test_fixtures as pvm_fx from nova_powervm.virt.powervm import host as pvm_host @@ -30,13 +31,17 @@ logging.basicConfig() class TestPowerVMHost(test.TestCase): def test_host_resources(self): # Create objects to test with + sriov_adaps = [mock.Mock(phys_ports=[mock.Mock(label='foo'), + mock.Mock(label='')]), + mock.Mock(phys_ports=[mock.Mock(label='bar')])] ms_wrapper = mock.MagicMock( proc_units_configurable=500, proc_units_avail=500, memory_configurable=5242880, memory_free=5242752, mtms=mock.MagicMock(mtms_str='8484923A123456'), - memory_region_size='big') + memory_region_size='big', + asio_config=mock.Mock(sriov_adapters=sriov_adaps)) # Run the actual test stats = pvm_host.build_host_resource_from_ms(ms_wrapper) @@ -47,7 +52,7 @@ class TestPowerVMHost(test.TestCase): ('memory_mb', 5242880), ('memory_mb_used', 128), 'hypervisor_type', 'hypervisor_version', 'hypervisor_hostname', 'cpu_info', - 'supported_instances', 'stats') + 'supported_instances', 'stats', 'pci_passthrough_devices') for fld in fields: if isinstance(fld, tuple): value = stats.get(fld[0], None) @@ -64,6 +69,20 @@ class TestPowerVMHost(test.TestCase): else: value = stats['stats'].get(stat, None) self.assertIsNotNone(value) + # pci_passthrough_devices. Parse json - entries can be in any order. + ppdstr = stats['pci_passthrough_devices'] + ppdlist = jsonutils.loads(ppdstr) + self.assertEqual({'foo', 'bar', 'default'}, {ppd['physical_network'] + for ppd in ppdlist}) + self.assertEqual({'foo', 'bar', 'default'}, {ppd['label'] + for ppd in ppdlist}) + for ppd in ppdlist: + self.assertEqual('type-PCI', ppd['dev_type']) + self.assertEqual('*:*:*.*', ppd['address']) + self.assertEqual('*:*:*.*', ppd['parent_addr']) + self.assertEqual('*', ppd['vendor_id']) + self.assertEqual('*', ppd['product_id']) + self.assertEqual(1, ppd['numa_node']) class TestHostCPUStats(test.TestCase): diff --git a/nova_powervm/tests/virt/powervm/test_vif.py b/nova_powervm/tests/virt/powervm/test_vif.py index 7d5f8e6e..0a334e2f 100644 --- a/nova_powervm/tests/virt/powervm/test_vif.py +++ b/nova_powervm/tests/virt/powervm/test_vif.py @@ -15,7 +15,6 @@ # under the License. import mock -from mock import call from nova import exception from nova import test @@ -111,6 +110,11 @@ class TestVifFunctions(test.TestCase): {'type': 'pvm_sea'}), vif.PvmSeaVifDriver) + self.assertIsInstance( + vif._build_vif_driver(self.adpt, 'host_uuid', mock_inst, + {'type': 'pvm_sriov'}), + vif.PvmVnicSriovVifDriver) + # Test raises exception for no type self.assertRaises(exception.VirtualInterfacePlugException, vif._build_vif_driver, self.adpt, 'host_uuid', @@ -122,6 +126,121 @@ class TestVifFunctions(test.TestCase): mock_inst, {'type': 'bad'}) +class TestVifSriovDriver(test.TestCase): + + def setUp(self): + super(TestVifSriovDriver, self).setUp() + + self.adpt = self.useFixture(pvm_fx.AdapterFx()).adpt + self.inst = mock.MagicMock() + self.drv = vif.PvmVnicSriovVifDriver(self.adpt, 'host_uuid', self.inst) + + def test_plug_no_pports(self): + """Raise when plug is called with a network with no physical ports.""" + self.assertRaises(exception.VirtualInterfacePlugException, + self.drv.plug, FakeDirectVif('net2', pports=[]), 1) + + @mock.patch('pypowervm.util.sanitize_mac_for_api') + @mock.patch('pypowervm.wrappers.iocard.VNIC.bld') + @mock.patch('pypowervm.tasks.sriov.set_vnic_back_devs') + @mock.patch('nova_powervm.virt.powervm.vm.get_pvm_uuid') + def test_plug(self, mock_pvm_uuid, mock_back_devs, mock_vnic_bld, + mock_san_mac): + slot = 10 + pports = ['port1', 'port2'] + self.drv.plug(FakeDirectVif('default', pports=pports), + slot) + mock_san_mac.assert_called_once_with('ab:ab:ab:ab:ab:ab') + mock_vnic_bld.assert_called_once_with( + self.drv.adapter, 79, slot_num=slot, + mac_addr=mock_san_mac.return_value, allowed_vlans='NONE', + allowed_macs='NONE') + mock_back_devs.assert_called_once_with( + mock_vnic_bld.return_value, ['port1', 'port2'], min_redundancy=3, + max_redundancy=3, capacity=None) + mock_pvm_uuid.assert_called_once_with(self.drv.instance) + mock_vnic_bld.return_value.create.assert_called_once_with( + parent_type=pvm_lpar.LPAR, parent_uuid=mock_pvm_uuid.return_value) + + # Now with redundancy/capacity values from binding:profile + mock_san_mac.reset_mock() + mock_vnic_bld.reset_mock() + mock_back_devs.reset_mock() + mock_pvm_uuid.reset_mock() + self.drv.plug(FakeDirectVif('default', pports=pports, cap=0.08), + slot) + mock_san_mac.assert_called_once_with('ab:ab:ab:ab:ab:ab') + mock_vnic_bld.assert_called_once_with( + self.drv.adapter, 79, slot_num=slot, + mac_addr=mock_san_mac.return_value, allowed_vlans='NONE', + allowed_macs='NONE') + mock_back_devs.assert_called_once_with( + mock_vnic_bld.return_value, ['port1', 'port2'], min_redundancy=3, + max_redundancy=3, capacity=0.08) + mock_pvm_uuid.assert_called_once_with(self.drv.instance) + mock_vnic_bld.return_value.create.assert_called_once_with( + parent_type=pvm_lpar.LPAR, parent_uuid=mock_pvm_uuid.return_value) + + # No-op with new_vif=False + mock_san_mac.reset_mock() + mock_vnic_bld.reset_mock() + mock_back_devs.reset_mock() + mock_pvm_uuid.reset_mock() + self.assertIsNone(self.drv.plug( + FakeDirectVif('default', pports=pports), slot, new_vif=False)) + mock_san_mac.assert_not_called() + mock_vnic_bld.assert_not_called() + mock_back_devs.assert_not_called() + mock_pvm_uuid.assert_not_called() + + @mock.patch('pypowervm.wrappers.iocard.VNIC.search') + @mock.patch('nova_powervm.virt.powervm.vm.get_pvm_uuid') + @mock.patch('pypowervm.util.sanitize_mac_for_api') + def test_unplug(self, mock_san_mac, mock_pvm_uuid, mock_find): + fvif = FakeDirectVif('default') + self.assertEqual(mock_find.return_value, self.drv.unplug(fvif)) + mock_find.assert_called_once_with( + self.drv.adapter, parent_type=pvm_lpar.LPAR, + parent_uuid=mock_pvm_uuid.return_value, + mac=mock_san_mac.return_value, one_result=True) + mock_pvm_uuid.assert_called_once_with(self.inst) + mock_san_mac.assert_called_once_with(fvif['address']) + mock_find.return_value.delete.assert_called_once_with() + + # Not found + mock_find.reset_mock() + mock_pvm_uuid.reset_mock() + mock_san_mac.reset_mock() + mock_find.return_value = None + self.assertIsNone(self.drv.unplug(fvif)) + mock_find.assert_called_once_with( + self.drv.adapter, parent_type=pvm_lpar.LPAR, + parent_uuid=mock_pvm_uuid.return_value, + mac=mock_san_mac.return_value, one_result=True) + mock_pvm_uuid.assert_called_once_with(self.inst) + mock_san_mac.assert_called_once_with(fvif['address']) + + +class FakeDirectVif(dict): + def __init__(self, physnet, pports=None, cap=None): + self._physnet = physnet + super(FakeDirectVif, self).__init__( + network={'id': 'net_id'}, + address='ab:ab:ab:ab:ab:ab', + details={ + 'vlan': '79', + 'physical_ports': [], + 'redundancy': 3}, + profile={}) + if pports is not None: + self['details']['physical_ports'] = pports + if cap is not None: + self['profile']['capacity'] = cap + + def get_physical_network(self): + return self._physnet + + class TestVifSeaDriver(test.TestCase): def setUp(self): @@ -289,10 +408,10 @@ class TestVifLBDriver(test.TestCase): self.assertTrue(mock_cna.delete.called) # Validate the execute - call_ip = call('ip', 'link', 'set', 'fake_dev', 'down', - run_as_root=True) - call_delif = call('brctl', 'delif', 'br0', 'fake_dev', - run_as_root=True) + call_ip = mock.call('ip', 'link', 'set', 'fake_dev', 'down', + run_as_root=True) + call_delif = mock.call('brctl', 'delif', 'br0', 'fake_dev', + run_as_root=True) mock_exec.assert_has_calls([call_ip, call_delif]) # Test unplug for the case where tap device diff --git a/nova_powervm/virt/powervm/driver.py b/nova_powervm/virt/powervm/driver.py index ef78b7a1..2c300052 100644 --- a/nova_powervm/virt/powervm/driver.py +++ b/nova_powervm/virt/powervm/driver.py @@ -992,11 +992,8 @@ class PowerVMDriver(driver.ComputeDriver): a driver that manages only one node can safely ignore this :return: Dictionary describing resources """ - - resp = self.adapter.read(pvm_ms.System.schema_type, - root_id=self.host_uuid) - if resp: - self.host_wrapper = pvm_ms.System.wrap(resp.entry) + # Do this here so it refreshes each time this method is called. + self.host_wrapper = pvm_ms.System.get(self.adapter)[0] # Get host information data = pvm_host.build_host_resource_from_ms(self.host_wrapper) diff --git a/nova_powervm/virt/powervm/host.py b/nova_powervm/virt/powervm/host.py index c96b93a1..d1bc89f4 100644 --- a/nova_powervm/virt/powervm/host.py +++ b/nova_powervm/virt/powervm/host.py @@ -18,6 +18,7 @@ import math from nova.compute import arch from nova.compute import hv_type from nova.compute import vm_mode +from nova.objects import fields from oslo_concurrency import lockutils from oslo_log import log as logging from oslo_serialization import jsonutils @@ -83,6 +84,27 @@ def build_host_resource_from_ms(ms_wrapper): } data["stats"] = stats + # Produce SR-IOV PCI data. Devices are validated by virtue of the network + # name associated with their label, which must be cleared via an entry in + # the pci_passthrough_whitelist in the nova.conf. + nets = {pport.label or 'default' + for sriov in ms_wrapper.asio_config.sriov_adapters + for pport in sriov.phys_ports} + pci_devs = [] + for net in nets: + # These fields are all required in order to satisfy a Claim. Their + # values are as general as possible. + LOG.debug("Registering SR-IOV passthrough for network '%s'", net) + pci_devs.append({"physical_network": net, + "label": net, + "dev_type": fields.PciDeviceType.STANDARD, + "address": "*:*:*.*", + "parent_addr": "*:*:*.*", + "vendor_id": "*", + "product_id": "*", + "numa_node": 1}) + data["pci_passthrough_devices"] = jsonutils.dumps(pci_devs) + return data diff --git a/nova_powervm/virt/powervm/vif.py b/nova_powervm/virt/powervm/vif.py index c7b974ca..3b07d2f1 100644 --- a/nova_powervm/virt/powervm/vif.py +++ b/nova_powervm/virt/powervm/vif.py @@ -27,7 +27,9 @@ from oslo_config import cfg from oslo_utils import importutils from pypowervm.tasks import cna as pvm_cna from pypowervm.tasks import partition as pvm_par +from pypowervm.tasks import sriov as sriovtask from pypowervm import util as pvm_util +from pypowervm.wrappers import iocard as pvm_card from pypowervm.wrappers import logical_partition as pvm_lpar from pypowervm.wrappers import managed_system as pvm_ms from pypowervm.wrappers import network as pvm_net @@ -45,7 +47,9 @@ SECURE_RMC_VLAN = 4094 VIF_MAPPING = {'pvm_sea': 'nova_powervm.virt.powervm.vif.PvmSeaVifDriver', 'ovs': 'nova_powervm.virt.powervm.vif.PvmOvsVifDriver', - 'bridge': 'nova_powervm.virt.powervm.vif.PvmLBVifDriver'} + 'bridge': 'nova_powervm.virt.powervm.vif.PvmLBVifDriver', + 'pvm_sriov': + 'nova_powervm.virt.powervm.vif.PvmVnicSriovVifDriver'} CONF = cfg.CONF @@ -104,12 +108,12 @@ def plug(adapter, host_uuid, instance, vif, slot_mgr, new_vif=True): slot_num = slot_mgr.build_map.get_vea_slot(vif['address']) # Invoke the plug - cna_w = vif_drv.plug(vif, slot_num, new_vif=new_vif) + vnet_w = vif_drv.plug(vif, slot_num, new_vif=new_vif) # If the slot number hadn't been provided initially, save it for the # next rebuild if not slot_num and new_vif: - slot_mgr.register_cna(cna_w) + slot_mgr.register_vnet(vnet_w) def unplug(adapter, host_uuid, instance, vif, slot_mgr, cna_w_list=None): @@ -126,9 +130,9 @@ def unplug(adapter, host_uuid, instance, vif, slot_mgr, cna_w_list=None): allows for an improvement in operation speed. """ vif_drv = _build_vif_driver(adapter, host_uuid, instance, vif) - cna_w = vif_drv.unplug(vif, cna_w_list=cna_w_list) - if cna_w: - slot_mgr.drop_cna(cna_w) + vnet_w = vif_drv.unplug(vif, cna_w_list=cna_w_list) + if vnet_w: + slot_mgr.drop_vnet(vnet_w) def post_live_migrate_at_destination(adapter, host_uuid, instance, vif): @@ -469,6 +473,66 @@ class PvmLBVifDriver(PvmLioVifDriver): return super(PvmLBVifDriver, self).unplug(vif, cna_w_list=cna_w_list) +class PvmVnicSriovVifDriver(PvmVifDriver): + """The SR-IOV VIF driver for PowerVM.""" + + def plug(self, vif, slot_num, new_vif=True): + if not new_vif: + return None + + physnet = vif.get_physical_network() + LOG.debug("Plugging vNIC SR-IOV vif for physical network " + "'%(physnet)s' into instance %(inst)s.", + {'physnet': physnet, 'inst': self.instance.name}) + + # Physical ports for the given physical network + pports = vif['details']['physical_ports'] + if not pports: + raise exception.VirtualInterfacePlugException( + _("Unable to find SR-IOV physical ports for physical " + "network '%(physnet)s' (instance %(inst)s). VIF: %(vif)s") % + {'physnet': physnet, 'inst': self.instance.name, 'vif': vif}) + + # MAC + mac_address = pvm_util.sanitize_mac_for_api(vif['address']) + + # vlan id + vlan_id = int(vif['details']['vlan']) + + # Redundancy: plugin sets from binding:profile, then conf, then default + redundancy = int(vif['details']['redundancy']) + + # Capacity: from binding:profile or pport default + capacity = vif['profile'].get('capacity') + + vnic = pvm_card.VNIC.bld( + self.adapter, vlan_id, slot_num=slot_num, mac_addr=mac_address, + allowed_vlans=pvm_util.VLANList.NONE, + allowed_macs=pvm_util.MACList.NONE) + + sriovtask.set_vnic_back_devs(vnic, pports, min_redundancy=redundancy, + max_redundancy=redundancy, + capacity=capacity) + + return vnic.create(parent_type=pvm_lpar.LPAR, + parent_uuid=vm.get_pvm_uuid(self.instance)) + + def unplug(self, vif, cna_w_list=None): + mac = pvm_util.sanitize_mac_for_api(vif['address']) + vnic = pvm_card.VNIC.search( + self.adapter, parent_type=pvm_lpar.LPAR, + parent_uuid=vm.get_pvm_uuid(self.instance), + mac=mac, one_result=True) + if not vnic: + LOG.warning(_LW('Unable to unplug VIF with mac %(mac)s for ' + 'instance %(inst)s. No matching vNIC was found ' + 'on the instance. VIF: %(vif)s'), + {'mac': mac, 'inst': self.instance.name, 'vif': vif}) + return None + vnic.delete() + return vnic + + class PvmOvsVifDriver(PvmLioVifDriver): """The Open vSwitch VIF driver for PowerVM."""