VIF driver implementation for SR-IOV

PvmVnicSriovVifDriver, a VIF driver for vNICs backed by SR-IOV VFs.  The
main plug method intercepts incoming vif requests and branches off to
this new vif driver if the vif type is 'pvm_sriov'.  The driver "plugs"
a new vNIC into the instance by:

- Building a new pypowervm.wrappers.iocard.VNIC object.
- Assigning backing devices from the pool of physical ports labeled
  according to the physical network name.
- Creating the VNIC on the PowerVM REST server as a child of the LPAR
  corresponding to the instance.

The unplug method locates the appropriate VNIC according to its MAC
address and deletes it from the PowerVM REST server.

Implements: bp/powervm-sriov-nova
Change-Id: I9b707e8c12c2124001089ac1a0f2ef28e5cc7478

README.rst

@@ -197,8 +197,8 @@ Shared Storage Pool (a PowerVM clustered file system) is supported.  For
 volume attachments, the driver supports Cinder-based attachments via
 protocols supported by the hypervisor (e.g. Fibre Channel).
 
-For networking, the networking-powervm project provides a Neutron ML2 Agent.
+For networking, the networking-powervm project provides Neutron ML2 Agents.
-The agent provides the necessary configuration on the Virtual I/O Server for
+The agents provide the necessary configuration on the Virtual I/O Server for
 networking.  The PowerVM Nova driver code creates the VIF for the client VM,
 but the Neutron agent creates the VIF for VLANs.
 

doc/source/devref/usage.rst

@@ -34,8 +34,14 @@ configured ahead of time.
 
 Configuration File Options
 --------------------------
-The standard nova configuration options are supported.  Additionally, a
+The standard nova configuration options are supported.  In particular, to use
-``[powervm]`` section is used to provide additional customization to the driver.
+PowerVM SR-IOV vNIC for networking, the ``pci_passthrough_whitelist`` option
+must be set.  See the `networking-powervm usage devref`_ for details.
+
+.. _`networking-powervm usage devref`: http://networking-powervm.readthedocs.io/en/latest/devref/usage.html
+
+Additionally, a ``[powervm]`` section is used to provide additional
+customization to the driver.
 
 By default, no additional inputs are needed.  The base configuration allows for
 a Nova driver to support ephemeral disks to a local volume group (only

nova_powervm/tests/virt/powervm/test_driver.py

@@ -1501,13 +1501,13 @@ class TestPowerVMDriver(test.TestCase):
                     'op': 'fake_op'}
         mock_log.info.assert_called_with(entry, msg_dict)
 
-    def test_host_resources(self):
+    @mock.patch('pypowervm.wrappers.managed_system.System.get',
-        # Mock methods not currently under test
+                new=mock.Mock(return_value=[mock.Mock()]))
-        with mock.patch.object(self.apt, 'read') as mock_read:
+    @mock.patch('nova_powervm.virt.powervm.host.build_host_resource_from_ms')
-            mock_read.return_value = None
+    def test_host_resources(self, mock_bhrfm):
-            # Run the actual test
+        mock_bhrfm.return_value = {}
-            stats = self.drv.get_available_resource('nodename')
+        stats = self.drv.get_available_resource('nodename')
-            self.assertIsNotNone(stats)
+        self.assertIsNotNone(stats)
 
         # Check for the presence of fields added to host stats
         fields = ('local_gb', 'local_gb_used')

nova_powervm/tests/virt/powervm/test_host.py

@@ -19,6 +19,7 @@ import mock
 
 import logging
 from nova import test
+from oslo_serialization import jsonutils
 import pypowervm.tests.test_fixtures as pvm_fx
 
 from nova_powervm.virt.powervm import host as pvm_host
@@ -30,13 +31,17 @@ logging.basicConfig()
 class TestPowerVMHost(test.TestCase):
     def test_host_resources(self):
         # Create objects to test with
+        sriov_adaps = [mock.Mock(phys_ports=[mock.Mock(label='foo'),
+                                             mock.Mock(label='')]),
+                       mock.Mock(phys_ports=[mock.Mock(label='bar')])]
         ms_wrapper = mock.MagicMock(
             proc_units_configurable=500,
             proc_units_avail=500,
             memory_configurable=5242880,
             memory_free=5242752,
             mtms=mock.MagicMock(mtms_str='8484923A123456'),
-            memory_region_size='big')
+            memory_region_size='big',
+            asio_config=mock.Mock(sriov_adapters=sriov_adaps))
 
         # Run the actual test
         stats = pvm_host.build_host_resource_from_ms(ms_wrapper)
@@ -47,7 +52,7 @@ class TestPowerVMHost(test.TestCase):
                   ('memory_mb', 5242880), ('memory_mb_used', 128),
                   'hypervisor_type', 'hypervisor_version',
                   'hypervisor_hostname', 'cpu_info',
-                  'supported_instances', 'stats')
+                  'supported_instances', 'stats', 'pci_passthrough_devices')
         for fld in fields:
             if isinstance(fld, tuple):
                 value = stats.get(fld[0], None)
@@ -64,6 +69,20 @@ class TestPowerVMHost(test.TestCase):
             else:
                 value = stats['stats'].get(stat, None)
                 self.assertIsNotNone(value)
+        # pci_passthrough_devices.  Parse json - entries can be in any order.
+        ppdstr = stats['pci_passthrough_devices']
+        ppdlist = jsonutils.loads(ppdstr)
+        self.assertEqual({'foo', 'bar', 'default'}, {ppd['physical_network']
+                                                     for ppd in ppdlist})
+        self.assertEqual({'foo', 'bar', 'default'}, {ppd['label']
+                                                     for ppd in ppdlist})
+        for ppd in ppdlist:
+            self.assertEqual('type-PCI', ppd['dev_type'])
+            self.assertEqual('*:*:*.*', ppd['address'])
+            self.assertEqual('*:*:*.*', ppd['parent_addr'])
+            self.assertEqual('*', ppd['vendor_id'])
+            self.assertEqual('*', ppd['product_id'])
+            self.assertEqual(1, ppd['numa_node'])
 
 
 class TestHostCPUStats(test.TestCase):

nova_powervm/tests/virt/powervm/test_vif.py

@@ -15,7 +15,6 @@
 #    under the License.
 
 import mock
-from mock import call
 
 from nova import exception
 from nova import test
@@ -111,6 +110,11 @@ class TestVifFunctions(test.TestCase):
                                   {'type': 'pvm_sea'}),
             vif.PvmSeaVifDriver)
 
+        self.assertIsInstance(
+            vif._build_vif_driver(self.adpt, 'host_uuid', mock_inst,
+                                  {'type': 'pvm_sriov'}),
+            vif.PvmVnicSriovVifDriver)
+
         # Test raises exception for no type
         self.assertRaises(exception.VirtualInterfacePlugException,
                           vif._build_vif_driver, self.adpt, 'host_uuid',
@@ -122,6 +126,121 @@ class TestVifFunctions(test.TestCase):
                           mock_inst, {'type': 'bad'})
 
 
+class TestVifSriovDriver(test.TestCase):
+
+    def setUp(self):
+        super(TestVifSriovDriver, self).setUp()
+
+        self.adpt = self.useFixture(pvm_fx.AdapterFx()).adpt
+        self.inst = mock.MagicMock()
+        self.drv = vif.PvmVnicSriovVifDriver(self.adpt, 'host_uuid', self.inst)
+
+    def test_plug_no_pports(self):
+        """Raise when plug is called with a network with no physical ports."""
+        self.assertRaises(exception.VirtualInterfacePlugException,
+                          self.drv.plug, FakeDirectVif('net2', pports=[]), 1)
+
+    @mock.patch('pypowervm.util.sanitize_mac_for_api')
+    @mock.patch('pypowervm.wrappers.iocard.VNIC.bld')
+    @mock.patch('pypowervm.tasks.sriov.set_vnic_back_devs')
+    @mock.patch('nova_powervm.virt.powervm.vm.get_pvm_uuid')
+    def test_plug(self, mock_pvm_uuid, mock_back_devs, mock_vnic_bld,
+                  mock_san_mac):
+        slot = 10
+        pports = ['port1', 'port2']
+        self.drv.plug(FakeDirectVif('default', pports=pports),
+                      slot)
+        mock_san_mac.assert_called_once_with('ab:ab:ab:ab:ab:ab')
+        mock_vnic_bld.assert_called_once_with(
+            self.drv.adapter, 79, slot_num=slot,
+            mac_addr=mock_san_mac.return_value, allowed_vlans='NONE',
+            allowed_macs='NONE')
+        mock_back_devs.assert_called_once_with(
+            mock_vnic_bld.return_value, ['port1', 'port2'], min_redundancy=3,
+            max_redundancy=3, capacity=None)
+        mock_pvm_uuid.assert_called_once_with(self.drv.instance)
+        mock_vnic_bld.return_value.create.assert_called_once_with(
+            parent_type=pvm_lpar.LPAR, parent_uuid=mock_pvm_uuid.return_value)
+
+        # Now with redundancy/capacity values from binding:profile
+        mock_san_mac.reset_mock()
+        mock_vnic_bld.reset_mock()
+        mock_back_devs.reset_mock()
+        mock_pvm_uuid.reset_mock()
+        self.drv.plug(FakeDirectVif('default', pports=pports, cap=0.08),
+                      slot)
+        mock_san_mac.assert_called_once_with('ab:ab:ab:ab:ab:ab')
+        mock_vnic_bld.assert_called_once_with(
+            self.drv.adapter, 79, slot_num=slot,
+            mac_addr=mock_san_mac.return_value, allowed_vlans='NONE',
+            allowed_macs='NONE')
+        mock_back_devs.assert_called_once_with(
+            mock_vnic_bld.return_value, ['port1', 'port2'], min_redundancy=3,
+            max_redundancy=3, capacity=0.08)
+        mock_pvm_uuid.assert_called_once_with(self.drv.instance)
+        mock_vnic_bld.return_value.create.assert_called_once_with(
+            parent_type=pvm_lpar.LPAR, parent_uuid=mock_pvm_uuid.return_value)
+
+        # No-op with new_vif=False
+        mock_san_mac.reset_mock()
+        mock_vnic_bld.reset_mock()
+        mock_back_devs.reset_mock()
+        mock_pvm_uuid.reset_mock()
+        self.assertIsNone(self.drv.plug(
+            FakeDirectVif('default', pports=pports), slot, new_vif=False))
+        mock_san_mac.assert_not_called()
+        mock_vnic_bld.assert_not_called()
+        mock_back_devs.assert_not_called()
+        mock_pvm_uuid.assert_not_called()
+
+    @mock.patch('pypowervm.wrappers.iocard.VNIC.search')
+    @mock.patch('nova_powervm.virt.powervm.vm.get_pvm_uuid')
+    @mock.patch('pypowervm.util.sanitize_mac_for_api')
+    def test_unplug(self, mock_san_mac, mock_pvm_uuid, mock_find):
+        fvif = FakeDirectVif('default')
+        self.assertEqual(mock_find.return_value, self.drv.unplug(fvif))
+        mock_find.assert_called_once_with(
+            self.drv.adapter, parent_type=pvm_lpar.LPAR,
+            parent_uuid=mock_pvm_uuid.return_value,
+            mac=mock_san_mac.return_value, one_result=True)
+        mock_pvm_uuid.assert_called_once_with(self.inst)
+        mock_san_mac.assert_called_once_with(fvif['address'])
+        mock_find.return_value.delete.assert_called_once_with()
+
+        # Not found
+        mock_find.reset_mock()
+        mock_pvm_uuid.reset_mock()
+        mock_san_mac.reset_mock()
+        mock_find.return_value = None
+        self.assertIsNone(self.drv.unplug(fvif))
+        mock_find.assert_called_once_with(
+            self.drv.adapter, parent_type=pvm_lpar.LPAR,
+            parent_uuid=mock_pvm_uuid.return_value,
+            mac=mock_san_mac.return_value, one_result=True)
+        mock_pvm_uuid.assert_called_once_with(self.inst)
+        mock_san_mac.assert_called_once_with(fvif['address'])
+
+
+class FakeDirectVif(dict):
+    def __init__(self, physnet, pports=None, cap=None):
+        self._physnet = physnet
+        super(FakeDirectVif, self).__init__(
+            network={'id': 'net_id'},
+            address='ab:ab:ab:ab:ab:ab',
+            details={
+                'vlan': '79',
+                'physical_ports': [],
+                'redundancy': 3},
+            profile={})
+        if pports is not None:
+            self['details']['physical_ports'] = pports
+        if cap is not None:
+            self['profile']['capacity'] = cap
+
+    def get_physical_network(self):
+        return self._physnet
+
+
 class TestVifSeaDriver(test.TestCase):
 
     def setUp(self):
@@ -289,10 +408,10 @@ class TestVifLBDriver(test.TestCase):
         self.assertTrue(mock_cna.delete.called)
 
         # Validate the execute
-        call_ip = call('ip', 'link', 'set', 'fake_dev', 'down',
+        call_ip = mock.call('ip', 'link', 'set', 'fake_dev', 'down',
-                       run_as_root=True)
+                            run_as_root=True)
-        call_delif = call('brctl', 'delif', 'br0', 'fake_dev',
+        call_delif = mock.call('brctl', 'delif', 'br0', 'fake_dev',
-                          run_as_root=True)
+                               run_as_root=True)
         mock_exec.assert_has_calls([call_ip, call_delif])
 
         # Test unplug for the case where tap device

nova_powervm/virt/powervm/driver.py

@@ -992,11 +992,8 @@ class PowerVMDriver(driver.ComputeDriver):
             a driver that manages only one node can safely ignore this
         :return: Dictionary describing resources
         """
-
+        # Do this here so it refreshes each time this method is called.
-        resp = self.adapter.read(pvm_ms.System.schema_type,
+        self.host_wrapper = pvm_ms.System.get(self.adapter)[0]
-                                 root_id=self.host_uuid)
-        if resp:
-            self.host_wrapper = pvm_ms.System.wrap(resp.entry)
         # Get host information
         data = pvm_host.build_host_resource_from_ms(self.host_wrapper)
 

nova_powervm/virt/powervm/host.py

@@ -18,6 +18,7 @@ import math
 from nova.compute import arch
 from nova.compute import hv_type
 from nova.compute import vm_mode
+from nova.objects import fields
 from oslo_concurrency import lockutils
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
@@ -83,6 +84,27 @@ def build_host_resource_from_ms(ms_wrapper):
              }
     data["stats"] = stats
 
+    # Produce SR-IOV PCI data.  Devices are validated by virtue of the network
+    # name associated with their label, which must be cleared via an entry in
+    # the pci_passthrough_whitelist in the nova.conf.
+    nets = {pport.label or 'default'
+            for sriov in ms_wrapper.asio_config.sriov_adapters
+            for pport in sriov.phys_ports}
+    pci_devs = []
+    for net in nets:
+        # These fields are all required in order to satisfy a Claim.  Their
+        # values are as general as possible.
+        LOG.debug("Registering SR-IOV passthrough for network '%s'", net)
+        pci_devs.append({"physical_network": net,
+                         "label": net,
+                         "dev_type": fields.PciDeviceType.STANDARD,
+                         "address": "*:*:*.*",
+                         "parent_addr": "*:*:*.*",
+                         "vendor_id": "*",
+                         "product_id": "*",
+                         "numa_node": 1})
+    data["pci_passthrough_devices"] = jsonutils.dumps(pci_devs)
+
     return data
 
 

nova_powervm/virt/powervm/vif.py

@@ -27,7 +27,9 @@ from oslo_config import cfg
 from oslo_utils import importutils
 from pypowervm.tasks import cna as pvm_cna
 from pypowervm.tasks import partition as pvm_par
+from pypowervm.tasks import sriov as sriovtask
 from pypowervm import util as pvm_util
+from pypowervm.wrappers import iocard as pvm_card
 from pypowervm.wrappers import logical_partition as pvm_lpar
 from pypowervm.wrappers import managed_system as pvm_ms
 from pypowervm.wrappers import network as pvm_net
@@ -45,7 +47,9 @@ SECURE_RMC_VLAN = 4094
 
 VIF_MAPPING = {'pvm_sea': 'nova_powervm.virt.powervm.vif.PvmSeaVifDriver',
                'ovs': 'nova_powervm.virt.powervm.vif.PvmOvsVifDriver',
-               'bridge': 'nova_powervm.virt.powervm.vif.PvmLBVifDriver'}
+               'bridge': 'nova_powervm.virt.powervm.vif.PvmLBVifDriver',
+               'pvm_sriov':
+               'nova_powervm.virt.powervm.vif.PvmVnicSriovVifDriver'}
 
 CONF = cfg.CONF
 
@@ -104,12 +108,12 @@ def plug(adapter, host_uuid, instance, vif, slot_mgr, new_vif=True):
     slot_num = slot_mgr.build_map.get_vea_slot(vif['address'])
 
     # Invoke the plug
-    cna_w = vif_drv.plug(vif, slot_num, new_vif=new_vif)
+    vnet_w = vif_drv.plug(vif, slot_num, new_vif=new_vif)
 
     # If the slot number hadn't been provided initially, save it for the
     # next rebuild
     if not slot_num and new_vif:
-        slot_mgr.register_cna(cna_w)
+        slot_mgr.register_vnet(vnet_w)
 
 
 def unplug(adapter, host_uuid, instance, vif, slot_mgr, cna_w_list=None):
@@ -126,9 +130,9 @@ def unplug(adapter, host_uuid, instance, vif, slot_mgr, cna_w_list=None):
                        allows for an improvement in operation speed.
     """
     vif_drv = _build_vif_driver(adapter, host_uuid, instance, vif)
-    cna_w = vif_drv.unplug(vif, cna_w_list=cna_w_list)
+    vnet_w = vif_drv.unplug(vif, cna_w_list=cna_w_list)
-    if cna_w:
+    if vnet_w:
-        slot_mgr.drop_cna(cna_w)
+        slot_mgr.drop_vnet(vnet_w)
 
 
 def post_live_migrate_at_destination(adapter, host_uuid, instance, vif):
@@ -469,6 +473,66 @@ class PvmLBVifDriver(PvmLioVifDriver):
         return super(PvmLBVifDriver, self).unplug(vif, cna_w_list=cna_w_list)
 
 
+class PvmVnicSriovVifDriver(PvmVifDriver):
+    """The SR-IOV VIF driver for PowerVM."""
+
+    def plug(self, vif, slot_num, new_vif=True):
+        if not new_vif:
+            return None
+
+        physnet = vif.get_physical_network()
+        LOG.debug("Plugging vNIC SR-IOV vif for physical network "
+                  "'%(physnet)s' into instance %(inst)s.",
+                  {'physnet': physnet, 'inst': self.instance.name})
+
+        # Physical ports for the given physical network
+        pports = vif['details']['physical_ports']
+        if not pports:
+            raise exception.VirtualInterfacePlugException(
+                _("Unable to find SR-IOV physical ports for physical "
+                  "network '%(physnet)s' (instance %(inst)s).  VIF: %(vif)s") %
+                {'physnet': physnet, 'inst': self.instance.name, 'vif': vif})
+
+        # MAC
+        mac_address = pvm_util.sanitize_mac_for_api(vif['address'])
+
+        # vlan id
+        vlan_id = int(vif['details']['vlan'])
+
+        # Redundancy: plugin sets from binding:profile, then conf, then default
+        redundancy = int(vif['details']['redundancy'])
+
+        # Capacity: from binding:profile or pport default
+        capacity = vif['profile'].get('capacity')
+
+        vnic = pvm_card.VNIC.bld(
+            self.adapter, vlan_id, slot_num=slot_num, mac_addr=mac_address,
+            allowed_vlans=pvm_util.VLANList.NONE,
+            allowed_macs=pvm_util.MACList.NONE)
+
+        sriovtask.set_vnic_back_devs(vnic, pports, min_redundancy=redundancy,
+                                     max_redundancy=redundancy,
+                                     capacity=capacity)
+
+        return vnic.create(parent_type=pvm_lpar.LPAR,
+                           parent_uuid=vm.get_pvm_uuid(self.instance))
+
+    def unplug(self, vif, cna_w_list=None):
+        mac = pvm_util.sanitize_mac_for_api(vif['address'])
+        vnic = pvm_card.VNIC.search(
+            self.adapter, parent_type=pvm_lpar.LPAR,
+            parent_uuid=vm.get_pvm_uuid(self.instance),
+            mac=mac, one_result=True)
+        if not vnic:
+            LOG.warning(_LW('Unable to unplug VIF with mac %(mac)s for '
+                            'instance %(inst)s.  No matching vNIC was found '
+                            'on the instance.  VIF: %(vif)s'),
+                        {'mac': mac, 'inst': self.instance.name, 'vif': vif})
+            return None
+        vnic.delete()
+        return vnic
+
+
 class PvmOvsVifDriver(PvmLioVifDriver):
     """The Open vSwitch VIF driver for PowerVM."""