VIF driver implementation for SR-IOV

PvmVnicSriovVifDriver, a VIF driver for vNICs backed by SR-IOV VFs.  The
main plug method intercepts incoming vif requests and branches off to
this new vif driver if the vif type is 'pvm_sriov'.  The driver "plugs"
a new vNIC into the instance by:

- Building a new pypowervm.wrappers.iocard.VNIC object.
- Assigning backing devices from the pool of physical ports labeled
  according to the physical network name.
- Creating the VNIC on the PowerVM REST server as a child of the LPAR
  corresponding to the instance.

The unplug method locates the appropriate VNIC according to its MAC
address and deletes it from the PowerVM REST server.

Implements: bp/powervm-sriov-nova
Change-Id: I9b707e8c12c2124001089ac1a0f2ef28e5cc7478
This commit is contained in:
Sridhar Venkat 2016-07-17 23:14:18 -04:00 committed by Eric Fried
parent 6ae78c692c
commit 0e8fdcdf63
8 changed files with 256 additions and 29 deletions

View File

@ -197,8 +197,8 @@ Shared Storage Pool (a PowerVM clustered file system) is supported. For
volume attachments, the driver supports Cinder-based attachments via
protocols supported by the hypervisor (e.g. Fibre Channel).
For networking, the networking-powervm project provides a Neutron ML2 Agent.
The agent provides the necessary configuration on the Virtual I/O Server for
For networking, the networking-powervm project provides Neutron ML2 Agents.
The agents provide the necessary configuration on the Virtual I/O Server for
networking. The PowerVM Nova driver code creates the VIF for the client VM,
but the Neutron agent creates the VIF for VLANs.

View File

@ -34,8 +34,14 @@ configured ahead of time.
Configuration File Options
--------------------------
The standard nova configuration options are supported. Additionally, a
``[powervm]`` section is used to provide additional customization to the driver.
The standard nova configuration options are supported. In particular, to use
PowerVM SR-IOV vNIC for networking, the ``pci_passthrough_whitelist`` option
must be set. See the `networking-powervm usage devref`_ for details.
.. _`networking-powervm usage devref`: http://networking-powervm.readthedocs.io/en/latest/devref/usage.html
Additionally, a ``[powervm]`` section is used to provide additional
customization to the driver.
By default, no additional inputs are needed. The base configuration allows for
a Nova driver to support ephemeral disks to a local volume group (only

View File

@ -1501,13 +1501,13 @@ class TestPowerVMDriver(test.TestCase):
'op': 'fake_op'}
mock_log.info.assert_called_with(entry, msg_dict)
def test_host_resources(self):
# Mock methods not currently under test
with mock.patch.object(self.apt, 'read') as mock_read:
mock_read.return_value = None
# Run the actual test
stats = self.drv.get_available_resource('nodename')
self.assertIsNotNone(stats)
@mock.patch('pypowervm.wrappers.managed_system.System.get',
new=mock.Mock(return_value=[mock.Mock()]))
@mock.patch('nova_powervm.virt.powervm.host.build_host_resource_from_ms')
def test_host_resources(self, mock_bhrfm):
mock_bhrfm.return_value = {}
stats = self.drv.get_available_resource('nodename')
self.assertIsNotNone(stats)
# Check for the presence of fields added to host stats
fields = ('local_gb', 'local_gb_used')

View File

@ -19,6 +19,7 @@ import mock
import logging
from nova import test
from oslo_serialization import jsonutils
import pypowervm.tests.test_fixtures as pvm_fx
from nova_powervm.virt.powervm import host as pvm_host
@ -30,13 +31,17 @@ logging.basicConfig()
class TestPowerVMHost(test.TestCase):
def test_host_resources(self):
# Create objects to test with
sriov_adaps = [mock.Mock(phys_ports=[mock.Mock(label='foo'),
mock.Mock(label='')]),
mock.Mock(phys_ports=[mock.Mock(label='bar')])]
ms_wrapper = mock.MagicMock(
proc_units_configurable=500,
proc_units_avail=500,
memory_configurable=5242880,
memory_free=5242752,
mtms=mock.MagicMock(mtms_str='8484923A123456'),
memory_region_size='big')
memory_region_size='big',
asio_config=mock.Mock(sriov_adapters=sriov_adaps))
# Run the actual test
stats = pvm_host.build_host_resource_from_ms(ms_wrapper)
@ -47,7 +52,7 @@ class TestPowerVMHost(test.TestCase):
('memory_mb', 5242880), ('memory_mb_used', 128),
'hypervisor_type', 'hypervisor_version',
'hypervisor_hostname', 'cpu_info',
'supported_instances', 'stats')
'supported_instances', 'stats', 'pci_passthrough_devices')
for fld in fields:
if isinstance(fld, tuple):
value = stats.get(fld[0], None)
@ -64,6 +69,20 @@ class TestPowerVMHost(test.TestCase):
else:
value = stats['stats'].get(stat, None)
self.assertIsNotNone(value)
# pci_passthrough_devices. Parse json - entries can be in any order.
ppdstr = stats['pci_passthrough_devices']
ppdlist = jsonutils.loads(ppdstr)
self.assertEqual({'foo', 'bar', 'default'}, {ppd['physical_network']
for ppd in ppdlist})
self.assertEqual({'foo', 'bar', 'default'}, {ppd['label']
for ppd in ppdlist})
for ppd in ppdlist:
self.assertEqual('type-PCI', ppd['dev_type'])
self.assertEqual('*:*:*.*', ppd['address'])
self.assertEqual('*:*:*.*', ppd['parent_addr'])
self.assertEqual('*', ppd['vendor_id'])
self.assertEqual('*', ppd['product_id'])
self.assertEqual(1, ppd['numa_node'])
class TestHostCPUStats(test.TestCase):

View File

@ -15,7 +15,6 @@
# under the License.
import mock
from mock import call
from nova import exception
from nova import test
@ -111,6 +110,11 @@ class TestVifFunctions(test.TestCase):
{'type': 'pvm_sea'}),
vif.PvmSeaVifDriver)
self.assertIsInstance(
vif._build_vif_driver(self.adpt, 'host_uuid', mock_inst,
{'type': 'pvm_sriov'}),
vif.PvmVnicSriovVifDriver)
# Test raises exception for no type
self.assertRaises(exception.VirtualInterfacePlugException,
vif._build_vif_driver, self.adpt, 'host_uuid',
@ -122,6 +126,121 @@ class TestVifFunctions(test.TestCase):
mock_inst, {'type': 'bad'})
class TestVifSriovDriver(test.TestCase):
def setUp(self):
super(TestVifSriovDriver, self).setUp()
self.adpt = self.useFixture(pvm_fx.AdapterFx()).adpt
self.inst = mock.MagicMock()
self.drv = vif.PvmVnicSriovVifDriver(self.adpt, 'host_uuid', self.inst)
def test_plug_no_pports(self):
"""Raise when plug is called with a network with no physical ports."""
self.assertRaises(exception.VirtualInterfacePlugException,
self.drv.plug, FakeDirectVif('net2', pports=[]), 1)
@mock.patch('pypowervm.util.sanitize_mac_for_api')
@mock.patch('pypowervm.wrappers.iocard.VNIC.bld')
@mock.patch('pypowervm.tasks.sriov.set_vnic_back_devs')
@mock.patch('nova_powervm.virt.powervm.vm.get_pvm_uuid')
def test_plug(self, mock_pvm_uuid, mock_back_devs, mock_vnic_bld,
mock_san_mac):
slot = 10
pports = ['port1', 'port2']
self.drv.plug(FakeDirectVif('default', pports=pports),
slot)
mock_san_mac.assert_called_once_with('ab:ab:ab:ab:ab:ab')
mock_vnic_bld.assert_called_once_with(
self.drv.adapter, 79, slot_num=slot,
mac_addr=mock_san_mac.return_value, allowed_vlans='NONE',
allowed_macs='NONE')
mock_back_devs.assert_called_once_with(
mock_vnic_bld.return_value, ['port1', 'port2'], min_redundancy=3,
max_redundancy=3, capacity=None)
mock_pvm_uuid.assert_called_once_with(self.drv.instance)
mock_vnic_bld.return_value.create.assert_called_once_with(
parent_type=pvm_lpar.LPAR, parent_uuid=mock_pvm_uuid.return_value)
# Now with redundancy/capacity values from binding:profile
mock_san_mac.reset_mock()
mock_vnic_bld.reset_mock()
mock_back_devs.reset_mock()
mock_pvm_uuid.reset_mock()
self.drv.plug(FakeDirectVif('default', pports=pports, cap=0.08),
slot)
mock_san_mac.assert_called_once_with('ab:ab:ab:ab:ab:ab')
mock_vnic_bld.assert_called_once_with(
self.drv.adapter, 79, slot_num=slot,
mac_addr=mock_san_mac.return_value, allowed_vlans='NONE',
allowed_macs='NONE')
mock_back_devs.assert_called_once_with(
mock_vnic_bld.return_value, ['port1', 'port2'], min_redundancy=3,
max_redundancy=3, capacity=0.08)
mock_pvm_uuid.assert_called_once_with(self.drv.instance)
mock_vnic_bld.return_value.create.assert_called_once_with(
parent_type=pvm_lpar.LPAR, parent_uuid=mock_pvm_uuid.return_value)
# No-op with new_vif=False
mock_san_mac.reset_mock()
mock_vnic_bld.reset_mock()
mock_back_devs.reset_mock()
mock_pvm_uuid.reset_mock()
self.assertIsNone(self.drv.plug(
FakeDirectVif('default', pports=pports), slot, new_vif=False))
mock_san_mac.assert_not_called()
mock_vnic_bld.assert_not_called()
mock_back_devs.assert_not_called()
mock_pvm_uuid.assert_not_called()
@mock.patch('pypowervm.wrappers.iocard.VNIC.search')
@mock.patch('nova_powervm.virt.powervm.vm.get_pvm_uuid')
@mock.patch('pypowervm.util.sanitize_mac_for_api')
def test_unplug(self, mock_san_mac, mock_pvm_uuid, mock_find):
fvif = FakeDirectVif('default')
self.assertEqual(mock_find.return_value, self.drv.unplug(fvif))
mock_find.assert_called_once_with(
self.drv.adapter, parent_type=pvm_lpar.LPAR,
parent_uuid=mock_pvm_uuid.return_value,
mac=mock_san_mac.return_value, one_result=True)
mock_pvm_uuid.assert_called_once_with(self.inst)
mock_san_mac.assert_called_once_with(fvif['address'])
mock_find.return_value.delete.assert_called_once_with()
# Not found
mock_find.reset_mock()
mock_pvm_uuid.reset_mock()
mock_san_mac.reset_mock()
mock_find.return_value = None
self.assertIsNone(self.drv.unplug(fvif))
mock_find.assert_called_once_with(
self.drv.adapter, parent_type=pvm_lpar.LPAR,
parent_uuid=mock_pvm_uuid.return_value,
mac=mock_san_mac.return_value, one_result=True)
mock_pvm_uuid.assert_called_once_with(self.inst)
mock_san_mac.assert_called_once_with(fvif['address'])
class FakeDirectVif(dict):
def __init__(self, physnet, pports=None, cap=None):
self._physnet = physnet
super(FakeDirectVif, self).__init__(
network={'id': 'net_id'},
address='ab:ab:ab:ab:ab:ab',
details={
'vlan': '79',
'physical_ports': [],
'redundancy': 3},
profile={})
if pports is not None:
self['details']['physical_ports'] = pports
if cap is not None:
self['profile']['capacity'] = cap
def get_physical_network(self):
return self._physnet
class TestVifSeaDriver(test.TestCase):
def setUp(self):
@ -289,10 +408,10 @@ class TestVifLBDriver(test.TestCase):
self.assertTrue(mock_cna.delete.called)
# Validate the execute
call_ip = call('ip', 'link', 'set', 'fake_dev', 'down',
run_as_root=True)
call_delif = call('brctl', 'delif', 'br0', 'fake_dev',
run_as_root=True)
call_ip = mock.call('ip', 'link', 'set', 'fake_dev', 'down',
run_as_root=True)
call_delif = mock.call('brctl', 'delif', 'br0', 'fake_dev',
run_as_root=True)
mock_exec.assert_has_calls([call_ip, call_delif])
# Test unplug for the case where tap device

View File

@ -992,11 +992,8 @@ class PowerVMDriver(driver.ComputeDriver):
a driver that manages only one node can safely ignore this
:return: Dictionary describing resources
"""
resp = self.adapter.read(pvm_ms.System.schema_type,
root_id=self.host_uuid)
if resp:
self.host_wrapper = pvm_ms.System.wrap(resp.entry)
# Do this here so it refreshes each time this method is called.
self.host_wrapper = pvm_ms.System.get(self.adapter)[0]
# Get host information
data = pvm_host.build_host_resource_from_ms(self.host_wrapper)

View File

@ -18,6 +18,7 @@ import math
from nova.compute import arch
from nova.compute import hv_type
from nova.compute import vm_mode
from nova.objects import fields
from oslo_concurrency import lockutils
from oslo_log import log as logging
from oslo_serialization import jsonutils
@ -83,6 +84,27 @@ def build_host_resource_from_ms(ms_wrapper):
}
data["stats"] = stats
# Produce SR-IOV PCI data. Devices are validated by virtue of the network
# name associated with their label, which must be cleared via an entry in
# the pci_passthrough_whitelist in the nova.conf.
nets = {pport.label or 'default'
for sriov in ms_wrapper.asio_config.sriov_adapters
for pport in sriov.phys_ports}
pci_devs = []
for net in nets:
# These fields are all required in order to satisfy a Claim. Their
# values are as general as possible.
LOG.debug("Registering SR-IOV passthrough for network '%s'", net)
pci_devs.append({"physical_network": net,
"label": net,
"dev_type": fields.PciDeviceType.STANDARD,
"address": "*:*:*.*",
"parent_addr": "*:*:*.*",
"vendor_id": "*",
"product_id": "*",
"numa_node": 1})
data["pci_passthrough_devices"] = jsonutils.dumps(pci_devs)
return data

View File

@ -27,7 +27,9 @@ from oslo_config import cfg
from oslo_utils import importutils
from pypowervm.tasks import cna as pvm_cna
from pypowervm.tasks import partition as pvm_par
from pypowervm.tasks import sriov as sriovtask
from pypowervm import util as pvm_util
from pypowervm.wrappers import iocard as pvm_card
from pypowervm.wrappers import logical_partition as pvm_lpar
from pypowervm.wrappers import managed_system as pvm_ms
from pypowervm.wrappers import network as pvm_net
@ -45,7 +47,9 @@ SECURE_RMC_VLAN = 4094
VIF_MAPPING = {'pvm_sea': 'nova_powervm.virt.powervm.vif.PvmSeaVifDriver',
'ovs': 'nova_powervm.virt.powervm.vif.PvmOvsVifDriver',
'bridge': 'nova_powervm.virt.powervm.vif.PvmLBVifDriver'}
'bridge': 'nova_powervm.virt.powervm.vif.PvmLBVifDriver',
'pvm_sriov':
'nova_powervm.virt.powervm.vif.PvmVnicSriovVifDriver'}
CONF = cfg.CONF
@ -104,12 +108,12 @@ def plug(adapter, host_uuid, instance, vif, slot_mgr, new_vif=True):
slot_num = slot_mgr.build_map.get_vea_slot(vif['address'])
# Invoke the plug
cna_w = vif_drv.plug(vif, slot_num, new_vif=new_vif)
vnet_w = vif_drv.plug(vif, slot_num, new_vif=new_vif)
# If the slot number hadn't been provided initially, save it for the
# next rebuild
if not slot_num and new_vif:
slot_mgr.register_cna(cna_w)
slot_mgr.register_vnet(vnet_w)
def unplug(adapter, host_uuid, instance, vif, slot_mgr, cna_w_list=None):
@ -126,9 +130,9 @@ def unplug(adapter, host_uuid, instance, vif, slot_mgr, cna_w_list=None):
allows for an improvement in operation speed.
"""
vif_drv = _build_vif_driver(adapter, host_uuid, instance, vif)
cna_w = vif_drv.unplug(vif, cna_w_list=cna_w_list)
if cna_w:
slot_mgr.drop_cna(cna_w)
vnet_w = vif_drv.unplug(vif, cna_w_list=cna_w_list)
if vnet_w:
slot_mgr.drop_vnet(vnet_w)
def post_live_migrate_at_destination(adapter, host_uuid, instance, vif):
@ -469,6 +473,66 @@ class PvmLBVifDriver(PvmLioVifDriver):
return super(PvmLBVifDriver, self).unplug(vif, cna_w_list=cna_w_list)
class PvmVnicSriovVifDriver(PvmVifDriver):
"""The SR-IOV VIF driver for PowerVM."""
def plug(self, vif, slot_num, new_vif=True):
if not new_vif:
return None
physnet = vif.get_physical_network()
LOG.debug("Plugging vNIC SR-IOV vif for physical network "
"'%(physnet)s' into instance %(inst)s.",
{'physnet': physnet, 'inst': self.instance.name})
# Physical ports for the given physical network
pports = vif['details']['physical_ports']
if not pports:
raise exception.VirtualInterfacePlugException(
_("Unable to find SR-IOV physical ports for physical "
"network '%(physnet)s' (instance %(inst)s). VIF: %(vif)s") %
{'physnet': physnet, 'inst': self.instance.name, 'vif': vif})
# MAC
mac_address = pvm_util.sanitize_mac_for_api(vif['address'])
# vlan id
vlan_id = int(vif['details']['vlan'])
# Redundancy: plugin sets from binding:profile, then conf, then default
redundancy = int(vif['details']['redundancy'])
# Capacity: from binding:profile or pport default
capacity = vif['profile'].get('capacity')
vnic = pvm_card.VNIC.bld(
self.adapter, vlan_id, slot_num=slot_num, mac_addr=mac_address,
allowed_vlans=pvm_util.VLANList.NONE,
allowed_macs=pvm_util.MACList.NONE)
sriovtask.set_vnic_back_devs(vnic, pports, min_redundancy=redundancy,
max_redundancy=redundancy,
capacity=capacity)
return vnic.create(parent_type=pvm_lpar.LPAR,
parent_uuid=vm.get_pvm_uuid(self.instance))
def unplug(self, vif, cna_w_list=None):
mac = pvm_util.sanitize_mac_for_api(vif['address'])
vnic = pvm_card.VNIC.search(
self.adapter, parent_type=pvm_lpar.LPAR,
parent_uuid=vm.get_pvm_uuid(self.instance),
mac=mac, one_result=True)
if not vnic:
LOG.warning(_LW('Unable to unplug VIF with mac %(mac)s for '
'instance %(inst)s. No matching vNIC was found '
'on the instance. VIF: %(vif)s'),
{'mac': mac, 'inst': self.instance.name, 'vif': vif})
return None
vnic.delete()
return vnic
class PvmOvsVifDriver(PvmLioVifDriver):
"""The Open vSwitch VIF driver for PowerVM."""