pci: changing the claiming and allocation logic for PF/VF assignment

Claiming and allocation logic of the pci devices will be changed to
reflect the relationship between VFs and PFs.

PFs cannot be claimed or assigned if one of it's dependent VFs is
not available. As well as, all of the dependent VFs should become
UNAVAILABLE when the PF is successfully claimed/assigned.

In the same way, VF cannot be claimed or assigned when it's related
PF is assigned. In case of a successful claim/assignment of a VF,
the PF should become UNAVAILABLE.

In general, all virtual function object should disappear from PCI
and be removed from the stats, when PF is assigned. The above logic
will serve as a precaution step.

Partially implements blueprint sriov-physical-function-passthrough

Change-Id: Iae62126a8471f3b8c7a35db39d960e3bed5a1390
This commit is contained in:
Vladik Romanovsky 2015-11-23 22:39:00 -05:00
parent b011421f44
commit 0fe0b4f63e
7 changed files with 419 additions and 7 deletions

View File

@ -1736,6 +1736,19 @@ class PciDeviceInvalidStatus(Invalid):
"instead of %(hopestatus)s")
class PciDeviceVFInvalidStatus(Invalid):
msg_fmt = _(
"Not all Virtual Functions of PF %(compute_node_id)s:%(address)s "
"are free.")
class PciDevicePFInvalidStatus(Invalid):
msg_fmt = _(
"Physical Function %(compute_node_id)s:%(address)s, related to VF"
" %(compute_node_id)s:%(vf_address)s is %(status)s "
"instead of %(hopestatus)s")
class PciDeviceInvalidOwner(Invalid):
msg_fmt = _(
"PCI device %(compute_node_id)s:%(address)s is owned by %(owner)s "

View File

@ -442,8 +442,11 @@ class PciDeviceStatus(Enum):
ALLOCATED = "allocated"
REMOVED = "removed" # The device has been hot-removed and not yet deleted
DELETED = "deleted" # The device is marked not available/deleted.
UNCLAIMABLE = "unclaimable"
UNAVAILABLE = "unavailable"
ALL = (AVAILABLE, CLAIMED, ALLOCATED, REMOVED, DELETED)
ALL = (AVAILABLE, CLAIMED, ALLOCATED, REMOVED, DELETED, UNAVAILABLE,
UNCLAIMABLE)
def __init__(self):
super(PciDeviceStatus, self).__init__(

View File

@ -15,6 +15,7 @@
import copy
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import versionutils
@ -25,6 +26,8 @@ from nova import objects
from nova.objects import base
from nova.objects import fields
LOG = logging.getLogger(__name__)
def compare_pci_device_attributes(obj_a, obj_b):
pci_ignore_fields = base.NovaPersistentObject.fields.keys()
@ -87,7 +90,8 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
# Version 1.2: added request_id field
# Version 1.3: Added field to represent PCI device NUMA node
# Version 1.4: Added parent_addr field
VERSION = '1.4'
# Version 1.5: Added 2 new device statuses: UNCLAIMABLE and UNAVAILABLE
VERSION = '1.5'
fields = {
'id': fields.IntegerField(),
@ -129,6 +133,15 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
extra_info = primitive.get('extra_info', {})
extra_info['phys_function'] = primitive['parent_addr']
del primitive['parent_addr']
if target_version < (1, 5) and 'parent_addr' in primitive:
added_statuses = (fields.PciDeviceStatus.UNCLAIMABLE,
fields.PciDeviceStatus.UNAVAILABLE)
status = primitive['status']
if status in added_statuses:
raise exception.ObjectActionError(
action='obj_make_compatible',
reason='status=%s not supported in version %s' % (
status, target_version))
def update_device(self, dev_dict):
"""Sync the content from device dictionary to device object.
@ -244,18 +257,73 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
self.address, updates)
self._from_db_object(self._context, self, db_pci)
@staticmethod
def _bulk_update_status(dev_list, status):
for dev in dev_list:
dev.status = status
def claim(self, instance):
if self.status != fields.PciDeviceStatus.AVAILABLE:
raise exception.PciDeviceInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.address, status=self.status,
hopestatus=[fields.PciDeviceStatus.AVAILABLE])
if self.dev_type == fields.PciDeviceType.SRIOV_PF:
# Update PF status to CLAIMED if all of it dependants are free
# and set their status to UNCLAIMABLE
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.address)
if not all([vf.is_available() for vf in vfs_list]):
raise exception.PciDeviceVFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.address)
self._bulk_update_status(vfs_list,
fields.PciDeviceStatus.UNCLAIMABLE)
elif self.dev_type == fields.PciDeviceType.SRIOV_VF:
# Update VF status to CLAIMED if it's parent has not been
# previuosly allocated or claimed
# When claiming/allocating a VF, it's parent PF becomes
# unclaimable/unavailable. Therefore, it is expected to find the
# parent PF in an unclaimable/unavailable state for any following
# claims to a sibling VF
parent_ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.UNCLAIMABLE,
fields.PciDeviceStatus.UNAVAILABLE)
try:
parent = self.get_by_dev_addr(self._context,
self.compute_node_id,
self.parent_addr)
if parent.status not in parent_ok_statuses:
raise exception.PciDevicePFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.parent_addr, status=self.status,
vf_address=self.address,
hopestatus=parent_ok_statuses)
# Set PF status
if parent.status == fields.PciDeviceStatus.AVAILABLE:
parent.status = fields.PciDeviceStatus.UNCLAIMABLE
except exception.PciDeviceNotFound:
LOG.debug('Physical function addr: %(pf_addr)s parent of '
'VF addr: %(vf_addr)s was not found',
{'pf_addr': self.parent_addr,
'vf_addr': self.address})
self.status = fields.PciDeviceStatus.CLAIMED
self.instance_uuid = instance['uuid']
def allocate(self, instance):
ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.CLAIMED)
parent_ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.UNCLAIMABLE,
fields.PciDeviceStatus.UNAVAILABLE)
dependatns_ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.UNCLAIMABLE)
if self.status not in ok_statuses:
raise exception.PciDeviceInvalidStatus(
compute_node_id=self.compute_node_id,
@ -267,6 +335,37 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
compute_node_id=self.compute_node_id,
address=self.address, owner=self.instance_uuid,
hopeowner=instance['uuid'])
if self.dev_type == fields.PciDeviceType.SRIOV_PF:
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.address)
if not all([vf.status in dependatns_ok_statuses for
vf in vfs_list]):
raise exception.PciDeviceVFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.address)
self._bulk_update_status(vfs_list,
fields.PciDeviceStatus.UNAVAILABLE)
elif (self.dev_type == fields.PciDeviceType.SRIOV_VF):
try:
parent = self.get_by_dev_addr(self._context,
self.compute_node_id,
self.parent_addr)
if parent.status not in parent_ok_statuses:
raise exception.PciDevicePFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.parent_addr, status=self.status,
vf_address=self.address,
hopestatus=parent_ok_statuses)
# Set PF status
parent.status = fields.PciDeviceStatus.UNAVAILABLE
except exception.PciDeviceNotFound:
LOG.debug('Physical function addr: %(pf_addr)s parent of '
'VF addr: %(vf_addr)s was not found',
{'pf_addr': self.parent_addr,
'vf_addr': self.address})
self.status = fields.PciDeviceStatus.ALLOCATED
self.instance_uuid = instance['uuid']
@ -293,6 +392,7 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
def free(self, instance=None):
ok_statuses = (fields.PciDeviceStatus.ALLOCATED,
fields.PciDeviceStatus.CLAIMED)
free_devs = []
if self.status not in ok_statuses:
raise exception.PciDeviceInvalidStatus(
compute_node_id=self.compute_node_id,
@ -303,8 +403,36 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
compute_node_id=self.compute_node_id,
address=self.address, owner=self.instance_uuid,
hopeowner=instance['uuid'])
if self.dev_type == fields.PciDeviceType.SRIOV_PF:
# Set all PF dependants status to AVAILABLE
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.address)
self._bulk_update_status(vfs_list,
fields.PciDeviceStatus.AVAILABLE)
free_devs.extend(vfs_list)
if self.dev_type == fields.PciDeviceType.SRIOV_VF:
# Set PF status to AVAILABLE if all of it's VFs are free
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.parent_addr)
if all([vf.is_available() for vf in vfs_list if vf.id != self.id]):
try:
parent = self.get_by_dev_addr(self._context,
self.compute_node_id,
self.parent_addr)
parent.status = fields.PciDeviceStatus.AVAILABLE
free_devs.append(parent)
except exception.PciDeviceNotFound:
LOG.debug('Physical function addr: %(pf_addr)s parent of '
'VF addr: %(vf_addr)s was not found',
{'pf_addr': self.parent_addr,
'vf_addr': self.address})
old_status = self.status
self.status = fields.PciDeviceStatus.AVAILABLE
free_devs.append(self)
self.instance_uuid = None
self.request_id = None
if old_status == fields.PciDeviceStatus.ALLOCATED and instance:
@ -316,6 +444,7 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
instance['pci_devices'].remove(existed)
else:
instance.pci_devices.objects.remove(existed)
return free_devs
def is_available(self):
return self.status == fields.PciDeviceStatus.AVAILABLE

View File

@ -215,11 +215,12 @@ class PciDevTracker(object):
return None
def _free_device(self, dev, instance=None):
dev.free(instance)
freed_devs = dev.free(instance)
stale = self.stale.pop(dev.address, None)
if stale:
dev.update_device(stale)
self.stats.add_device(dev)
for dev in freed_devs:
self.stats.add_device(dev)
def _free_instance(self, instance):
# Note(yjiang5): When an instance is resized, the devices in the

View File

@ -1169,7 +1169,7 @@ object_data = {
'NetworkList': '1.2-69eca910d8fa035dfecd8ba10877ee59',
'NetworkRequest': '1.1-7a3e4ca2ce1e7b62d8400488f2f2b756',
'NetworkRequestList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e',
'PciDevice': '1.4-4f54e80054bbb6414e17eb9babc97a44',
'PciDevice': '1.5-0d5abe5c91645b8469eb2a93fc53f932',
'PciDeviceList': '1.3-52ff14355491c8c580bdc0ba34c26210',
'PciDevicePool': '1.1-3f5ddc3ff7bfa14da7f6c7e9904cc000',
'PciDevicePoolList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e',

View File

@ -25,6 +25,7 @@ from nova import objects
from nova.objects import fields
from nova.objects import instance
from nova.objects import pci_device
from nova import test
from nova.tests.unit.objects import test_objects
dev_dict = {
@ -33,6 +34,8 @@ dev_dict = {
'product_id': 'p',
'vendor_id': 'v',
'numa_node': 0,
'dev_type': fields.PciDeviceType.STANDARD,
'parent_addr': None,
'status': fields.PciDeviceStatus.AVAILABLE}
@ -122,7 +125,7 @@ class _TestPciDeviceObject(object):
self.assertEqual(self.pci_device.obj_what_changed(),
set(['compute_node_id', 'product_id', 'vendor_id',
'numa_node', 'status', 'address', 'extra_info',
'parent_addr']))
'dev_type', 'parent_addr']))
def test_pci_device_extra_info(self):
self.dev_dict = copy.copy(dev_dict)
@ -135,7 +138,7 @@ class _TestPciDeviceObject(object):
self.assertEqual(self.pci_device.obj_what_changed(),
set(['compute_node_id', 'address', 'product_id',
'vendor_id', 'numa_node', 'status',
'parent_addr', 'extra_info']))
'extra_info', 'dev_type', 'parent_addr']))
def test_update_device(self):
self.pci_device = pci_device.PciDevice.create(None, dev_dict)
@ -181,6 +184,15 @@ class _TestPciDeviceObject(object):
self.assertEqual('blah', dev.parent_addr)
self.assertEqual({'phys_function': 'blah'}, dev.extra_info)
def test_from_db_obj_pre_1_5_format(self):
ctxt = context.get_admin_context()
fake_dev_pre_1_5 = copy.deepcopy(fake_db_dev_old)
fake_dev_pre_1_5['status'] = fields.PciDeviceStatus.UNAVAILABLE
dev = pci_device.PciDevice._from_db_object(
ctxt, pci_device.PciDevice(), fake_dev_pre_1_5)
self.assertRaises(exception.ObjectActionError,
dev.obj_to_primitive, '1.4')
def test_save_empty_parent_addr(self):
ctxt = context.get_admin_context()
dev = pci_device.PciDevice._from_db_object(
@ -476,3 +488,243 @@ class TestPciDeviceListObject(test_objects._LocalTest,
class TestPciDeviceListObjectRemote(test_objects._RemoteTest,
_TestPciDeviceListObject):
pass
class _TestSRIOVPciDeviceObject(object):
def _create_pci_devices(self, vf_product_id=1515, pf_product_id=1528,
num_pfs=2, num_vfs=8):
self.sriov_pf_devices = []
for dev in range(num_pfs):
pci_dev = {'compute_node_id': 1,
'address': '0000:81:00.%d' % dev,
'vendor_id': '8086',
'product_id': '%d' % pf_product_id,
'status': 'available',
'request_id': None,
'dev_type': fields.PciDeviceType.SRIOV_PF,
'parent_addr': None,
'numa_node': 0}
pci_dev_obj = objects.PciDevice.create(None, pci_dev)
pci_dev_obj.id = num_pfs + 81
self.sriov_pf_devices.append(pci_dev_obj)
self.sriov_vf_devices = []
for dev in range(num_vfs):
pci_dev = {'compute_node_id': 1,
'address': '0000:81:10.%d' % dev,
'vendor_id': '8086',
'product_id': '%d' % vf_product_id,
'status': 'available',
'request_id': None,
'dev_type': fields.PciDeviceType.SRIOV_VF,
'parent_addr': '0000:81:00.%d' % int(dev / 4),
'numa_node': 0}
pci_dev_obj = objects.PciDevice.create(None, pci_dev)
pci_dev_obj.id = num_vfs + 1
self.sriov_vf_devices.append(pci_dev_obj)
def _create_fake_instance(self):
self.inst = instance.Instance()
self.inst.uuid = 'fake-inst-uuid'
self.inst.pci_devices = pci_device.PciDeviceList()
def _create_fake_pci_device(self, ctxt=None):
if not ctxt:
ctxt = context.get_admin_context()
self.mox.StubOutWithMock(db, 'pci_device_get_by_addr')
db.pci_device_get_by_addr(ctxt, 1, 'a').AndReturn(fake_db_dev)
self.mox.ReplayAll()
self.pci_device = pci_device.PciDevice.get_by_dev_addr(ctxt, 1, 'a')
def _fake_get_by_parent_address(self, ctxt, node_id, addr):
vf_devs = []
for dev in self.sriov_vf_devices:
if dev.parent_addr == addr:
vf_devs.append(dev)
return vf_devs
def _fake_pci_device_get_by_addr(self, ctxt, id, addr):
for dev in self.sriov_pf_devices:
if dev.address == addr:
return dev
def test_claim_PF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
devobj.claim(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.CLAIMED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 0)
# check if the all the dependants are UNCLAIMABLE
self.assertTrue(all(
[dev.status == fields.PciDeviceStatus.UNCLAIMABLE for
dev in self._fake_get_by_parent_address(None, None,
self.sriov_pf_devices[0].address)]))
def test_claim_VF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
devobj.claim(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.CLAIMED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 0)
# check if parent device status has been changed to UNCLAIMABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.UNCLAIMABLE, parent.status)
def test_allocate_PF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
devobj.claim(self.inst)
devobj.allocate(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.ALLOCATED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 1)
# check if the all the dependants are UNAVAILABLE
self.assertTrue(all(
[dev.status == fields.PciDeviceStatus.UNAVAILABLE for
dev in self._fake_get_by_parent_address(None, None,
self.sriov_pf_devices[0].address)]))
def test_allocate_VF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
devobj.claim(self.inst)
devobj.allocate(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.ALLOCATED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 1)
# check if parent device status has been changed to UNAVAILABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.UNAVAILABLE, parent.status)
def test_claim_PF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
self.sriov_vf_devices[0].status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDeviceVFInvalidStatus,
devobj.claim, self.inst)
def test_claim_VF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
parent.status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDevicePFInvalidStatus,
devobj.claim, self.inst)
def test_allocate_PF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
self.sriov_vf_devices[0].status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDeviceVFInvalidStatus,
devobj.allocate, self.inst)
def test_allocate_VF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
parent.status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDevicePFInvalidStatus,
devobj.allocate, self.inst)
def test_free_allocated_PF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
devobj.claim(self.inst)
devobj.allocate(self.inst)
devobj.free(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.AVAILABLE)
self.assertIsNone(devobj.instance_uuid)
# check if the all the dependants are AVAILABLE
self.assertTrue(all(
[dev.status == fields.PciDeviceStatus.AVAILABLE for
dev in self._fake_get_by_parent_address(None, None,
self.sriov_pf_devices[0].address)]))
def test_free_allocated_VF(self):
self._create_fake_instance()
with test.nested(
mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr),
mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address)):
self._create_pci_devices()
vf = self.sriov_vf_devices[0]
dependents = self._fake_get_by_parent_address(None, None,
vf.parent_addr)
for devobj in dependents:
devobj.claim(self.inst)
devobj.allocate(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.ALLOCATED)
for devobj in dependents[:3]:
devobj.free(self.inst)
# check if parent device status is still UNAVAILABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.UNAVAILABLE,
parent.status)
for devobj in dependents[3:]:
devobj.free(self.inst)
# check if parent device status is now AVAILABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.AVAILABLE,
parent.status)
class TestSRIOVPciDeviceListObject(test_objects._LocalTest,
_TestSRIOVPciDeviceObject):
pass
class TestSRIOVPciDeviceListObjectRemote(test_objects._RemoteTest,
_TestSRIOVPciDeviceObject):
pass

View File

@ -370,6 +370,20 @@ class PciDevTrackerTestCase(test.NoDBTestCase):
set([dev.address for dev in free_devs]),
set(['0000:00:00.1', '0000:00:00.2', '0000:00:00.3']))
@mock.patch('nova.objects.InstancePCIRequests.get_by_instance')
def test_free_devices(self, mock_get):
self._create_pci_requests_object(mock_get,
[{'count': 1, 'spec': [{'vendor_id': 'v'}]}])
self.tracker.claim_instance(None, self.inst)
self.tracker.update_pci_for_instance(None, self.inst, sign=1)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 2)
self.tracker.free_instance(None, self.inst)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 3)
class PciGetInstanceDevs(test.TestCase):
def setUp(self):