openstack/nova
Code Issues Proposed changes

Merge "pci: changing the claiming and allocation logic for PF/VF assignment"

Browse Source
This commit is contained in:
Jenkins 2016-01-22 12:21:44 +00:00 committed by Gerrit Code Review
commit 4dbc6abef9
7 changed files with 419 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
nova/exception.py
View File

@ -1736,6 +1736,19 @@ class PciDeviceInvalidStatus(Invalid):
"instead of %(hopestatus)s")
class PciDeviceVFInvalidStatus(Invalid):
msg_fmt = _(
"Not all Virtual Functions of PF %(compute_node_id)s:%(address)s "
"are free.")
class PciDevicePFInvalidStatus(Invalid):
msg_fmt = _(
"Physical Function %(compute_node_id)s:%(address)s, related to VF"
" %(compute_node_id)s:%(vf_address)s is %(status)s "
"instead of %(hopestatus)s")
class PciDeviceInvalidOwner(Invalid):
msg_fmt = _(
"PCI device %(compute_node_id)s:%(address)s is owned by %(owner)s "

5
nova/objects/fields.py
View File

@ -442,8 +442,11 @@ class PciDeviceStatus(Enum):
ALLOCATED = "allocated"
REMOVED = "removed" # The device has been hot-removed and not yet deleted
DELETED = "deleted" # The device is marked not available/deleted.
UNCLAIMABLE = "unclaimable"
UNAVAILABLE = "unavailable"
ALL = (AVAILABLE, CLAIMED, ALLOCATED, REMOVED, DELETED)
ALL = (AVAILABLE, CLAIMED, ALLOCATED, REMOVED, DELETED, UNAVAILABLE,
UNCLAIMABLE)
def __init__(self):
super(PciDeviceStatus, self).__init__(

131
nova/objects/pci_device.py
View File

@ -15,6 +15,7 @@
import copy
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import versionutils
@ -25,6 +26,8 @@ from nova import objects
from nova.objects import base
from nova.objects import fields
LOG = logging.getLogger(__name__)
def compare_pci_device_attributes(obj_a, obj_b):
pci_ignore_fields = base.NovaPersistentObject.fields.keys()
@ -87,7 +90,8 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
# Version 1.2: added request_id field
# Version 1.3: Added field to represent PCI device NUMA node
# Version 1.4: Added parent_addr field
VERSION = '1.4'
# Version 1.5: Added 2 new device statuses: UNCLAIMABLE and UNAVAILABLE
VERSION = '1.5'
fields = {
'id': fields.IntegerField(),
@ -129,6 +133,15 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
extra_info = primitive.get('extra_info', {})
extra_info['phys_function'] = primitive['parent_addr']
del primitive['parent_addr']
if target_version < (1, 5) and 'parent_addr' in primitive:
added_statuses = (fields.PciDeviceStatus.UNCLAIMABLE,
fields.PciDeviceStatus.UNAVAILABLE)
status = primitive['status']
if status in added_statuses:
raise exception.ObjectActionError(
action='obj_make_compatible',
reason='status=%s not supported in version %s' % (
status, target_version))
def update_device(self, dev_dict):
"""Sync the content from device dictionary to device object.
@ -244,18 +257,73 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
self.address, updates)
self._from_db_object(self._context, self, db_pci)
@staticmethod
def _bulk_update_status(dev_list, status):
for dev in dev_list:
dev.status = status
def claim(self, instance):
if self.status != fields.PciDeviceStatus.AVAILABLE:
raise exception.PciDeviceInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.address, status=self.status,
hopestatus=[fields.PciDeviceStatus.AVAILABLE])
if self.dev_type == fields.PciDeviceType.SRIOV_PF:
# Update PF status to CLAIMED if all of it dependants are free
# and set their status to UNCLAIMABLE
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.address)
if not all([vf.is_available() for vf in vfs_list]):
raise exception.PciDeviceVFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.address)
self._bulk_update_status(vfs_list,
fields.PciDeviceStatus.UNCLAIMABLE)
elif self.dev_type == fields.PciDeviceType.SRIOV_VF:
# Update VF status to CLAIMED if it's parent has not been
# previuosly allocated or claimed
# When claiming/allocating a VF, it's parent PF becomes
# unclaimable/unavailable. Therefore, it is expected to find the
# parent PF in an unclaimable/unavailable state for any following
# claims to a sibling VF
parent_ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.UNCLAIMABLE,
fields.PciDeviceStatus.UNAVAILABLE)
try:
parent = self.get_by_dev_addr(self._context,
self.compute_node_id,
self.parent_addr)
if parent.status not in parent_ok_statuses:
raise exception.PciDevicePFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.parent_addr, status=self.status,
vf_address=self.address,
hopestatus=parent_ok_statuses)
# Set PF status
if parent.status == fields.PciDeviceStatus.AVAILABLE:
parent.status = fields.PciDeviceStatus.UNCLAIMABLE
except exception.PciDeviceNotFound:
LOG.debug('Physical function addr: %(pf_addr)s parent of '
'VF addr: %(vf_addr)s was not found',
{'pf_addr': self.parent_addr,
'vf_addr': self.address})
self.status = fields.PciDeviceStatus.CLAIMED
self.instance_uuid = instance['uuid']
def allocate(self, instance):
ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.CLAIMED)
parent_ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.UNCLAIMABLE,
fields.PciDeviceStatus.UNAVAILABLE)
dependatns_ok_statuses = (fields.PciDeviceStatus.AVAILABLE,
fields.PciDeviceStatus.UNCLAIMABLE)
if self.status not in ok_statuses:
raise exception.PciDeviceInvalidStatus(
compute_node_id=self.compute_node_id,
@ -267,6 +335,37 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
compute_node_id=self.compute_node_id,
address=self.address, owner=self.instance_uuid,
hopeowner=instance['uuid'])
if self.dev_type == fields.PciDeviceType.SRIOV_PF:
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.address)
if not all([vf.status in dependatns_ok_statuses for
vf in vfs_list]):
raise exception.PciDeviceVFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.address)
self._bulk_update_status(vfs_list,
fields.PciDeviceStatus.UNAVAILABLE)
elif (self.dev_type == fields.PciDeviceType.SRIOV_VF):
try:
parent = self.get_by_dev_addr(self._context,
self.compute_node_id,
self.parent_addr)
if parent.status not in parent_ok_statuses:
raise exception.PciDevicePFInvalidStatus(
compute_node_id=self.compute_node_id,
address=self.parent_addr, status=self.status,
vf_address=self.address,
hopestatus=parent_ok_statuses)
# Set PF status
parent.status = fields.PciDeviceStatus.UNAVAILABLE
except exception.PciDeviceNotFound:
LOG.debug('Physical function addr: %(pf_addr)s parent of '
'VF addr: %(vf_addr)s was not found',
{'pf_addr': self.parent_addr,
'vf_addr': self.address})
self.status = fields.PciDeviceStatus.ALLOCATED
self.instance_uuid = instance['uuid']
@ -293,6 +392,7 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
def free(self, instance=None):
ok_statuses = (fields.PciDeviceStatus.ALLOCATED,
fields.PciDeviceStatus.CLAIMED)
free_devs = []
if self.status not in ok_statuses:
raise exception.PciDeviceInvalidStatus(
compute_node_id=self.compute_node_id,
@ -303,8 +403,36 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
compute_node_id=self.compute_node_id,
address=self.address, owner=self.instance_uuid,
hopeowner=instance['uuid'])
if self.dev_type == fields.PciDeviceType.SRIOV_PF:
# Set all PF dependants status to AVAILABLE
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.address)
self._bulk_update_status(vfs_list,
fields.PciDeviceStatus.AVAILABLE)
free_devs.extend(vfs_list)
if self.dev_type == fields.PciDeviceType.SRIOV_VF:
# Set PF status to AVAILABLE if all of it's VFs are free
vfs_list = objects.PciDeviceList.get_by_parent_address(
self._context,
self.compute_node_id,
self.parent_addr)
if all([vf.is_available() for vf in vfs_list if vf.id != self.id]):
try:
parent = self.get_by_dev_addr(self._context,
self.compute_node_id,
self.parent_addr)
parent.status = fields.PciDeviceStatus.AVAILABLE
free_devs.append(parent)
except exception.PciDeviceNotFound:
LOG.debug('Physical function addr: %(pf_addr)s parent of '
'VF addr: %(vf_addr)s was not found',
{'pf_addr': self.parent_addr,
'vf_addr': self.address})
old_status = self.status
self.status = fields.PciDeviceStatus.AVAILABLE
free_devs.append(self)
self.instance_uuid = None
self.request_id = None
if old_status == fields.PciDeviceStatus.ALLOCATED and instance:
@ -316,6 +444,7 @@ class PciDevice(base.NovaPersistentObject, base.NovaObject):
instance['pci_devices'].remove(existed)
else:
instance.pci_devices.objects.remove(existed)
return free_devs
def is_available(self):
return self.status == fields.PciDeviceStatus.AVAILABLE

5
nova/pci/manager.py
View File

@ -215,11 +215,12 @@ class PciDevTracker(object):
return None
def _free_device(self, dev, instance=None):
dev.free(instance)
freed_devs = dev.free(instance)
stale = self.stale.pop(dev.address, None)
if stale:
dev.update_device(stale)
self.stats.add_device(dev)
for dev in freed_devs:
self.stats.add_device(dev)
def _free_instance(self, instance):
# Note(yjiang5): When an instance is resized, the devices in the

2
nova/tests/unit/objects/test_objects.py
View File

@ -1173,7 +1173,7 @@ object_data = {
'NetworkList': '1.2-69eca910d8fa035dfecd8ba10877ee59',
'NetworkRequest': '1.1-7a3e4ca2ce1e7b62d8400488f2f2b756',
'NetworkRequestList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e',
'PciDevice': '1.4-4f54e80054bbb6414e17eb9babc97a44',
'PciDevice': '1.5-0d5abe5c91645b8469eb2a93fc53f932',
'PciDeviceList': '1.3-52ff14355491c8c580bdc0ba34c26210',
'PciDevicePool': '1.1-3f5ddc3ff7bfa14da7f6c7e9904cc000',
'PciDevicePoolList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e',

256
nova/tests/unit/objects/test_pci_device.py
View File

@ -25,6 +25,7 @@ from nova import objects
from nova.objects import fields
from nova.objects import instance
from nova.objects import pci_device
from nova import test
from nova.tests.unit.objects import test_objects
dev_dict = {
@ -33,6 +34,8 @@ dev_dict = {
'product_id': 'p',
'vendor_id': 'v',
'numa_node': 0,
'dev_type': fields.PciDeviceType.STANDARD,
'parent_addr': None,
'status': fields.PciDeviceStatus.AVAILABLE}
@ -122,7 +125,7 @@ class _TestPciDeviceObject(object):
self.assertEqual(self.pci_device.obj_what_changed(),
set(['compute_node_id', 'product_id', 'vendor_id',
'numa_node', 'status', 'address', 'extra_info',
'parent_addr']))
'dev_type', 'parent_addr']))
def test_pci_device_extra_info(self):
self.dev_dict = copy.copy(dev_dict)
@ -135,7 +138,7 @@ class _TestPciDeviceObject(object):
self.assertEqual(self.pci_device.obj_what_changed(),
set(['compute_node_id', 'address', 'product_id',
'vendor_id', 'numa_node', 'status',
'parent_addr', 'extra_info']))
'extra_info', 'dev_type', 'parent_addr']))
def test_update_device(self):
self.pci_device = pci_device.PciDevice.create(None, dev_dict)
@ -181,6 +184,15 @@ class _TestPciDeviceObject(object):
self.assertEqual('blah', dev.parent_addr)
self.assertEqual({'phys_function': 'blah'}, dev.extra_info)
def test_from_db_obj_pre_1_5_format(self):
ctxt = context.get_admin_context()
fake_dev_pre_1_5 = copy.deepcopy(fake_db_dev_old)
fake_dev_pre_1_5['status'] = fields.PciDeviceStatus.UNAVAILABLE
dev = pci_device.PciDevice._from_db_object(
ctxt, pci_device.PciDevice(), fake_dev_pre_1_5)
self.assertRaises(exception.ObjectActionError,
dev.obj_to_primitive, '1.4')
def test_save_empty_parent_addr(self):
ctxt = context.get_admin_context()
dev = pci_device.PciDevice._from_db_object(
@ -476,3 +488,243 @@ class TestPciDeviceListObject(test_objects._LocalTest,
class TestPciDeviceListObjectRemote(test_objects._RemoteTest,
_TestPciDeviceListObject):
pass
class _TestSRIOVPciDeviceObject(object):
def _create_pci_devices(self, vf_product_id=1515, pf_product_id=1528,
num_pfs=2, num_vfs=8):
self.sriov_pf_devices = []
for dev in range(num_pfs):
pci_dev = {'compute_node_id': 1,
'address': '0000:81:00.%d' % dev,
'vendor_id': '8086',
'product_id': '%d' % pf_product_id,
'status': 'available',
'request_id': None,
'dev_type': fields.PciDeviceType.SRIOV_PF,
'parent_addr': None,
'numa_node': 0}
pci_dev_obj = objects.PciDevice.create(None, pci_dev)
pci_dev_obj.id = num_pfs + 81
self.sriov_pf_devices.append(pci_dev_obj)
self.sriov_vf_devices = []
for dev in range(num_vfs):
pci_dev = {'compute_node_id': 1,
'address': '0000:81:10.%d' % dev,
'vendor_id': '8086',
'product_id': '%d' % vf_product_id,
'status': 'available',
'request_id': None,
'dev_type': fields.PciDeviceType.SRIOV_VF,
'parent_addr': '0000:81:00.%d' % int(dev / 4),
'numa_node': 0}
pci_dev_obj = objects.PciDevice.create(None, pci_dev)
pci_dev_obj.id = num_vfs + 1
self.sriov_vf_devices.append(pci_dev_obj)
def _create_fake_instance(self):
self.inst = instance.Instance()
self.inst.uuid = 'fake-inst-uuid'
self.inst.pci_devices = pci_device.PciDeviceList()
def _create_fake_pci_device(self, ctxt=None):
if not ctxt:
ctxt = context.get_admin_context()
self.mox.StubOutWithMock(db, 'pci_device_get_by_addr')
db.pci_device_get_by_addr(ctxt, 1, 'a').AndReturn(fake_db_dev)
self.mox.ReplayAll()
self.pci_device = pci_device.PciDevice.get_by_dev_addr(ctxt, 1, 'a')
def _fake_get_by_parent_address(self, ctxt, node_id, addr):
vf_devs = []
for dev in self.sriov_vf_devices:
if dev.parent_addr == addr:
vf_devs.append(dev)
return vf_devs
def _fake_pci_device_get_by_addr(self, ctxt, id, addr):
for dev in self.sriov_pf_devices:
if dev.address == addr:
return dev
def test_claim_PF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
devobj.claim(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.CLAIMED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 0)
# check if the all the dependants are UNCLAIMABLE
self.assertTrue(all(
[dev.status == fields.PciDeviceStatus.UNCLAIMABLE for
dev in self._fake_get_by_parent_address(None, None,
self.sriov_pf_devices[0].address)]))
def test_claim_VF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
devobj.claim(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.CLAIMED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 0)
# check if parent device status has been changed to UNCLAIMABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.UNCLAIMABLE, parent.status)
def test_allocate_PF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
devobj.claim(self.inst)
devobj.allocate(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.ALLOCATED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 1)
# check if the all the dependants are UNAVAILABLE
self.assertTrue(all(
[dev.status == fields.PciDeviceStatus.UNAVAILABLE for
dev in self._fake_get_by_parent_address(None, None,
self.sriov_pf_devices[0].address)]))
def test_allocate_VF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
devobj.claim(self.inst)
devobj.allocate(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.ALLOCATED)
self.assertEqual(devobj.instance_uuid,
self.inst.uuid)
self.assertEqual(len(self.inst.pci_devices), 1)
# check if parent device status has been changed to UNAVAILABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.UNAVAILABLE, parent.status)
def test_claim_PF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
self.sriov_vf_devices[0].status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDeviceVFInvalidStatus,
devobj.claim, self.inst)
def test_claim_VF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
parent.status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDevicePFInvalidStatus,
devobj.claim, self.inst)
def test_allocate_PF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
self.sriov_vf_devices[0].status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDeviceVFInvalidStatus,
devobj.allocate, self.inst)
def test_allocate_VF_fail(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr):
self._create_pci_devices()
devobj = self.sriov_vf_devices[0]
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
parent.status = fields.PciDeviceStatus.CLAIMED
self.assertRaises(exception.PciDevicePFInvalidStatus,
devobj.allocate, self.inst)
def test_free_allocated_PF(self):
self._create_fake_instance()
with mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address):
self._create_pci_devices()
devobj = self.sriov_pf_devices[0]
devobj.claim(self.inst)
devobj.allocate(self.inst)
devobj.free(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.AVAILABLE)
self.assertIsNone(devobj.instance_uuid)
# check if the all the dependants are AVAILABLE
self.assertTrue(all(
[dev.status == fields.PciDeviceStatus.AVAILABLE for
dev in self._fake_get_by_parent_address(None, None,
self.sriov_pf_devices[0].address)]))
def test_free_allocated_VF(self):
self._create_fake_instance()
with test.nested(
mock.patch.object(objects.PciDevice, 'get_by_dev_addr',
side_effect=self._fake_pci_device_get_by_addr),
mock.patch.object(objects.PciDeviceList, 'get_by_parent_address',
side_effect=self._fake_get_by_parent_address)):
self._create_pci_devices()
vf = self.sriov_vf_devices[0]
dependents = self._fake_get_by_parent_address(None, None,
vf.parent_addr)
for devobj in dependents:
devobj.claim(self.inst)
devobj.allocate(self.inst)
self.assertEqual(devobj.status,
fields.PciDeviceStatus.ALLOCATED)
for devobj in dependents[:3]:
devobj.free(self.inst)
# check if parent device status is still UNAVAILABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.UNAVAILABLE,
parent.status)
for devobj in dependents[3:]:
devobj.free(self.inst)
# check if parent device status is now AVAILABLE
parent = self._fake_pci_device_get_by_addr(None, None,
devobj.parent_addr)
self.assertTrue(fields.PciDeviceStatus.AVAILABLE,
parent.status)
class TestSRIOVPciDeviceListObject(test_objects._LocalTest,
_TestSRIOVPciDeviceObject):
pass
class TestSRIOVPciDeviceListObjectRemote(test_objects._RemoteTest,
_TestSRIOVPciDeviceObject):
pass

14
nova/tests/unit/pci/test_manager.py
View File

@ -370,6 +370,20 @@ class PciDevTrackerTestCase(test.NoDBTestCase):
set([dev.address for dev in free_devs]),
set(['0000:00:00.1', '0000:00:00.2', '0000:00:00.3']))
@mock.patch('nova.objects.InstancePCIRequests.get_by_instance')
def test_free_devices(self, mock_get):
self._create_pci_requests_object(mock_get,
[{'count': 1, 'spec': [{'vendor_id': 'v'}]}])
self.tracker.claim_instance(None, self.inst)
self.tracker.update_pci_for_instance(None, self.inst, sign=1)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 2)
self.tracker.free_instance(None, self.inst)
free_devs = self.tracker.pci_stats.get_free_devs()
self.assertEqual(len(free_devs), 3)
class PciGetInstanceDevs(test.TestCase):
def setUp(self):