openstack/nova
Code Issues Proposed changes

Use the correct mdev allocated from the pGPU

Browse Source 
In the next change in the series, we will use nested resource providers
for pGPUs. Accordingly, the allocation will be made on a child RP, not the
root RP.
This change aims to prepare this by allocating the right mediated device from the
same physical device.

Note that given we don't pass allocations when initializing the virt driver, in
case some mdevs are missing, we will recreate them blindly. Next change will
fix it.

Part of blueprint reshape-provider-tree
Part of blueprint vgpu-stein

Change-Id: Ia8f43199f70da9c5867acd9a76c43ca8c2c11c80
This commit is contained in:
Sylvain Bauza
2019-02-12 11:22:46 +01:00
parent 33f367ec2f
commit 2d00da78d5
2 changed files with 142 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
nova/tests/unit/virt/libvirt/test_driver.py
View File

@ -20603,11 +20603,42 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
self.assertIsNone(drvr._allocate_mdevs(allocations=allocations))
def _get_fake_provider_tree_with_vgpu(self):
"""Returns a fake ProviderTree with VGPU inventory on two children RPs
with one with a correct name and the other one wrong.
The child provider is named rp1 and its UUID is uuids.rp1.
"""
cn_rp = dict(
uuid=uuids.cn,
name='cn',
)
vgpu_rp_inv = {
orc.VGPU: {
'total': 1,
'min_unit': 1,
'max_unit': 1,
'step_size': 1,
}
}
pt = provider_tree.ProviderTree()
pt.new_root(cn_rp['name'], cn_rp['uuid'], generation=0)
# Create a first child with a correct naming attribute
pt.new_child(cn_rp['name'] + '_' + 'pci_0000_06_00_0', cn_rp['uuid'],
uuid=uuids.rp1, generation=0)
pt.update_inventory(uuids.rp1, vgpu_rp_inv)
# Create a second child with a bad naming convention
pt.new_child('oops_I_did_it_again', cn_rp['uuid'],
uuid=uuids.rp2, generation=0)
pt.update_inventory(uuids.rp2, vgpu_rp_inv)
return pt
@mock.patch.object(libvirt_driver.LibvirtDriver,
'_get_existing_mdevs_not_assigned')
def test_allocate_mdevs_with_available_mdevs(self, get_unassigned_mdevs):
self.flags(enabled_vgpu_types=['nvidia-11'], group='devices')
allocations = {
'rp1': {
uuids.rp1: {
'resources': {
orc.VGPU: 1,
}
@ -20615,8 +20646,12 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
}
get_unassigned_mdevs.return_value = set([uuids.mdev1])
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
# Mock the fact update_provider_tree() should have run
drvr.provider_tree = self._get_fake_provider_tree_with_vgpu()
self.assertEqual([uuids.mdev1],
drvr._allocate_mdevs(allocations=allocations))
get_unassigned_mdevs.assert_called_once_with(['nvidia-11'],
'pci_0000_06_00_0')
@mock.patch.object(nova.privsep.libvirt, 'create_mdev')
@mock.patch.object(libvirt_driver.LibvirtDriver,
@ -20629,7 +20664,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
privsep_create_mdev):
self.flags(enabled_vgpu_types=['nvidia-11'], group='devices')
allocations = {
'rp1': {
uuids.rp1: {
'resources': {
orc.VGPU: 1,
}
@ -20646,6 +20681,8 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
}]
privsep_create_mdev.return_value = uuids.mdev1
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
# Mock the fact update_provider_tree() should have run
drvr.provider_tree = self._get_fake_provider_tree_with_vgpu()
self.assertEqual([uuids.mdev1],
drvr._allocate_mdevs(allocations=allocations))
privsep_create_mdev.assert_called_once_with("0000:06:00.0",
@ -20663,7 +20700,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
privsep_create_mdev):
self.flags(enabled_vgpu_types=['nvidia-11'], group='devices')
allocations = {
'rp1': {
uuids.rp1: {
'resources': {
orc.VGPU: 1,
}
@ -20681,6 +20718,36 @@ class LibvirtDriverTestCase(test.NoDBTestCase, TraitsComparisonMixin):
}
}]
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
# Mock the fact update_provider_tree() should have run
drvr.provider_tree = self._get_fake_provider_tree_with_vgpu()
self.assertRaises(exception.ComputeResourcesUnavailable,
drvr._allocate_mdevs, allocations=allocations)
def test_allocate_mdevs_with_no_idea_of_the_provider(self):
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
# Mock the fact update_provider_tree() should have run
drvr.provider_tree = self._get_fake_provider_tree_with_vgpu()
# Test that the allocated RP doesn't exist in the tree
allocations = {
uuids.wrong_rp: {
'resources': {
orc.VGPU: 1,
}
}
}
self.assertRaises(exception.ComputeResourcesUnavailable,
drvr._allocate_mdevs, allocations=allocations)
# Test that we were unable to guess the RP name
allocations = {
uuids.rp2: {
'resources': {
orc.VGPU: 1,
}
}
}
# Remember, rp2 has a wrong naming convention
self.assertRaises(exception.ComputeResourcesUnavailable,
drvr._allocate_mdevs, allocations=allocations)

79
nova/virt/libvirt/driver.py
View File

@ -29,6 +29,7 @@ import binascii
import collections
from collections import deque
import contextlib
import copy
import errno
import functools
import glob
@ -381,6 +382,12 @@ class LibvirtDriver(driver.ComputeDriver):
# avoid any re-calculation when computing resources.
self._reserved_hugepages = hardware.numa_get_reserved_huge_pages()
# Copy of the compute service ProviderTree object that is updated
# every time update_provider_tree() is called.
# NOTE(sbauza): We only want a read-only cache, this attribute is not
# intended to be updatable directly
self.provider_tree = None
def _get_volume_drivers(self):
driver_registry = dict()
@ -573,6 +580,10 @@ class LibvirtDriver(driver.ComputeDriver):
"""Recreate assigned mdevs that could have disappeared if we reboot
the host.
"""
# FIXME(sbauza): We blindly recreate mediated devices without checking
# which ResourceProvider was allocated for the instance so it would use
# another pGPU.
# TODO(sbauza): Pass all instances' allocations here.
mdevs = self._get_all_assigned_mediated_devices()
requested_types = self._get_supported_vgpu_types()
for (mdev_uuid, instance_uuid) in six.iteritems(mdevs):
@ -6139,26 +6150,34 @@ class LibvirtDriver(driver.ComputeDriver):
vgpu_allocations[rp] = {'resources': {RC_VGPU: res[RC_VGPU]}}
return vgpu_allocations
def _get_existing_mdevs_not_assigned(self, requested_types=None):
def _get_existing_mdevs_not_assigned(self, requested_types=None,
parent=None):
"""Returns the already created mediated devices that are not assigned
to a guest yet.
:param requested_types: Filter out the result for only mediated devices
having those types.
:param parent: Filter out result for only mdevs from the parent device.
"""
allocated_mdevs = self._get_all_assigned_mediated_devices()
mdevs = self._get_mediated_devices(requested_types)
available_mdevs = set([mdev["uuid"]
for mdev in mdevs]) - set(allocated_mdevs)
available_mdevs = set()
for mdev in mdevs:
if parent is None or mdev['parent'] == parent:
available_mdevs.add(mdev["uuid"])
available_mdevs -= set(allocated_mdevs)
return available_mdevs
def _create_new_mediated_device(self, requested_types, uuid=None):
def _create_new_mediated_device(self, requested_types, uuid=None,
parent=None):
"""Find a physical device that can support a new mediated device and
create it.
:param requested_types: Filter only capable devices supporting those
types.
:param uuid: The possible mdev UUID we want to create again
:param parent: Only create a mdev for this device
:returns: the newly created mdev UUID or None if not possible
"""
@ -6173,6 +6192,11 @@ class LibvirtDriver(driver.ComputeDriver):
if device['types'][asked_type]['availableInstances'] > 0:
# That physical GPU has enough room for a new mdev
dev_name = device['dev_id']
# the parent attribute can be None
if parent is not None and dev_name != parent:
# The device is not the one that was called, not creating
# the mdev
continue
# We need the PCI address, not the libvirt name
# The libvirt name is like 'pci_0000_84_00_0'
pci_addr = "{}:{}:{}.{}".format(*dev_name[4:].split('_'))
@ -6209,13 +6233,49 @@ class LibvirtDriver(driver.ComputeDriver):
LOG.warning('More than one allocation was passed over to libvirt '
'while at the moment libvirt only supports one. Only '
'the first allocation will be looked up.')
alloc = six.next(six.itervalues(vgpu_allocations))
rp_uuid, alloc = six.next(six.iteritems(vgpu_allocations))
vgpus_asked = alloc['resources'][orc.VGPU]
# Find if we allocated against a specific pGPU (and then the allocation
# is made against a child RP) or any pGPU (in case the VGPU inventory
# is still on the root RP)
try:
allocated_rp = self.provider_tree.data(rp_uuid)
except ValueError:
# The provider doesn't exist, return a better understandable
# exception
raise exception.ComputeResourcesUnavailable(
reason='vGPU resource is not available')
# TODO(sbauza): Remove this conditional in Train once all VGPU
# inventories are related to a child RP
if allocated_rp.parent_uuid is None:
# We are on a root RP
parent_device = None
else:
rp_name = allocated_rp.name
# There can be multiple roots, we need to find the root name
# to guess the physical device name
roots = self.provider_tree.roots
for root in roots:
if rp_name.startswith(root.name + '_'):
# The RP name convention is :
# root_name + '_' + parent_device
parent_device = rp_name[len(root.name) + 1:]
break
else:
LOG.warning("pGPU device name %(name)s can't be guessed from "
"the ProviderTree "
"roots %(roots)s", {'name': rp_name,
'roots': roots})
# We f... have no idea what was the parent device
# If we can't find devices having available VGPUs, just raise
raise exception.ComputeResourcesUnavailable(
reason='vGPU resource is not available')
requested_types = self._get_supported_vgpu_types()
# Which mediated devices are created but not assigned to a guest ?
mdevs_available = self._get_existing_mdevs_not_assigned(
requested_types)
requested_types, parent_device)
chosen_mdevs = []
for c in six.moves.range(vgpus_asked):
@ -6224,7 +6284,8 @@ class LibvirtDriver(driver.ComputeDriver):
# Take the first available mdev
chosen_mdev = mdevs_available.pop()
else:
chosen_mdev = self._create_new_mediated_device(requested_types)
chosen_mdev = self._create_new_mediated_device(
requested_types, parent=parent_device)
if not chosen_mdev:
# If we can't find devices having available VGPUs, just raise
raise exception.ComputeResourcesUnavailable(
@ -6560,6 +6621,10 @@ class LibvirtDriver(driver.ComputeDriver):
provider_tree.add_traits(nodename, *traits_to_add)
provider_tree.remove_traits(nodename, *traits_to_remove)
# Now that we updated the ProviderTree, we want to store it locally
# so that spawn() or other methods can access it thru a getter
self.provider_tree = copy.deepcopy(provider_tree)
def get_available_resource(self, nodename):
"""Retrieve resource information.