openstack
/
nova
Code Issues Proposed changes

Fix migration with remote-managed ports & add FT 

`binding:profile` updates are handled differently for migration from
instance creation which was not taken into account previously. Relevant
fields (card_serial_number, pf_mac_address, vf_num) are now added to the
`binding:profile` after a new remote-managed PCI device is determined at
the destination node.

Likewise, there is special handling for the unshelve operation which is
fixed too.

Func testing:

* Allow the generated device XML to contain the PCI VPD capability;
* Add test cases for basic operations on instances with remote-managed
  ports (tunnel or physical);
* Add a live migration test case similar to how it is done for
  non-remote-managed SR-IOV ports but taking remote-managed port related
  specifics into account;
* Add evacuate, shelve/unshelve, cold migration test cases.

Change-Id: I9a1532e9a98f89db69b9ae3b41b06318a43519b3
This commit is contained in:
Dmitrii Shcherbakov 2022-02-18 21:27:07 +03:00
parent 3f274c65cc
commit 3fd7e94893
7 changed files with 739 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
nova/compute/manager.py
View File

@ -84,6 +84,7 @@ from nova.objects import fields
from nova.objects import instance as obj_instance
from nova.objects import migrate_data as migrate_data_obj
from nova.pci import request as pci_req_module
from nova.pci import utils as pci_utils
from nova.pci import whitelist
from nova import safe_utils
from nova.scheduler.client import query
@ -10885,6 +10886,17 @@ class ComputeManager(manager.Manager):
profile['pci_slot'] = pci_dev.address
profile['pci_vendor_info'] = ':'.join([pci_dev.vendor_id,
pci_dev.product_id])
if profile.get('card_serial_number'):
# Assume it is there since Nova makes sure that PCI devices
# tagged as remote-managed have a serial in PCI VPD.
profile['card_serial_number'] = pci_dev.card_serial_number
if profile.get('pf_mac_address'):
profile['pf_mac_address'] = pci_utils.get_mac_by_pci_address(
pci_dev.parent_addr)
if profile.get('vf_num'):
profile['vf_num'] = pci_utils.get_vf_num_by_pci_address(
pci_dev.address)
mig_vif.profile = profile
LOG.debug("Updating migrate VIF profile for port %(port_id)s:"
"%(profile)s", {'port_id': port_id,

7
nova/compute/resource_tracker.py
View File

@ -328,7 +328,12 @@ class ResourceTracker(object):
migration_id=migration.id,
old_numa_topology=instance.numa_topology,
new_numa_topology=claim.claimed_numa_topology,
old_pci_devices=instance.pci_devices,
# NOTE(gibi): the _update_usage_from_migration call below appends
# the newly claimed pci devices to the instance.pci_devices list
# to keep the migration context independent we need to make a copy
# that list here. We need a deep copy as we need to duplicate
# the instance.pci_devices.objects list
old_pci_devices=copy.deepcopy(instance.pci_devices),
new_pci_devices=claimed_pci_devices,
old_pci_requests=instance.pci_requests,
new_pci_requests=new_pci_requests,

35
nova/network/neutron.py
View File

@ -3663,15 +3663,15 @@ class API:
migration.get('status') == 'reverted')
return instance.migration_context.get_pci_mapping_for_migration(revert)
def _get_port_pci_slot(self, context, instance, port):
"""Find the PCI address of the device corresponding to the port.
def _get_port_pci_dev(self, context, instance, port):
"""Find the PCI device corresponding to the port.
Assumes the port is an SRIOV one.
:param context: The request context.
:param instance: The instance to which the port is attached.
:param port: The Neutron port, as obtained from the Neutron API
JSON form.
:return: The PCI address as a string, or None if unable to find.
:return: The PciDevice object, or None if unable to find.
"""
# Find the port's PCIRequest, or return None
for r in instance.pci_requests.requests:
@ -3691,8 +3691,26 @@ class API:
LOG.debug('No PCI device found for request %s',
request.request_id, instance=instance)
return None
# Return the device's PCI address
return device.address
return device
def _update_port_pci_binding_profile(self, pci_dev, binding_profile):
"""Update the binding profile dict with new PCI device data.
:param pci_dev: The PciDevice object to update the profile with.
:param binding_profile: The dict to update.
"""
binding_profile.update({'pci_slot': pci_dev.address})
if binding_profile.get('card_serial_number'):
binding_profile.update({
'card_serial_number': pci_dev.card_serial_number})
if binding_profile.get('pf_mac_address'):
binding_profile.update({
'pf_mac_address': pci_utils.get_mac_by_pci_address(
pci_dev.parent_addr)})
if binding_profile.get('vf_num'):
binding_profile.update({
'vf_num': pci_utils.get_vf_num_by_pci_address(
pci_dev.address)})
def _update_port_binding_for_instance(
self, context, instance, host, migration=None,
@ -3760,9 +3778,10 @@ class API:
# need to figure out the pci_slot from the InstancePCIRequest
# and PciDevice objects.
else:
pci_slot = self._get_port_pci_slot(context, instance, p)
if pci_slot:
binding_profile.update({'pci_slot': pci_slot})
pci_dev = self._get_port_pci_dev(context, instance, p)
if pci_dev:
self._update_port_pci_binding_profile(pci_dev,
binding_profile)
updates[constants.BINDING_PROFILE] = binding_profile
# NOTE(gibi): during live migration the conductor already sets the

45
nova/tests/fixtures/libvirt.py vendored
View File

@ -275,6 +275,7 @@ class FakePCIDevice(object):
<product id='0x%(prod_id)s'>%(prod_name)s</product>
<vendor id='0x%(vend_id)s'>%(vend_name)s</vendor>
%(capability)s
%(vpd_capability)s
<iommuGroup number='%(iommu_group)d'>
<address domain='0x0000' bus='%(bus)#02x' slot='%(slot)#02x' function='0x%(function)d'/>
</iommuGroup>
@ -293,13 +294,22 @@ class FakePCIDevice(object):
<availableInstances>%(instances)s</availableInstances>
</type>""".strip()) # noqa
vpd_cap_templ = textwrap.dedent("""
<capability type='vpd'>
<name>%(name)s</name>
%(fields)s
</capability>""".strip())
vpd_fields_templ = textwrap.dedent("""
<fields access='%(access)s'>%(section_fields)s</fields>""".strip())
vpd_field_templ = """<%(field_name)s>%(field_value)s</%(field_name)s>"""
is_capable_of_mdevs = False
def __init__(
self, dev_type, bus, slot, function, iommu_group, numa_node, *,
vf_ratio=None, multiple_gpu_types=False, generic_types=False,
parent=None, vend_id=None, vend_name=None, prod_id=None,
prod_name=None, driver_name=None,
prod_name=None, driver_name=None, vpd_fields=None
):
"""Populate pci devices
@ -340,6 +350,8 @@ class FakePCIDevice(object):
self.prod_name = prod_name
self.driver_name = driver_name
self.vpd_fields = vpd_fields
self.generate_xml()
def generate_xml(self, skip_capability=False):
@ -447,6 +459,7 @@ class FakePCIDevice(object):
'prod_name': prod_name,
'driver': driver,
'capability': capability,
'vpd_capability': self.format_vpd_cap(),
'iommu_group': self.iommu_group,
'numa_node': self.numa_node,
'parent': parent,
@ -457,6 +470,30 @@ class FakePCIDevice(object):
if self.numa_node == -1:
self.pci_device = self.pci_device.replace("<numa node='-1'/>", "")
def format_vpd_cap(self):
if not self.vpd_fields:
return ''
fields = []
for access_type in ('readonly', 'readwrite'):
section_fields = []
for field_name, field_value in self.vpd_fields.get(
access_type, {}).items():
section_fields.append(self.vpd_field_templ % {
'field_name': field_name,
'field_value': field_value,
})
if section_fields:
fields.append(
self.vpd_fields_templ % {
'access': access_type,
'section_fields': '\n'.join(section_fields),
}
)
return self.vpd_cap_templ % {
'name': self.vpd_fields.get('name', ''),
'fields': '\n'.join(fields)
}
def XMLDesc(self, flags):
return self.pci_device
@ -572,7 +609,7 @@ class HostPCIDevicesInfo(object):
self, dev_type, bus, slot, function, iommu_group, numa_node,
vf_ratio=None, multiple_gpu_types=False, generic_types=False,
parent=None, vend_id=None, vend_name=None, prod_id=None,
prod_name=None, driver_name=None,
prod_name=None, driver_name=None, vpd_fields=None,
):
pci_dev_name = _get_libvirt_nodedev_name(bus, slot, function)
@ -593,7 +630,9 @@ class HostPCIDevicesInfo(object):
vend_name=vend_name,
prod_id=prod_id,
prod_name=prod_name,
driver_name=driver_name)
driver_name=driver_name,
vpd_fields=vpd_fields,
)
self.devices[pci_dev_name] = dev
return dev

30
nova/tests/functional/libvirt/base.py
View File

@ -316,6 +316,21 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
'binding:vif_type': 'ovs',
'binding:vnic_type': 'normal',
}
network_3_port_2 = {
'id': '132c3875-b175-4b20-8a57-7a76219a13ae',
'network_id': network_3['id'],
'status': 'ACTIVE',
'mac_address': 'd2:0b:fd:99:89:8b',
'fixed_ips': [
{
'ip_address': '192.168.2.7',
'subnet_id': subnet_3['id']
}
],
'binding:vif_details': {},
'binding:vif_type': 'ovs',
'binding:vnic_type': 'remote-managed',
}
network_4_port_1 = {
'id': 'b4cd0b93-2ac8-40a7-9fa4-2cd680ccdf3e',
'network_id': network_4['id'],
@ -361,6 +376,21 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
'binding:vif_type': 'hw_veb',
'binding:vnic_type': 'direct',
}
network_4_port_4 = {
'id': 'a31e381d-41ec-41e4-b5a5-ec4ef705fafa',
'network_id': network_1['id'],
'status': 'ACTIVE',
'mac_address': '71:ce:c7:2b:cd:dd',
'fixed_ips': [
{
'ip_address': '192.168.4.9',
'subnet_id': subnet_4['id']
}
],
'binding:vif_details': {},
'binding:vif_type': 'ovs',
'binding:vnic_type': 'remote-managed',
}
def __init__(self, test):
super(LibvirtNeutronFixture, self).__init__(test)

652
nova/tests/functional/libvirt/test_pci_sriov_servers.py
View File

@ -31,6 +31,7 @@ from nova import context
from nova.network import constants
from nova import objects
from nova.objects import fields
from nova.pci.utils import parse_address
from nova.tests import fixtures as nova_fixtures
from nova.tests.fixtures import libvirt as fakelibvirt
from nova.tests.functional.api import client
@ -72,7 +73,47 @@ class _PCIServersTestBase(base.ServersTestBase):
self.assertEqual(free, len([d for d in devices if d.is_available()]))
class SRIOVServersTest(_PCIServersTestBase):
class _PCIServersWithMigrationTestBase(_PCIServersTestBase):
def setUp(self):
super().setUp()
self.useFixture(fixtures.MonkeyPatch(
'nova.tests.fixtures.libvirt.Domain.migrateToURI3',
self._migrate_stub))
def _migrate_stub(self, domain, destination, params, flags):
"""Stub out migrateToURI3."""
src_hostname = domain._connection.hostname
dst_hostname = urlparse.urlparse(destination).netloc
# In a real live migration, libvirt and QEMU on the source and
# destination talk it out, resulting in the instance starting to exist
# on the destination. Fakelibvirt cannot do that, so we have to
# manually create the "incoming" instance on the destination
# fakelibvirt.
dst = self.computes[dst_hostname]
dst.driver._host.get_connection().createXML(
params['destination_xml'],
'fake-createXML-doesnt-care-about-flags')
src = self.computes[src_hostname]
conn = src.driver._host.get_connection()
# because migrateToURI3 is spawned in a background thread, this method
# does not block the upper nova layers. Because we don't want nova to
# think the live migration has finished until this method is done, the
# last thing we do is make fakelibvirt's Domain.jobStats() return
# VIR_DOMAIN_JOB_COMPLETED.
server = etree.fromstring(
params['destination_xml']
).find('./uuid').text
dom = conn.lookupByUUIDString(server)
dom.complete_job()
class SRIOVServersTest(_PCIServersWithMigrationTestBase):
# TODO(stephenfin): We're using this because we want to be able to force
# the host during scheduling. We should instead look at overriding policy
@ -120,40 +161,6 @@ class SRIOVServersTest(_PCIServersTestBase):
# fixture already stubbed.
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
self.useFixture(fixtures.MonkeyPatch(
'nova.tests.fixtures.libvirt.Domain.migrateToURI3',
self._migrate_stub))
def _migrate_stub(self, domain, destination, params, flags):
"""Stub out migrateToURI3."""
src_hostname = domain._connection.hostname
dst_hostname = urlparse.urlparse(destination).netloc
# In a real live migration, libvirt and QEMU on the source and
# destination talk it out, resulting in the instance starting to exist
# on the destination. Fakelibvirt cannot do that, so we have to
# manually create the "incoming" instance on the destination
# fakelibvirt.
dst = self.computes[dst_hostname]
dst.driver._host.get_connection().createXML(
params['destination_xml'],
'fake-createXML-doesnt-care-about-flags')
src = self.computes[src_hostname]
conn = src.driver._host.get_connection()
# because migrateToURI3 is spawned in a background thread, this method
# does not block the upper nova layers. Because we don't want nova to
# think the live migration has finished until this method is done, the
# last thing we do is make fakelibvirt's Domain.jobStats() return
# VIR_DOMAIN_JOB_COMPLETED.
server = etree.fromstring(
params['destination_xml']
).find('./uuid').text
dom = conn.lookupByUUIDString(server)
dom.complete_job()
def _disable_sriov_in_pf(self, pci_info):
# Check for PF and change the capability from virt_functions
# Delete all the VFs
@ -1686,3 +1693,578 @@ class PCIServersWithPortNUMAPoliciesTest(_PCIServersTestBase):
],
)
self.assertTrue(self.mock_filter.called)
class RemoteManagedServersTest(_PCIServersWithMigrationTestBase):
ADMIN_API = True
microversion = 'latest'
PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in (
# A PF with access to physnet4.
{
'vendor_id': '15b3',
'product_id': 'a2dc',
'physical_network': 'physnet4',
'remote_managed': 'false',
},
# A VF with access to physnet4.
{
'vendor_id': '15b3',
'product_id': '1021',
'physical_network': 'physnet4',
'remote_managed': 'true',
},
# A PF programmed to forward traffic to an overlay network.
{
'vendor_id': '15b3',
'product_id': 'a2d6',
'physical_network': None,
'remote_managed': 'false',
},
# A VF programmed to forward traffic to an overlay network.
{
'vendor_id': '15b3',
'product_id': '101e',
'physical_network': None,
'remote_managed': 'true',
},
)]
PCI_ALIAS = []
NUM_PFS = 1
NUM_VFS = 4
vf_ratio = NUM_VFS // NUM_PFS
# Min Libvirt version that supports working with PCI VPD.
FAKE_LIBVIRT_VERSION = 7_009_000 # 7.9.0
FAKE_QEMU_VERSION = 5_001_000 # 5.1.0
def setUp(self):
super().setUp()
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
self.useFixture(fixtures.MockPatch(
'nova.pci.utils.get_vf_num_by_pci_address',
new=mock.MagicMock(
side_effect=lambda addr: self._get_pci_function_number(addr))))
self.useFixture(fixtures.MockPatch(
'nova.pci.utils.get_mac_by_pci_address',
new=mock.MagicMock(
side_effect=(
lambda addr: {
"0000:80:00.0": "52:54:00:1e:59:42",
"0000:81:00.0": "52:54:00:1e:59:01",
"0000:82:00.0": "52:54:00:1e:59:02",
}.get(addr)
)
)
))
@classmethod
def _get_pci_function_number(cls, pci_addr: str):
"""Get a VF function number based on a PCI address.
Assume that the PCI ARI capability is enabled (slot bits become a part
of a function number).
"""
_, _, slot, function = parse_address(pci_addr)
# The number of PFs is extracted to get a VF number.
return int(slot, 16) + int(function, 16) - cls.NUM_PFS
def start_compute(
self, hostname='test_compute0', host_info=None, pci_info=None,
mdev_info=None, vdpa_info=None,
libvirt_version=None,
qemu_version=None):
if not pci_info:
pci_info = fakelibvirt.HostPCIDevicesInfo(
num_pci=0, num_pfs=0, num_vfs=0)
pci_info.add_device(
dev_type='PF',
bus=0x81,
slot=0x0,
function=0,
iommu_group=42,
numa_node=0,
vf_ratio=self.vf_ratio,
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='a2dc',
prod_name='BlueField-3 integrated ConnectX-7 controller',
driver_name='mlx5_core',
vpd_fields={
'name': 'MT43244 BlueField-3 integrated ConnectX-7',
'readonly': {
'serial_number': 'MT0000X00001',
},
}
)
for idx in range(self.NUM_VFS):
pci_info.add_device(
dev_type='VF',
bus=0x81,
slot=0x0,
function=idx + 1,
iommu_group=idx + 43,
numa_node=0,
vf_ratio=self.vf_ratio,
parent=(0x81, 0x0, 0),
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='1021',
prod_name='MT2910 Family [ConnectX-7]',
driver_name='mlx5_core',
vpd_fields={
'name': 'MT2910 Family [ConnectX-7]',
'readonly': {
'serial_number': 'MT0000X00001',
},
}
)
pci_info.add_device(
dev_type='PF',
bus=0x82,
slot=0x0,
function=0,
iommu_group=84,
numa_node=0,
vf_ratio=self.vf_ratio,
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='a2d6',
prod_name='MT42822 BlueField-2 integrated ConnectX-6',
driver_name='mlx5_core',
vpd_fields={
'name': 'MT42822 BlueField-2 integrated ConnectX-6',
'readonly': {
'serial_number': 'MT0000X00002',
},
}
)
for idx in range(self.NUM_VFS):
pci_info.add_device(
dev_type='VF',
bus=0x82,
slot=0x0,
function=idx + 1,
iommu_group=idx + 85,
numa_node=0,
vf_ratio=self.vf_ratio,
parent=(0x82, 0x0, 0),
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='101e',
prod_name='ConnectX Family mlx5Gen Virtual Function',
driver_name='mlx5_core')
return super().start_compute(
hostname=hostname, host_info=host_info, pci_info=pci_info,
mdev_info=mdev_info, vdpa_info=vdpa_info,
libvirt_version=libvirt_version or self.FAKE_LIBVIRT_VERSION,
qemu_version=qemu_version or self.FAKE_QEMU_VERSION)
def create_remote_managed_tunnel_port(self):
dpu_tunnel_port = {
'id': uuids.dpu_tunnel_port,
'network_id': self.neutron.network_3['id'],
'status': 'ACTIVE',
'mac_address': 'fa:16:3e:f0:a4:bb',
'fixed_ips': [
{
'ip_address': '192.168.2.8',
'subnet_id': self.neutron.subnet_3['id']
}
],
'binding:vif_details': {},
'binding:vif_type': 'ovs',
'binding:vnic_type': 'remote-managed',
}
self.neutron.create_port({'port': dpu_tunnel_port})
return dpu_tunnel_port
def create_remote_managed_physnet_port(self):
dpu_physnet_port = {
'id': uuids.dpu_physnet_port,
'network_id': self.neutron.network_4['id'],
'status': 'ACTIVE',
'mac_address': 'd2:0b:fd:99:89:8b',
'fixed_ips': [
{
'ip_address': '192.168.4.10',
'subnet_id': self.neutron.subnet_4['id']
}
],
'binding:vif_details': {},
'binding:vif_type': 'ovs',
'binding:vnic_type': 'remote-managed',
}
self.neutron.create_port({'port': dpu_physnet_port})
return dpu_physnet_port
def test_create_server_physnet(self):
"""Create an instance with a tunnel remote-managed port."""
hostname = self.start_compute()
num_pci = (self.NUM_PFS + self.NUM_VFS) * 2
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
dpu_port = self.create_remote_managed_physnet_port()
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertNotIn('binding:profile', port)
self._create_server(networks=[{'port': dpu_port['id']}])
# Ensure there is one less VF available and that the PF
# is no longer usable.
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
# Ensure the binding:profile details sent to Neutron are correct after
# a port update.
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual({
'card_serial_number': 'MT0000X00001',
'pci_slot': '0000:81:00.4',
'pci_vendor_info': '15b3:1021',
'pf_mac_address': '52:54:00:1e:59:01',
'physical_network': 'physnet4',
'vf_num': 3
}, port['binding:profile'])
def test_create_server_tunnel(self):
"""Create an instance with a tunnel remote-managed port."""
hostname = self.start_compute()
num_pci = (self.NUM_PFS + self.NUM_VFS) * 2
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
dpu_port = self.create_remote_managed_tunnel_port()
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertNotIn('binding:profile', port)
self._create_server(networks=[{'port': dpu_port['id']}])
# Ensure there is one less VF available and that the PF
# is no longer usable.
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
# Ensure the binding:profile details sent to Neutron are correct after
# a port update.
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual({
'card_serial_number': 'MT0000X00002',
'pci_slot': '0000:82:00.4',
'pci_vendor_info': '15b3:101e',
'pf_mac_address': '52:54:00:1e:59:02',
'physical_network': None,
'vf_num': 3
}, port['binding:profile'])
def _test_common(self, op, *args, **kwargs):
self.start_compute()
dpu_port = self.create_remote_managed_tunnel_port()
server = self._create_server(networks=[{'port': dpu_port['id']}])
op(server, *args, **kwargs)
def test_attach_interface(self):
self.start_compute()
dpu_port = self.create_remote_managed_tunnel_port()
server = self._create_server(networks='none')
self._attach_interface(server, dpu_port['id'])
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:82:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:02',
'vf_num': 3,
'card_serial_number': 'MT0000X00002',
},
port['binding:profile'],
)
def test_detach_interface(self):
self._test_common(self._detach_interface, uuids.dpu_tunnel_port)
port = self.neutron.show_port(uuids.dpu_tunnel_port)['port']
self.assertIn('binding:profile', port)
self.assertEqual({}, port['binding:profile'])
def test_shelve(self):
self._test_common(self._shelve_server)
port = self.neutron.show_port(uuids.dpu_tunnel_port)['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:82:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:02',
'vf_num': 3,
'card_serial_number': 'MT0000X00002',
},
port['binding:profile'],
)
def test_suspend(self):
self.start_compute()
dpu_port = self.create_remote_managed_tunnel_port()
server = self._create_server(networks=[{'port': dpu_port['id']}])
self._suspend_server(server)
# TODO(dmitriis): detachDevice does not properly handle hostdevs
# so full suspend/resume testing is problematic.
def _test_move_operation_with_neutron(self, move_operation, dpu_port):
"""Test a move operation with a remote-managed port.
"""
compute1_pci_info = fakelibvirt.HostPCIDevicesInfo(
num_pfs=0, num_vfs=0)
compute1_pci_info.add_device(
dev_type='PF',
bus=0x80,
slot=0x0,
function=0,
iommu_group=84,
numa_node=1,
vf_ratio=self.vf_ratio,
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='a2d6',
prod_name='MT42822 BlueField-2 integrated ConnectX-6',
driver_name='mlx5_core',
vpd_fields={
'name': 'MT42822 BlueField-2 integrated ConnectX-6',
'readonly': {
'serial_number': 'MT0000X00042',
},
}
)
for idx in range(self.NUM_VFS):
compute1_pci_info.add_device(
dev_type='VF',
bus=0x80,
slot=0x0,
function=idx + 1,
iommu_group=idx + 85,
numa_node=1,
vf_ratio=self.vf_ratio,
parent=(0x80, 0x0, 0),
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='101e',
prod_name='ConnectX Family mlx5Gen Virtual Function',
driver_name='mlx5_core',
vpd_fields={
'name': 'MT42822 BlueField-2 integrated ConnectX-6',
'readonly': {
'serial_number': 'MT0000X00042',
},
}
)
self.start_compute(hostname='test_compute0')
self.start_compute(hostname='test_compute1',
pci_info=compute1_pci_info)
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertNotIn('binding:profile', port)
flavor_id = self._create_flavor(vcpu=4)
server = self._create_server(
flavor_id=flavor_id,
networks=[{'port': dpu_port['id']}],
host='test_compute0',
)
self.assertEqual('test_compute0', server['OS-EXT-SRV-ATTR:host'])
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
self.assertPCIDeviceCounts('test_compute1', total=5, free=5)
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:82:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:02',
'vf_num': 3,
'card_serial_number': 'MT0000X00002',
},
port['binding:profile'],
)
move_operation(server)
def test_unshelve_server_with_neutron(self):
def move_operation(source_server):
self._shelve_server(source_server)
# Disable the source compute, to force unshelving on the dest.
self.api.put_service(
self.computes['test_compute0'].service_ref.uuid,
{'status': 'disabled'})
self._unshelve_server(source_server)
dpu_port = self.create_remote_managed_tunnel_port()
self._test_move_operation_with_neutron(move_operation, dpu_port)
self.assertPCIDeviceCounts('test_compute0', total=10, free=10)
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
# Ensure the binding:profile details got updated, including the
# fields relevant to remote-managed ports.
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:80:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:42',
'vf_num': 3,
'card_serial_number': 'MT0000X00042',
},
port['binding:profile'],
)
def test_cold_migrate_server_with_neutron(self):
def move_operation(source_server):
with mock.patch('nova.virt.libvirt.driver.LibvirtDriver'
'.migrate_disk_and_power_off', return_value='{}'):
server = self._migrate_server(source_server)
self._confirm_resize(server)
self.assertPCIDeviceCounts('test_compute0', total=10, free=10)
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
# Ensure the binding:profile details got updated, including the
# fields relevant to remote-managed ports.
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:80:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:42',
'vf_num': 3,
'card_serial_number': 'MT0000X00042',
},
port['binding:profile'],
)
dpu_port = self.create_remote_managed_tunnel_port()
self._test_move_operation_with_neutron(move_operation, dpu_port)
def test_cold_migrate_server_with_neutron_revert(self):
def move_operation(source_server):
with mock.patch('nova.virt.libvirt.driver.LibvirtDriver'
'.migrate_disk_and_power_off', return_value='{}'):
server = self._migrate_server(source_server)
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
self._revert_resize(server)
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
self.assertPCIDeviceCounts('test_compute1', total=5, free=5)
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:82:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:02',
'vf_num': 3,
'card_serial_number': 'MT0000X00002',
},
port['binding:profile'],
)
dpu_port = self.create_remote_managed_tunnel_port()
self._test_move_operation_with_neutron(move_operation, dpu_port)
def test_evacuate_server_with_neutron(self):
def move_operation(source_server):
# Down the source compute to enable the evacuation
self.api.put_service(
self.computes['test_compute0'].service_ref.uuid,
{'forced_down': True})
self.computes['test_compute0'].stop()
self._evacuate_server(source_server)
dpu_port = self.create_remote_managed_tunnel_port()
self._test_move_operation_with_neutron(move_operation, dpu_port)
self.assertPCIDeviceCounts('test_compute0', total=10, free=8)
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
# Ensure the binding:profile details got updated, including the
# fields relevant to remote-managed ports.
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:80:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:42',
'vf_num': 3,
'card_serial_number': 'MT0000X00042',
},
port['binding:profile'],
)
def test_live_migrate_server_with_neutron(self):
"""Live migrate an instance using a remote-managed port.
This should succeed since we support this via detach and attach of the
PCI device similar to how this is done for SR-IOV ports.
"""
def move_operation(source_server):
self._live_migrate(source_server, 'completed')
dpu_port = self.create_remote_managed_tunnel_port()
self._test_move_operation_with_neutron(move_operation, dpu_port)
self.assertPCIDeviceCounts('test_compute0', total=10, free=10)
self.assertPCIDeviceCounts('test_compute1', total=5, free=3)
# Ensure the binding:profile details got updated, including the
# fields relevant to remote-managed ports.
port = self.neutron.show_port(dpu_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:80:00.4',
'physical_network': None,
'pf_mac_address': '52:54:00:1e:59:42',
'vf_num': 3,
'card_serial_number': 'MT0000X00042',
},
port['binding:profile'],
)

9
nova/tests/unit/network/test_neutron.py
View File

@ -7126,7 +7126,7 @@ class TestAPI(TestAPIBase):
self.context, uuids.subnet_id)
@mock.patch.object(neutronapi.LOG, 'debug')
def test_get_port_pci_slot(self, mock_debug):
def test_get_port_pci_dev(self, mock_debug):
fake_port = {'id': uuids.fake_port_id}
request = objects.InstancePCIRequest(requester_id=uuids.fake_port_id,
request_id=uuids.pci_request_id)
@ -7141,13 +7141,14 @@ class TestAPI(TestAPIBase):
pci_devices=objects.PciDeviceList(objects=[device]))
self.assertEqual(
'fake-pci-address',
self.api._get_port_pci_slot(self.context, instance, fake_port))
self.api._get_port_pci_dev(
self.context, instance, fake_port).address)
# Test not finding the request
instance = objects.Instance(
pci_requests=objects.InstancePCIRequests(
requests=[objects.InstancePCIRequest(bad_request)]))
self.assertIsNone(
self.api._get_port_pci_slot(self.context, instance, fake_port))
self.api._get_port_pci_dev(self.context, instance, fake_port))
mock_debug.assert_called_with('No PCI request found for port %s',
uuids.fake_port_id, instance=instance)
mock_debug.reset_mock()
@ -7156,7 +7157,7 @@ class TestAPI(TestAPIBase):
pci_requests=objects.InstancePCIRequests(requests=[request]),
pci_devices=objects.PciDeviceList(objects=[bad_device]))
self.assertIsNone(
self.api._get_port_pci_slot(self.context, instance, fake_port))
self.api._get_port_pci_dev(self.context, instance, fake_port))
mock_debug.assert_called_with('No PCI device found for request %s',
uuids.pci_request_id, instance=instance)