# Copyright (C) 2016 Red Hat, Inc # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import copy from urllib import parse as urlparse import ddt import fixtures from lxml import etree import mock from oslo_config import cfg from oslo_log import log as logging from oslo_serialization import jsonutils from oslo_utils.fixture import uuidsentinel as uuids from oslo_utils import units import nova from nova import context from nova.network import constants from nova import objects from nova.objects import fields from nova.pci.utils import parse_address from nova.tests import fixtures as nova_fixtures from nova.tests.fixtures import libvirt as fakelibvirt from nova.tests.functional.api import client from nova.tests.functional.libvirt import base CONF = cfg.CONF LOG = logging.getLogger(__name__) class _PCIServersTestBase(base.ServersTestBase): ADDITIONAL_FILTERS = ['NUMATopologyFilter', 'PciPassthroughFilter'] def setUp(self): self.ctxt = context.get_admin_context() self.flags(passthrough_whitelist=self.PCI_PASSTHROUGH_WHITELIST, alias=self.PCI_ALIAS, group='pci') super(_PCIServersTestBase, self).setUp() # Mock the 'PciPassthroughFilter' filter, as most tests need to inspect # this host_manager = self.scheduler.manager.host_manager pci_filter_class = host_manager.filter_cls_map['PciPassthroughFilter'] host_pass_mock = mock.Mock(wraps=pci_filter_class().host_passes) self.mock_filter = self.useFixture(fixtures.MockPatch( 'nova.scheduler.filters.pci_passthrough_filter' '.PciPassthroughFilter.host_passes', side_effect=host_pass_mock)).mock def assertPCIDeviceCounts(self, hostname, total, free): """Ensure $hostname has $total devices, $free of which are free.""" devices = objects.PciDeviceList.get_by_compute_node( self.ctxt, objects.ComputeNode.get_by_nodename(self.ctxt, hostname).id, ) self.assertEqual(total, len(devices)) self.assertEqual(free, len([d for d in devices if d.is_available()])) class _PCIServersWithMigrationTestBase(_PCIServersTestBase): def setUp(self): super().setUp() self.useFixture(fixtures.MonkeyPatch( 'nova.tests.fixtures.libvirt.Domain.migrateToURI3', self._migrate_stub)) def _migrate_stub(self, domain, destination, params, flags): """Stub out migrateToURI3.""" src_hostname = domain._connection.hostname dst_hostname = urlparse.urlparse(destination).netloc # In a real live migration, libvirt and QEMU on the source and # destination talk it out, resulting in the instance starting to exist # on the destination. Fakelibvirt cannot do that, so we have to # manually create the "incoming" instance on the destination # fakelibvirt. dst = self.computes[dst_hostname] dst.driver._host.get_connection().createXML( params['destination_xml'], 'fake-createXML-doesnt-care-about-flags') src = self.computes[src_hostname] conn = src.driver._host.get_connection() # because migrateToURI3 is spawned in a background thread, this method # does not block the upper nova layers. Because we don't want nova to # think the live migration has finished until this method is done, the # last thing we do is make fakelibvirt's Domain.jobStats() return # VIR_DOMAIN_JOB_COMPLETED. server = etree.fromstring( params['destination_xml'] ).find('./uuid').text dom = conn.lookupByUUIDString(server) dom.complete_job() class SRIOVServersTest(_PCIServersWithMigrationTestBase): # TODO(stephenfin): We're using this because we want to be able to force # the host during scheduling. We should instead look at overriding policy ADMIN_API = True microversion = 'latest' VFS_ALIAS_NAME = 'vfs' PFS_ALIAS_NAME = 'pfs' PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in ( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PF_PROD_ID, 'physical_network': 'physnet4', }, { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.VF_PROD_ID, 'physical_network': 'physnet4', }, )] # PFs will be removed from pools unless they are specifically # requested, so we explicitly request them with the 'device_type' # attribute PCI_ALIAS = [jsonutils.dumps(x) for x in ( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PF_PROD_ID, 'device_type': fields.PciDeviceType.SRIOV_PF, 'name': PFS_ALIAS_NAME, }, { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.VF_PROD_ID, 'name': VFS_ALIAS_NAME, }, )] def setUp(self): super().setUp() # The ultimate base class _IntegratedTestBase uses NeutronFixture but # we need a bit more intelligent neutron for these tests. Applying the # new fixture here means that we re-stub what the previous neutron # fixture already stubbed. self.neutron = self.useFixture(base.LibvirtNeutronFixture(self)) def _disable_sriov_in_pf(self, pci_info): # Check for PF and change the capability from virt_functions # Delete all the VFs vfs_to_delete = [] for device_name, device in pci_info.devices.items(): if 'virt_functions' in device.pci_device: device.generate_xml(skip_capability=True) elif 'phys_function' in device.pci_device: vfs_to_delete.append(device_name) for device in vfs_to_delete: del pci_info.devices[device] def test_create_server_with_VF(self): """Create a server with an SR-IOV VF-type PCI device.""" pci_info = fakelibvirt.HostPCIDevicesInfo() self.start_compute(pci_info=pci_info) # create a server extra_spec = {"pci_passthrough:alias": "%s:1" % self.VFS_ALIAS_NAME} flavor_id = self._create_flavor(extra_spec=extra_spec) self._create_server(flavor_id=flavor_id, networks='none') # ensure the filter was called self.assertTrue(self.mock_filter.called) def test_create_server_with_PF(self): """Create a server with an SR-IOV PF-type PCI device.""" pci_info = fakelibvirt.HostPCIDevicesInfo() self.start_compute(pci_info=pci_info) # create a server extra_spec = {"pci_passthrough:alias": "%s:1" % self.PFS_ALIAS_NAME} flavor_id = self._create_flavor(extra_spec=extra_spec) self._create_server(flavor_id=flavor_id, networks='none') # ensure the filter was called self.assertTrue(self.mock_filter.called) def test_create_server_with_PF_no_VF(self): """Create a server with a PF and ensure the VFs are then reserved.""" pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=4) self.start_compute(pci_info=pci_info) # create a server using the PF extra_spec_pfs = {"pci_passthrough:alias": f"{self.PFS_ALIAS_NAME}:1"} flavor_id_pfs = self._create_flavor(extra_spec=extra_spec_pfs) self._create_server(flavor_id=flavor_id_pfs, networks='none') # now attempt to build another server, this time using the VF; this # should fail because the VF is used by an instance extra_spec_vfs = {"pci_passthrough:alias": f"{self.VFS_ALIAS_NAME}:1"} flavor_id_vfs = self._create_flavor(extra_spec=extra_spec_vfs) self._create_server( flavor_id=flavor_id_vfs, networks='none', expected_state='ERROR', ) def test_create_server_with_VF_no_PF(self): """Create a server with a VF and ensure the PF is then reserved.""" pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=4) self.start_compute(pci_info=pci_info) # create a server using the VF extra_spec_vfs = {'pci_passthrough:alias': f'{self.VFS_ALIAS_NAME}:1'} flavor_id_vfs = self._create_flavor(extra_spec=extra_spec_vfs) self._create_server(flavor_id=flavor_id_vfs, networks='none') # now attempt to build another server, this time using the PF; this # should fail because the PF is used by an instance extra_spec_pfs = {'pci_passthrough:alias': f'{self.PFS_ALIAS_NAME}:1'} flavor_id_pfs = self._create_flavor(extra_spec=extra_spec_pfs) self._create_server( flavor_id=flavor_id_pfs, networks='none', expected_state='ERROR', ) def test_create_server_with_neutron(self): """Create an instance using a neutron-provisioned SR-IOV VIF.""" pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=2) orig_create = nova.virt.libvirt.guest.Guest.create def fake_create(cls, xml, host): tree = etree.fromstring(xml) elem = tree.find('./devices/interface/source/address') # compare address expected = ('0x81', '0x00', '0x2') actual = ( elem.get('bus'), elem.get('slot'), elem.get('function'), ) self.assertEqual(expected, actual) return orig_create(xml, host) self.stub_out( 'nova.virt.libvirt.guest.Guest.create', fake_create, ) self.start_compute(pci_info=pci_info) # create the port self.neutron.create_port({'port': self.neutron.network_4_port_1}) # ensure the binding details are currently unset port = self.neutron.show_port( base.LibvirtNeutronFixture.network_4_port_1['id'], )['port'] self.assertNotIn('binding:profile', port) # create a server using the VF via neutron self._create_server( networks=[ {'port': base.LibvirtNeutronFixture.network_4_port_1['id']}, ], ) # ensure the binding details sent to "neutron" were correct port = self.neutron.show_port( base.LibvirtNeutronFixture.network_4_port_1['id'], )['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '8086:1515', 'pci_slot': '0000:81:00.2', 'physical_network': 'physnet4', }, port['binding:profile'], ) def test_live_migrate_server_with_PF(self): """Live migrate an instance with a PCI PF. This should fail because it's not possible to live migrate an instance with a PCI passthrough device, even if it's a SR-IOV PF. """ # start two compute services self.start_compute( hostname='test_compute0', pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4)) self.start_compute( hostname='test_compute1', pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4)) # create a server extra_spec = {'pci_passthrough:alias': f'{self.PFS_ALIAS_NAME}:1'} flavor_id = self._create_flavor(extra_spec=extra_spec) server = self._create_server(flavor_id=flavor_id, networks='none') # now live migrate that server ex = self.assertRaises( client.OpenStackApiException, self._live_migrate, server, 'completed') # NOTE(stephenfin): this wouldn't happen in a real deployment since # live migration is a cast, but since we are using CastAsCallFixture # this will bubble to the API self.assertEqual(500, ex.response.status_code) self.assertIn('NoValidHost', str(ex)) def test_live_migrate_server_with_VF(self): """Live migrate an instance with a PCI VF. This should fail because it's not possible to live migrate an instance with a PCI passthrough device, even if it's a SR-IOV VF. """ # start two compute services self.start_compute( hostname='test_compute0', pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4)) self.start_compute( hostname='test_compute1', pci_info=fakelibvirt.HostPCIDevicesInfo(num_pfs=2, num_vfs=4)) # create a server extra_spec = {'pci_passthrough:alias': f'{self.VFS_ALIAS_NAME}:1'} flavor_id = self._create_flavor(extra_spec=extra_spec) server = self._create_server(flavor_id=flavor_id, networks='none') # now live migrate that server ex = self.assertRaises( client.OpenStackApiException, self._live_migrate, server, 'completed') # NOTE(stephenfin): this wouldn't happen in a real deployment since # live migration is a cast, but since we are using CastAsCallFixture # this will bubble to the API self.assertEqual(500, ex.response.status_code) self.assertIn('NoValidHost', str(ex)) def _test_move_operation_with_neutron(self, move_operation, expect_fail=False): # The purpose here is to force an observable PCI slot update when # moving from source to dest. This is accomplished by having a single # PCI VF device on the source, 2 PCI VF devices on the dest, and # relying on the fact that our fake HostPCIDevicesInfo creates # predictable PCI addresses. The PCI VF device on source and the first # PCI VF device on dest will have identical PCI addresses. By sticking # a "placeholder" instance on that first PCI VF device on the dest, the # incoming instance from source will be forced to consume the second # dest PCI VF device, with a different PCI address. # We want to test server operations with SRIOV VFs and SRIOV PFs so # the config of the compute hosts also have one extra PCI PF devices # without any VF children. But the two compute has different PCI PF # addresses and MAC so that the test can observe the slot update as # well as the MAC updated during migration and after revert. source_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1) # add an extra PF without VF to be used by direct-physical ports source_pci_info.add_device( dev_type='PF', bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default slot=0x0, function=0, iommu_group=42, numa_node=0, vf_ratio=0, mac_address='b4:96:91:34:f4:aa', ) self.start_compute( hostname='source', pci_info=source_pci_info) dest_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=2) # add an extra PF without VF to be used by direct-physical ports dest_pci_info.add_device( dev_type='PF', bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default slot=0x6, # make it different from the source host function=0, iommu_group=42, numa_node=0, vf_ratio=0, mac_address='b4:96:91:34:f4:bb', ) self.start_compute( hostname='dest', pci_info=dest_pci_info) source_port = self.neutron.create_port( {'port': self.neutron.network_4_port_1}) source_pf_port = self.neutron.create_port( {'port': self.neutron.network_4_port_pf}) dest_port1 = self.neutron.create_port( {'port': self.neutron.network_4_port_2}) dest_port2 = self.neutron.create_port( {'port': self.neutron.network_4_port_3}) source_server = self._create_server( networks=[ {'port': source_port['port']['id']}, {'port': source_pf_port['port']['id']} ], host='source', ) dest_server1 = self._create_server( networks=[{'port': dest_port1['port']['id']}], host='dest') dest_server2 = self._create_server( networks=[{'port': dest_port2['port']['id']}], host='dest') # Refresh the ports. source_port = self.neutron.show_port(source_port['port']['id']) source_pf_port = self.neutron.show_port(source_pf_port['port']['id']) dest_port1 = self.neutron.show_port(dest_port1['port']['id']) dest_port2 = self.neutron.show_port(dest_port2['port']['id']) # Find the server on the dest compute that's using the same pci_slot as # the server on the source compute, and delete the other one to make # room for the incoming server from the source. source_pci_slot = source_port['port']['binding:profile']['pci_slot'] dest_pci_slot1 = dest_port1['port']['binding:profile']['pci_slot'] if dest_pci_slot1 == source_pci_slot: same_slot_port = dest_port1 self._delete_server(dest_server2) else: same_slot_port = dest_port2 self._delete_server(dest_server1) # Before moving, explicitly assert that the servers on source and dest # have the same pci_slot in their port's binding profile self.assertEqual(source_port['port']['binding:profile']['pci_slot'], same_slot_port['port']['binding:profile']['pci_slot']) # Assert that the direct-physical port got the pci_slot information # according to the source host PF PCI device. self.assertEqual( '0000:82:00.0', # which is in sync with the source host pci_info source_pf_port['port']['binding:profile']['pci_slot'] ) # Assert that the direct-physical port is updated with the MAC address # of the PF device from the source host self.assertEqual( 'b4:96:91:34:f4:aa', source_pf_port['port']['binding:profile']['device_mac_address'] ) # Before moving, assert that the servers on source and dest have the # same PCI source address in their XML for their SRIOV nic. source_conn = self.computes['source'].driver._host.get_connection() dest_conn = self.computes['source'].driver._host.get_connection() source_vms = [vm._def for vm in source_conn._vms.values()] dest_vms = [vm._def for vm in dest_conn._vms.values()] self.assertEqual(1, len(source_vms)) self.assertEqual(1, len(dest_vms)) self.assertEqual(1, len(source_vms[0]['devices']['nics'])) self.assertEqual(1, len(dest_vms[0]['devices']['nics'])) self.assertEqual(source_vms[0]['devices']['nics'][0]['source'], dest_vms[0]['devices']['nics'][0]['source']) move_operation(source_server) # Refresh the ports again, keeping in mind the source_port is now bound # on the dest after the move. source_port = self.neutron.show_port(source_port['port']['id']) same_slot_port = self.neutron.show_port(same_slot_port['port']['id']) source_pf_port = self.neutron.show_port(source_pf_port['port']['id']) self.assertNotEqual( source_port['port']['binding:profile']['pci_slot'], same_slot_port['port']['binding:profile']['pci_slot']) # Assert that the direct-physical port got the pci_slot information # according to the dest host PF PCI device. self.assertEqual( '0000:82:06.0', # which is in sync with the dest host pci_info source_pf_port['port']['binding:profile']['pci_slot'] ) # Assert that the direct-physical port is updated with the MAC address # of the PF device from the dest host self.assertEqual( 'b4:96:91:34:f4:bb', source_pf_port['port']['binding:profile']['device_mac_address'] ) conn = self.computes['dest'].driver._host.get_connection() vms = [vm._def for vm in conn._vms.values()] self.assertEqual(2, len(vms)) for vm in vms: self.assertEqual(1, len(vm['devices']['nics'])) self.assertNotEqual(vms[0]['devices']['nics'][0]['source'], vms[1]['devices']['nics'][0]['source']) def test_unshelve_server_with_neutron(self): def move_operation(source_server): self._shelve_server(source_server) # Disable the source compute, to force unshelving on the dest. self.api.put_service(self.computes['source'].service_ref.uuid, {'status': 'disabled'}) self._unshelve_server(source_server) self._test_move_operation_with_neutron(move_operation) def test_cold_migrate_server_with_neutron(self): def move_operation(source_server): # TODO(stephenfin): The mock of 'migrate_disk_and_power_off' should # probably be less...dumb with mock.patch('nova.virt.libvirt.driver.LibvirtDriver' '.migrate_disk_and_power_off', return_value='{}'): self._migrate_server(source_server) self._confirm_resize(source_server) self._test_move_operation_with_neutron(move_operation) def test_cold_migrate_and_rever_server_with_neutron(self): # The purpose here is to force an observable PCI slot update when # moving from source to dest and the from dest to source after the # revert. This is accomplished by having a single # PCI VF device on the source, 2 PCI VF devices on the dest, and # relying on the fact that our fake HostPCIDevicesInfo creates # predictable PCI addresses. The PCI VF device on source and the first # PCI VF device on dest will have identical PCI addresses. By sticking # a "placeholder" instance on that first PCI VF device on the dest, the # incoming instance from source will be forced to consume the second # dest PCI VF device, with a different PCI address. # We want to test server operations with SRIOV VFs and SRIOV PFs so # the config of the compute hosts also have one extra PCI PF devices # without any VF children. But the two compute has different PCI PF # addresses and MAC so that the test can observe the slot update as # well as the MAC updated during migration and after revert. source_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1) # add an extra PF without VF to be used by direct-physical ports source_pci_info.add_device( dev_type='PF', bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default slot=0x0, function=0, iommu_group=42, numa_node=0, vf_ratio=0, mac_address='b4:96:91:34:f4:aa', ) self.start_compute( hostname='source', pci_info=source_pci_info) dest_pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=2) # add an extra PF without VF to be used by direct-physical ports dest_pci_info.add_device( dev_type='PF', bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default slot=0x6, # make it different from the source host function=0, iommu_group=42, numa_node=0, vf_ratio=0, mac_address='b4:96:91:34:f4:bb', ) self.start_compute( hostname='dest', pci_info=dest_pci_info) source_port = self.neutron.create_port( {'port': self.neutron.network_4_port_1}) source_pf_port = self.neutron.create_port( {'port': self.neutron.network_4_port_pf}) dest_port1 = self.neutron.create_port( {'port': self.neutron.network_4_port_2}) dest_port2 = self.neutron.create_port( {'port': self.neutron.network_4_port_3}) source_server = self._create_server( networks=[ {'port': source_port['port']['id']}, {'port': source_pf_port['port']['id']} ], host='source', ) dest_server1 = self._create_server( networks=[{'port': dest_port1['port']['id']}], host='dest') dest_server2 = self._create_server( networks=[{'port': dest_port2['port']['id']}], host='dest') # Refresh the ports. source_port = self.neutron.show_port(source_port['port']['id']) source_pf_port = self.neutron.show_port(source_pf_port['port']['id']) dest_port1 = self.neutron.show_port(dest_port1['port']['id']) dest_port2 = self.neutron.show_port(dest_port2['port']['id']) # Find the server on the dest compute that's using the same pci_slot as # the server on the source compute, and delete the other one to make # room for the incoming server from the source. source_pci_slot = source_port['port']['binding:profile']['pci_slot'] dest_pci_slot1 = dest_port1['port']['binding:profile']['pci_slot'] if dest_pci_slot1 == source_pci_slot: same_slot_port = dest_port1 self._delete_server(dest_server2) else: same_slot_port = dest_port2 self._delete_server(dest_server1) # Before moving, explicitly assert that the servers on source and dest # have the same pci_slot in their port's binding profile self.assertEqual(source_port['port']['binding:profile']['pci_slot'], same_slot_port['port']['binding:profile']['pci_slot']) # Assert that the direct-physical port got the pci_slot information # according to the source host PF PCI device. self.assertEqual( '0000:82:00.0', # which is in sync with the source host pci_info source_pf_port['port']['binding:profile']['pci_slot'] ) # Assert that the direct-physical port is updated with the MAC address # of the PF device from the source host self.assertEqual( 'b4:96:91:34:f4:aa', source_pf_port['port']['binding:profile']['device_mac_address'] ) # Before moving, assert that the servers on source and dest have the # same PCI source address in their XML for their SRIOV nic. source_conn = self.computes['source'].driver._host.get_connection() dest_conn = self.computes['source'].driver._host.get_connection() source_vms = [vm._def for vm in source_conn._vms.values()] dest_vms = [vm._def for vm in dest_conn._vms.values()] self.assertEqual(1, len(source_vms)) self.assertEqual(1, len(dest_vms)) self.assertEqual(1, len(source_vms[0]['devices']['nics'])) self.assertEqual(1, len(dest_vms[0]['devices']['nics'])) self.assertEqual(source_vms[0]['devices']['nics'][0]['source'], dest_vms[0]['devices']['nics'][0]['source']) # TODO(stephenfin): The mock of 'migrate_disk_and_power_off' should # probably be less...dumb with mock.patch('nova.virt.libvirt.driver.LibvirtDriver' '.migrate_disk_and_power_off', return_value='{}'): self._migrate_server(source_server) # Refresh the ports again, keeping in mind the ports are now bound # on the dest after migrating. source_port = self.neutron.show_port(source_port['port']['id']) same_slot_port = self.neutron.show_port(same_slot_port['port']['id']) source_pf_port = self.neutron.show_port(source_pf_port['port']['id']) self.assertNotEqual( source_port['port']['binding:profile']['pci_slot'], same_slot_port['port']['binding:profile']['pci_slot']) # Assert that the direct-physical port got the pci_slot information # according to the dest host PF PCI device. self.assertEqual( '0000:82:06.0', # which is in sync with the dest host pci_info source_pf_port['port']['binding:profile']['pci_slot'] ) # Assert that the direct-physical port is updated with the MAC address # of the PF device from the dest host self.assertEqual( 'b4:96:91:34:f4:bb', source_pf_port['port']['binding:profile']['device_mac_address'] ) conn = self.computes['dest'].driver._host.get_connection() vms = [vm._def for vm in conn._vms.values()] self.assertEqual(2, len(vms)) for vm in vms: self.assertEqual(1, len(vm['devices']['nics'])) self.assertNotEqual(vms[0]['devices']['nics'][0]['source'], vms[1]['devices']['nics'][0]['source']) self._revert_resize(source_server) # Refresh the ports again, keeping in mind the ports are now bound # on the source as the migration is reverted source_pf_port = self.neutron.show_port(source_pf_port['port']['id']) # Assert that the direct-physical port got the pci_slot information # according to the source host PF PCI device. self.assertEqual( '0000:82:00.0', # which is in sync with the source host pci_info source_pf_port['port']['binding:profile']['pci_slot'] ) # Assert that the direct-physical port is updated with the MAC address # of the PF device from the source host self.assertEqual( 'b4:96:91:34:f4:aa', source_pf_port['port']['binding:profile']['device_mac_address'] ) def test_evacuate_server_with_neutron(self): def move_operation(source_server): # Down the source compute to enable the evacuation self.api.put_service(self.computes['source'].service_ref.uuid, {'forced_down': True}) self.computes['source'].stop() self._evacuate_server(source_server) self._test_move_operation_with_neutron(move_operation) def test_live_migrate_server_with_neutron(self): """Live migrate an instance using a neutron-provisioned SR-IOV VIF. This should succeed since we support this, via detach and attach of the PCI device. """ # start two compute services with differing PCI device inventory source_pci_info = fakelibvirt.HostPCIDevicesInfo( num_pfs=2, num_vfs=8, numa_node=0) # add an extra PF without VF to be used by direct-physical ports source_pci_info.add_device( dev_type='PF', bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default slot=0x0, function=0, iommu_group=42, numa_node=0, vf_ratio=0, mac_address='b4:96:91:34:f4:aa', ) self.start_compute(hostname='test_compute0', pci_info=source_pci_info) dest_pci_info = fakelibvirt.HostPCIDevicesInfo( num_pfs=1, num_vfs=2, numa_node=1) # add an extra PF without VF to be used by direct-physical ports dest_pci_info.add_device( dev_type='PF', bus=0x82, # the HostPCIDevicesInfo use the 0x81 by default slot=0x6, # make it different from the source host function=0, iommu_group=42, # numa node needs to be aligned with the other pci devices in this # host as the instance needs to fit into a single host numa node numa_node=1, vf_ratio=0, mac_address='b4:96:91:34:f4:bb', ) self.start_compute(hostname='test_compute1', pci_info=dest_pci_info) # create the ports port = self.neutron.create_port( {'port': self.neutron.network_4_port_1})['port'] pf_port = self.neutron.create_port( {'port': self.neutron.network_4_port_pf})['port'] # create a server using the VF via neutron extra_spec = {'hw:cpu_policy': 'dedicated'} flavor_id = self._create_flavor(vcpu=4, extra_spec=extra_spec) server = self._create_server( flavor_id=flavor_id, networks=[ {'port': port['id']}, {'port': pf_port['id']}, ], host='test_compute0', ) # our source host should have marked two PCI devices as used, the VF # and the parent PF, while the future destination is currently unused self.assertEqual('test_compute0', server['OS-EXT-SRV-ATTR:host']) self.assertPCIDeviceCounts('test_compute0', total=11, free=8) self.assertPCIDeviceCounts('test_compute1', total=4, free=4) # the instance should be on host NUMA node 0, since that's where our # PCI devices are host_numa = objects.NUMATopology.obj_from_db_obj( objects.ComputeNode.get_by_nodename( self.ctxt, 'test_compute0', ).numa_topology ) self.assertEqual({0, 1, 2, 3}, host_numa.cells[0].pinned_cpus) self.assertEqual(set(), host_numa.cells[1].pinned_cpus) # ensure the binding details sent to "neutron" are correct port = self.neutron.show_port( base.LibvirtNeutronFixture.network_4_port_1['id'], )['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '8086:1515', # TODO(stephenfin): Stop relying on a side-effect of how nova # chooses from multiple PCI devices (apparently the last # matching one) 'pci_slot': '0000:81:01.4', 'physical_network': 'physnet4', }, port['binding:profile'], ) # ensure the binding details sent to "neutron" are correct pf_port = self.neutron.show_port(pf_port['id'],)['port'] self.assertIn('binding:profile', pf_port) self.assertEqual( { 'pci_vendor_info': '8086:1528', 'pci_slot': '0000:82:00.0', 'physical_network': 'physnet4', 'device_mac_address': 'b4:96:91:34:f4:aa', }, pf_port['binding:profile'], ) # now live migrate that server self._live_migrate(server, 'completed') # we should now have transitioned our usage to the destination, freeing # up the source in the process self.assertPCIDeviceCounts('test_compute0', total=11, free=11) self.assertPCIDeviceCounts('test_compute1', total=4, free=1) # the instance should now be on host NUMA node 1, since that's where # our PCI devices are for this second host host_numa = objects.NUMATopology.obj_from_db_obj( objects.ComputeNode.get_by_nodename( self.ctxt, 'test_compute1', ).numa_topology ) self.assertEqual(set(), host_numa.cells[0].pinned_cpus) self.assertEqual({4, 5, 6, 7}, host_numa.cells[1].pinned_cpus) # ensure the binding details sent to "neutron" have been updated port = self.neutron.show_port( base.LibvirtNeutronFixture.network_4_port_1['id'], )['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '8086:1515', 'pci_slot': '0000:81:00.2', 'physical_network': 'physnet4', }, port['binding:profile'], ) # ensure the binding details sent to "neutron" are correct pf_port = self.neutron.show_port(pf_port['id'],)['port'] self.assertIn('binding:profile', pf_port) self.assertEqual( { 'pci_vendor_info': '8086:1528', 'pci_slot': '0000:82:06.0', 'physical_network': 'physnet4', 'device_mac_address': 'b4:96:91:34:f4:bb', }, pf_port['binding:profile'], ) def test_get_server_diagnostics_server_with_VF(self): """Ensure server disagnostics include info on VF-type PCI devices.""" pci_info = fakelibvirt.HostPCIDevicesInfo() self.start_compute(pci_info=pci_info) # create the SR-IOV port port = self.neutron.create_port( {'port': self.neutron.network_4_port_1}) flavor_id = self._create_flavor() server = self._create_server( flavor_id=flavor_id, networks=[ {'uuid': base.LibvirtNeutronFixture.network_1['id']}, {'port': port['port']['id']}, ], ) # now check the server diagnostics to ensure the VF-type PCI device is # attached diagnostics = self.api.get_server_diagnostics( server['id'] ) self.assertEqual( base.LibvirtNeutronFixture.network_1_port_2['mac_address'], diagnostics['nic_details'][0]['mac_address'], ) for key in ('rx_packets', 'tx_packets'): self.assertIn(key, diagnostics['nic_details'][0]) self.assertEqual( base.LibvirtNeutronFixture.network_4_port_1['mac_address'], diagnostics['nic_details'][1]['mac_address'], ) for key in ('rx_packets', 'tx_packets'): self.assertIn(key, diagnostics['nic_details'][1]) def test_create_server_after_change_in_nonsriov_pf_to_sriov_pf(self): # Starts a compute with PF not configured with SRIOV capabilities # Updates the PF with SRIOV capability and restart the compute service # Then starts a VM with the sriov port. The VM should be in active # state with sriov port attached. # To emulate the device type changing, we first create a # HostPCIDevicesInfo object with PFs and VFs. Then we make a copy # and remove the VFs and the virt_function capability. This is # done to ensure the physical function product id is same in both # the versions. pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1) pci_info_no_sriov = copy.deepcopy(pci_info) # Disable SRIOV capabilties in PF and delete the VFs self._disable_sriov_in_pf(pci_info_no_sriov) fake_connection = self._get_connection(pci_info=pci_info_no_sriov, hostname='test_compute0') self.mock_conn.return_value = fake_connection self.compute = self.start_service('compute', host='test_compute0') ctxt = context.get_admin_context() pci_devices = objects.PciDeviceList.get_by_compute_node( ctxt, objects.ComputeNode.get_by_nodename( ctxt, 'test_compute0', ).id, ) self.assertEqual(1, len(pci_devices)) self.assertEqual('type-PCI', pci_devices[0].dev_type) # Update connection with original pci info with sriov PFs fake_connection = self._get_connection(pci_info=pci_info, hostname='test_compute0') self.mock_conn.return_value = fake_connection # Restart the compute service self.restart_compute_service(self.compute) # Verify if PCI devices are of type type-PF or type-VF pci_devices = objects.PciDeviceList.get_by_compute_node( ctxt, objects.ComputeNode.get_by_nodename( ctxt, 'test_compute0', ).id, ) for pci_device in pci_devices: self.assertIn(pci_device.dev_type, ['type-PF', 'type-VF']) # create the port self.neutron.create_port({'port': self.neutron.network_4_port_1}) # create a server using the VF via neutron self._create_server( networks=[ {'port': base.LibvirtNeutronFixture.network_4_port_1['id']}, ], ) class SRIOVAttachDetachTest(_PCIServersTestBase): # no need for aliases as these test will request SRIOV via neutron PCI_ALIAS = [] PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in ( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PF_PROD_ID, "physical_network": "physnet2", }, { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.VF_PROD_ID, "physical_network": "physnet2", }, )] def setUp(self): super().setUp() self.neutron = self.useFixture(nova_fixtures.NeutronFixture(self)) # add extra ports and the related network to the neutron fixture # specifically for these tests. It cannot be added globally in the # fixture init as it adds a second network that makes auto allocation # based test to fail due to ambiguous networks. self.neutron._networks[ self.neutron.network_2['id']] = self.neutron.network_2 self.neutron._subnets[ self.neutron.subnet_2['id']] = self.neutron.subnet_2 for port in [self.neutron.sriov_port, self.neutron.sriov_port2, self.neutron.sriov_pf_port, self.neutron.sriov_pf_port2, self.neutron.macvtap_port, self.neutron.macvtap_port2]: self.neutron._ports[port['id']] = copy.deepcopy(port) def _get_attached_port_ids(self, instance_uuid): return [ attachment['port_id'] for attachment in self.api.get_port_interfaces(instance_uuid)] def _detach_port(self, instance_uuid, port_id): self.api.detach_interface(instance_uuid, port_id) self.notifier.wait_for_versioned_notifications( 'instance.interface_detach.end') def _attach_port(self, instance_uuid, port_id): self.api.attach_interface( instance_uuid, {'interfaceAttachment': {'port_id': port_id}}) self.notifier.wait_for_versioned_notifications( 'instance.interface_attach.end') def _test_detach_attach(self, first_port_id, second_port_id): # This test takes two ports that requires PCI claim. # Starts a compute with one PF and one connected VF. # Then starts a VM with the first port. Then detach it, then # re-attach it. These expected to be successful. Then try to attach the # second port and asserts that it fails as no free PCI device left on # the host. host_info = fakelibvirt.HostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2) pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1) fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection self.compute = self.start_service('compute', host='test_compute0') # Create server with a port server = self._create_server(networks=[{'port': first_port_id}]) updated_port = self.neutron.show_port(first_port_id)['port'] self.assertEqual('test_compute0', updated_port['binding:host_id']) self.assertIn(first_port_id, self._get_attached_port_ids(server['id'])) self._detach_port(server['id'], first_port_id) updated_port = self.neutron.show_port(first_port_id)['port'] self.assertIsNone(updated_port['binding:host_id']) self.assertNotIn( first_port_id, self._get_attached_port_ids(server['id'])) # Attach back the port self._attach_port(server['id'], first_port_id) updated_port = self.neutron.show_port(first_port_id)['port'] self.assertEqual('test_compute0', updated_port['binding:host_id']) self.assertIn(first_port_id, self._get_attached_port_ids(server['id'])) # Try to attach the second port but no free PCI device left ex = self.assertRaises( client.OpenStackApiException, self._attach_port, server['id'], second_port_id) self.assertEqual(400, ex.response.status_code) self.assertIn('Failed to claim PCI device', str(ex)) attached_ports = self._get_attached_port_ids(server['id']) self.assertIn(first_port_id, attached_ports) self.assertNotIn(second_port_id, attached_ports) def test_detach_attach_direct(self): self._test_detach_attach( self.neutron.sriov_port['id'], self.neutron.sriov_port2['id']) def test_detach_macvtap(self): self._test_detach_attach( self.neutron.macvtap_port['id'], self.neutron.macvtap_port2['id']) def test_detach_direct_physical(self): self._test_detach_attach( self.neutron.sriov_pf_port['id'], self.neutron.sriov_pf_port2['id']) class VDPAServersTest(_PCIServersTestBase): # this is needed for os_compute_api:os-migrate-server:migrate policy ADMIN_API = True microversion = 'latest' # Whitelist both the PF and VF; in reality, you probably wouldn't do this # but we want to make sure that the PF is correctly taken off the table # once any VF is used PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in ( { 'vendor_id': '15b3', 'product_id': '101d', 'physical_network': 'physnet4', }, { 'vendor_id': '15b3', 'product_id': '101e', 'physical_network': 'physnet4', }, )] # No need for aliases as these test will request SRIOV via neutron PCI_ALIAS = [] NUM_PFS = 1 NUM_VFS = 4 FAKE_LIBVIRT_VERSION = 6_009_000 # 6.9.0 FAKE_QEMU_VERSION = 5_001_000 # 5.1.0 def setUp(self): super().setUp() # The ultimate base class _IntegratedTestBase uses NeutronFixture but # we need a bit more intelligent neutron for these tests. Applying the # new fixture here means that we re-stub what the previous neutron # fixture already stubbed. self.neutron = self.useFixture(base.LibvirtNeutronFixture(self)) def start_compute(self): vf_ratio = self.NUM_VFS // self.NUM_PFS pci_info = fakelibvirt.HostPCIDevicesInfo( num_pci=0, num_pfs=0, num_vfs=0) vdpa_info = fakelibvirt.HostVDPADevicesInfo() pci_info.add_device( dev_type='PF', bus=0x6, slot=0x0, function=0, iommu_group=40, # totally arbitrary number numa_node=0, vf_ratio=vf_ratio, vend_id='15b3', vend_name='Mellanox Technologies', prod_id='101d', prod_name='MT2892 Family [ConnectX-6 Dx]', driver_name='mlx5_core') for idx in range(self.NUM_VFS): vf = pci_info.add_device( dev_type='VF', bus=0x6, slot=0x0, function=idx + 1, iommu_group=idx + 41, # totally arbitrary number + offset numa_node=0, vf_ratio=vf_ratio, parent=(0x6, 0x0, 0), vend_id='15b3', vend_name='Mellanox Technologies', prod_id='101e', prod_name='ConnectX Family mlx5Gen Virtual Function', driver_name='mlx5_core') vdpa_info.add_device(f'vdpa_vdpa{idx}', idx, vf) return super().start_compute( pci_info=pci_info, vdpa_info=vdpa_info, libvirt_version=self.FAKE_LIBVIRT_VERSION, qemu_version=self.FAKE_QEMU_VERSION) def create_vdpa_port(self): vdpa_port = { 'id': uuids.vdpa_port, 'network_id': self.neutron.network_4['id'], 'status': 'ACTIVE', 'mac_address': 'b5:bc:2e:e7:51:ee', 'fixed_ips': [ { 'ip_address': '192.168.4.6', 'subnet_id': self.neutron.subnet_4['id'] } ], 'binding:vif_details': {}, 'binding:vif_type': 'ovs', 'binding:vnic_type': 'vdpa', } # create the port self.neutron.create_port({'port': vdpa_port}) return vdpa_port def test_create_server(self): """Create an instance using a neutron-provisioned vDPA VIF.""" orig_create = nova.virt.libvirt.guest.Guest.create def fake_create(cls, xml, host): tree = etree.fromstring(xml) elem = tree.find('./devices/interface/[@type="vdpa"]') # compare source device # the MAC address is derived from the neutron port, while the # source dev path assumes we attach vDPA devs in order expected = """ """ actual = etree.tostring(elem, encoding='unicode') self.assertXmlEqual(expected, actual) return orig_create(xml, host) self.stub_out( 'nova.virt.libvirt.guest.Guest.create', fake_create, ) hostname = self.start_compute() num_pci = self.NUM_PFS + self.NUM_VFS # both the PF and VF with vDPA capabilities (dev_type=vdpa) should have # been counted self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci) # create the port vdpa_port = self.create_vdpa_port() # ensure the binding details are currently unset port = self.neutron.show_port(vdpa_port['id'])['port'] self.assertNotIn('binding:profile', port) # create a server using the vDPA device via neutron self._create_server(networks=[{'port': vdpa_port['id']}]) # ensure there is one less VF available and that the PF is no longer # usable self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2) # ensure the binding details sent to "neutron" were correct port = self.neutron.show_port(vdpa_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:06:00.4', 'physical_network': 'physnet4', }, port['binding:profile'], ) def _test_common(self, op, *args, **kwargs): self.start_compute() # create the port and a server, with the port attached to the server vdpa_port = self.create_vdpa_port() server = self._create_server(networks=[{'port': vdpa_port['id']}]) # attempt the unsupported action and ensure it fails ex = self.assertRaises( client.OpenStackApiException, op, server, *args, **kwargs) self.assertIn( 'not supported for instance with vDPA ports', ex.response.text) def test_attach_interface(self): self.start_compute() # create the port and a server, but don't attach the port to the server # yet vdpa_port = self.create_vdpa_port() server = self._create_server(networks='none') # attempt to attach the port to the server ex = self.assertRaises( client.OpenStackApiException, self._attach_interface, server, vdpa_port['id']) self.assertIn( 'not supported for instance with vDPA ports', ex.response.text) def test_detach_interface(self): self._test_common(self._detach_interface, uuids.vdpa_port) def test_shelve(self): self._test_common(self._shelve_server) def test_suspend(self): self._test_common(self._suspend_server) def test_evacute(self): self._test_common(self._evacuate_server) def test_resize(self): flavor_id = self._create_flavor() self._test_common(self._resize_server, flavor_id) def test_cold_migrate(self): self._test_common(self._migrate_server) class PCIServersTest(_PCIServersTestBase): ADMIN_API = True microversion = 'latest' ALIAS_NAME = 'a1' PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, } )] PCI_ALIAS = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, 'name': ALIAS_NAME, } )] def test_create_server_with_pci_dev_and_numa(self): """Verifies that an instance can be booted with cpu pinning and with an assigned pci device. """ self.flags(cpu_dedicated_set='0-7', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=1) self.start_compute(pci_info=pci_info) # create a flavor extra_spec = { 'hw:cpu_policy': 'dedicated', 'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME, } flavor_id = self._create_flavor(extra_spec=extra_spec) self._create_server(flavor_id=flavor_id, networks='none') def test_create_server_with_pci_dev_and_numa_fails(self): """This test ensures that it is not possible to allocated CPU and memory resources from one NUMA node and a PCI device from another. """ self.flags(cpu_dedicated_set='0-7', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=0) self.start_compute(pci_info=pci_info) # boot one instance with no PCI device to "fill up" NUMA node 0 extra_spec = {'hw:cpu_policy': 'dedicated'} flavor_id = self._create_flavor(vcpu=4, extra_spec=extra_spec) self._create_server(flavor_id=flavor_id, networks='none') # now boot one with a PCI device, which should fail to boot extra_spec['pci_passthrough:alias'] = '%s:1' % self.ALIAS_NAME flavor_id = self._create_flavor(extra_spec=extra_spec) self._create_server( flavor_id=flavor_id, networks='none', expected_state='ERROR') def test_live_migrate_server_with_pci(self): """Live migrate an instance with a PCI passthrough device. This should fail because it's not possible to live migrate an instance with a PCI passthrough device, even if it's a SR-IOV VF. """ # start two compute services self.start_compute( hostname='test_compute0', pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1)) self.start_compute( hostname='test_compute1', pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1)) # create a server extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'} flavor_id = self._create_flavor(extra_spec=extra_spec) server = self._create_server(flavor_id=flavor_id, networks='none') # now live migrate that server ex = self.assertRaises( client.OpenStackApiException, self._live_migrate, server, 'completed') # NOTE(stephenfin): this wouldn't happen in a real deployment since # live migration is a cast, but since we are using CastAsCallFixture # this will bubble to the API self.assertEqual(500, ex.response.status_code) self.assertIn('NoValidHost', str(ex)) def test_resize_pci_to_vanilla(self): # Start two computes, one with PCI and one without. self.start_compute( hostname='test_compute0', pci_info=fakelibvirt.HostPCIDevicesInfo(num_pci=1)) self.start_compute(hostname='test_compute1') # Boot a server with a single PCI device. extra_spec = {'pci_passthrough:alias': f'{self.ALIAS_NAME}:1'} pci_flavor_id = self._create_flavor(extra_spec=extra_spec) server = self._create_server(flavor_id=pci_flavor_id, networks='none') # Resize it to a flavor without PCI devices. We expect this to work, as # test_compute1 is available. # FIXME(artom) This is bug 1941005. flavor_id = self._create_flavor() ex = self.assertRaises(client.OpenStackApiException, self._resize_server, server, flavor_id) self.assertEqual(500, ex.response.status_code) self.assertIn('NoValidHost', str(ex)) # self._confirm_resize(server) def _confirm_resize(self, server, host='host1'): # NOTE(sbauza): Unfortunately, _cleanup_resize() in libvirt checks the # host option to know the source hostname but given we have a global # CONF, the value will be the hostname of the last compute service that # was created, so we need to change it here. # TODO(sbauza): Remove the below once we stop using CONF.host in # libvirt and rather looking at the compute host value. orig_host = CONF.host self.flags(host=host) super()._confirm_resize(server) self.flags(host=orig_host) def test_cold_migrate_server_with_pci(self): host_devices = {} orig_create = nova.virt.libvirt.guest.Guest.create def fake_create(cls, xml, host): tree = etree.fromstring(xml) elem = tree.find('./devices/hostdev/source/address') hostname = host.get_hostname() address = ( elem.get('bus'), elem.get('slot'), elem.get('function'), ) if hostname in host_devices: self.assertNotIn(address, host_devices[hostname]) else: host_devices[hostname] = [] host_devices[host.get_hostname()].append(address) return orig_create(xml, host) self.stub_out( 'nova.virt.libvirt.guest.Guest.create', fake_create, ) # start two compute services for hostname in ('test_compute0', 'test_compute1'): pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=2) self.start_compute(hostname=hostname, pci_info=pci_info) # boot an instance with a PCI device on each host extra_spec = { 'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME, } flavor_id = self._create_flavor(extra_spec=extra_spec) server_a = self._create_server( flavor_id=flavor_id, networks='none', host='test_compute0') server_b = self._create_server( flavor_id=flavor_id, networks='none', host='test_compute1') # the instances should have landed on separate hosts; ensure both hosts # have one used PCI device and one free PCI device self.assertNotEqual( server_a['OS-EXT-SRV-ATTR:host'], server_b['OS-EXT-SRV-ATTR:host'], ) for hostname in ('test_compute0', 'test_compute1'): self.assertPCIDeviceCounts(hostname, total=2, free=1) # TODO(stephenfin): The mock of 'migrate_disk_and_power_off' should # probably be less...dumb with mock.patch( 'nova.virt.libvirt.driver.LibvirtDriver' '.migrate_disk_and_power_off', return_value='{}', ): # TODO(stephenfin): Use a helper self.api.post_server_action(server_a['id'], {'migrate': None}) server_a = self._wait_for_state_change(server_a, 'VERIFY_RESIZE') # the instances should now be on the same host; ensure the source host # still has one used PCI device while the destination now has two used # test_compute0 initially self.assertEqual( server_a['OS-EXT-SRV-ATTR:host'], server_b['OS-EXT-SRV-ATTR:host'], ) self.assertPCIDeviceCounts('test_compute0', total=2, free=1) self.assertPCIDeviceCounts('test_compute1', total=2, free=0) # now, confirm the migration and check our counts once again self._confirm_resize(server_a) self.assertPCIDeviceCounts('test_compute0', total=2, free=2) self.assertPCIDeviceCounts('test_compute1', total=2, free=0) class PCIServersWithPreferredNUMATest(_PCIServersTestBase): ALIAS_NAME = 'a1' PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, } )] PCI_ALIAS = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, 'name': ALIAS_NAME, 'device_type': fields.PciDeviceType.STANDARD, 'numa_policy': fields.PCINUMAAffinityPolicy.PREFERRED, } )] expected_state = 'ACTIVE' def test_create_server_with_pci_dev_and_numa(self): """Validate behavior of 'preferred' PCI NUMA policy. This test ensures that it *is* possible to allocate CPU and memory resources from one NUMA node and a PCI device from another *if* PCI NUMA policies are in use. """ self.flags(cpu_dedicated_set='0-7', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=0) self.start_compute(pci_info=pci_info) # boot one instance with no PCI device to "fill up" NUMA node 0 extra_spec = { 'hw:cpu_policy': 'dedicated', } flavor_id = self._create_flavor(vcpu=4, extra_spec=extra_spec) self._create_server(flavor_id=flavor_id) # now boot one with a PCI device, which should succeed thanks to the # use of the PCI policy extra_spec['pci_passthrough:alias'] = '%s:1' % self.ALIAS_NAME flavor_id = self._create_flavor(extra_spec=extra_spec) self._create_server( flavor_id=flavor_id, expected_state=self.expected_state) class PCIServersWithRequiredNUMATest(PCIServersWithPreferredNUMATest): ALIAS_NAME = 'a1' PCI_ALIAS = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, 'name': ALIAS_NAME, 'device_type': fields.PciDeviceType.STANDARD, 'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED, } )] expected_state = 'ERROR' @ddt.ddt class PCIServersWithSRIOVAffinityPoliciesTest(_PCIServersTestBase): ALIAS_NAME = 'a1' PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, } )] # we set the numa_affinity policy to required to ensure strict affinity # between pci devices and the guest cpu and memory will be enforced. PCI_ALIAS = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, 'name': ALIAS_NAME, 'device_type': fields.PciDeviceType.STANDARD, 'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED, } )] # NOTE(sean-k-mooney): i could just apply the ddt decorators # to this function for the most part but i have chosen to # keep one top level function per policy to make documenting # the test cases simpler. def _test_policy(self, pci_numa_node, status, policy): # only allow cpus on numa node 1 to be used for pinning self.flags(cpu_dedicated_set='4-7', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo( num_pci=1, numa_node=pci_numa_node) self.start_compute(pci_info=pci_info) # request cpu pinning to create a numa toplogy and allow the test to # force which numa node the vm would have to be pinned too. extra_spec = { 'hw:cpu_policy': 'dedicated', 'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME, 'hw:pci_numa_affinity_policy': policy } flavor_id = self._create_flavor(extra_spec=extra_spec) self._create_server(flavor_id=flavor_id, expected_state=status) if status == 'ACTIVE': self.assertTrue(self.mock_filter.called) else: # the PciPassthroughFilter should not have been called, since the # NUMATopologyFilter should have eliminated the filter first self.assertFalse(self.mock_filter.called) @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) # the preferred policy should always pass regardless of numa affinity @ddt.data((-1, 'ACTIVE'), (0, 'ACTIVE'), (1, 'ACTIVE')) def test_create_server_with_sriov_numa_affinity_policy_preferred( self, pci_numa_node, status): """Validate behavior of 'preferred' PCI NUMA affinity policy. This test ensures that it *is* possible to allocate CPU and memory resources from one NUMA node and a PCI device from another *if* the SR-IOV NUMA affinity policy is set to preferred. """ self._test_policy(pci_numa_node, status, 'preferred') @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) # the legacy policy allow a PCI device to be used if it has NUMA # affinity or if no NUMA info is available so we set the NUMA # node for this device to -1 which is the sentinel value use by the # Linux kernel for a device with no NUMA affinity. @ddt.data((-1, 'ACTIVE'), (0, 'ERROR'), (1, 'ACTIVE')) def test_create_server_with_sriov_numa_affinity_policy_legacy( self, pci_numa_node, status): """Validate behavior of 'legacy' PCI NUMA affinity policy. This test ensures that it *is* possible to allocate CPU and memory resources from one NUMA node and a PCI device from another *if* the SR-IOV NUMA affinity policy is set to legacy and the device does not report NUMA information. """ self._test_policy(pci_numa_node, status, 'legacy') @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) # The required policy requires a PCI device to both report a NUMA # and for the guest cpus and ram to be affinitized to the same # NUMA node so we create 1 pci device in the first NUMA node. @ddt.data((-1, 'ERROR'), (0, 'ERROR'), (1, 'ACTIVE')) def test_create_server_with_sriov_numa_affinity_policy_required( self, pci_numa_node, status): """Validate behavior of 'required' PCI NUMA affinity policy. This test ensures that it *is not* possible to allocate CPU and memory resources from one NUMA node and a PCI device from another *if* the SR-IOV NUMA affinity policy is set to required and the device does reports NUMA information. """ # we set the numa_affinity policy to preferred to allow the PCI device # to be selected from any numa node so we can prove the flavor # overrides the alias. alias = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, 'name': self.ALIAS_NAME, 'device_type': fields.PciDeviceType.STANDARD, 'numa_policy': fields.PCINUMAAffinityPolicy.PREFERRED, } )] self.flags(passthrough_whitelist=self.PCI_PASSTHROUGH_WHITELIST, alias=alias, group='pci') self._test_policy(pci_numa_node, status, 'required') def test_socket_policy_pass(self): # With 1 socket containing 2 NUMA nodes, make the first node's CPU # available for pinning, but affine the PCI device to the second node. # This should pass. host_info = fakelibvirt.HostInfo( cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=(16 * units.Gi) // units.Ki) self.flags(cpu_dedicated_set='0-3', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=1) self.start_compute(host_info=host_info, pci_info=pci_info) extra_spec = { 'hw:cpu_policy': 'dedicated', 'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME, 'hw:pci_numa_affinity_policy': 'socket' } flavor_id = self._create_flavor(extra_spec=extra_spec) self._create_server(flavor_id=flavor_id) self.assertTrue(self.mock_filter.called) def test_socket_policy_fail(self): # With 2 sockets containing 1 NUMA node each, make the first socket's # CPUs available for pinning, but affine the PCI device to the second # NUMA node in the second socket. This should fail. host_info = fakelibvirt.HostInfo( cpu_nodes=1, cpu_sockets=2, cpu_cores=2, cpu_threads=2, kB_mem=(16 * units.Gi) // units.Ki) self.flags(cpu_dedicated_set='0-3', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=1) self.start_compute(host_info=host_info, pci_info=pci_info) extra_spec = { 'hw:cpu_policy': 'dedicated', 'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME, 'hw:pci_numa_affinity_policy': 'socket' } flavor_id = self._create_flavor(extra_spec=extra_spec) server = self._create_server( flavor_id=flavor_id, expected_state='ERROR') self.assertIn('fault', server) self.assertIn('No valid host', server['fault']['message']) def test_socket_policy_multi_numa_pass(self): # 2 sockets, 2 NUMA nodes each, with the PCI device on NUMA 0 and # socket 0. If we restrict cpu_dedicated_set to NUMA 1, 2 and 3, we # should still be able to boot an instance with hw:numa_nodes=3 and the # `socket` policy, because one of the instance's NUMA nodes will be on # the same socket as the PCI device (even if there is no direct NUMA # node affinity). host_info = fakelibvirt.HostInfo( cpu_nodes=2, cpu_sockets=2, cpu_cores=2, cpu_threads=1, kB_mem=(16 * units.Gi) // units.Ki) self.flags(cpu_dedicated_set='2-7', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo(num_pci=1, numa_node=0) self.start_compute(host_info=host_info, pci_info=pci_info) extra_spec = { 'hw:numa_nodes': '3', 'hw:cpu_policy': 'dedicated', 'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME, 'hw:pci_numa_affinity_policy': 'socket' } flavor_id = self._create_flavor(vcpu=6, memory_mb=3144, extra_spec=extra_spec) self._create_server(flavor_id=flavor_id) self.assertTrue(self.mock_filter.called) @ddt.ddt class PCIServersWithPortNUMAPoliciesTest(_PCIServersTestBase): ALIAS_NAME = 'a1' PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in ( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PF_PROD_ID, 'physical_network': 'physnet4', }, { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.VF_PROD_ID, 'physical_network': 'physnet4', }, )] # we set the numa_affinity policy to required to ensure strict affinity # between pci devices and the guest cpu and memory will be enforced. PCI_ALIAS = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, 'name': ALIAS_NAME, 'device_type': fields.PciDeviceType.STANDARD, 'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED, } )] def setUp(self): super().setUp() # The ultimate base class _IntegratedTestBase uses NeutronFixture but # we need a bit more intelligent neutron for these tests. Applying the # new fixture here means that we re-stub what the previous neutron # fixture already stubbed. self.neutron = self.useFixture(base.LibvirtNeutronFixture(self)) self.flags(disable_fallback_pcpu_query=True, group='workarounds') def _create_port_with_policy(self, policy): port_data = copy.deepcopy( base.LibvirtNeutronFixture.network_4_port_1) port_data[constants.NUMA_POLICY] = policy # create the port new_port = self.neutron.create_port({'port': port_data}) port_id = new_port['port']['id'] port = self.neutron.show_port(port_id)['port'] self.assertEqual(port[constants.NUMA_POLICY], policy) return port_id # NOTE(sean-k-mooney): i could just apply the ddt decorators # to this function for the most part but i have chosen to # keep one top level function per policy to make documenting # the test cases simpler. def _test_policy(self, pci_numa_node, status, policy): # only allow cpus on numa node 1 to be used for pinning self.flags(cpu_dedicated_set='4-7', group='compute') pci_info = fakelibvirt.HostPCIDevicesInfo( num_pfs=1, num_vfs=2, numa_node=pci_numa_node) self.start_compute(pci_info=pci_info) # request cpu pinning to create a numa toplogy and allow the test to # force which numa node the vm would have to be pinned too. extra_spec = { 'hw:cpu_policy': 'dedicated', } flavor_id = self._create_flavor(extra_spec=extra_spec) port_id = self._create_port_with_policy(policy) # create a server using the VF via neutron self._create_server( flavor_id=flavor_id, networks=[ {'port': port_id}, ], expected_state=status ) if status == 'ACTIVE': self.assertTrue(self.mock_filter.called) else: # the PciPassthroughFilter should not have been called, since the # NUMATopologyFilter should have eliminated the filter first self.assertFalse(self.mock_filter.called) @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) # the preferred policy should always pass regardless of numa affinity @ddt.data((-1, 'ACTIVE'), (0, 'ACTIVE'), (1, 'ACTIVE')) def test_create_server_with_sriov_numa_affinity_policy_preferred( self, pci_numa_node, status): """Validate behavior of 'preferred' PCI NUMA affinity policy. This test ensures that it *is* possible to allocate CPU and memory resources from one NUMA node and a PCI device from another *if* the port NUMA affinity policy is set to preferred. """ self._test_policy(pci_numa_node, status, 'preferred') @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) # the legacy policy allow a PCI device to be used if it has NUMA # affinity or if no NUMA info is available so we set the NUMA # node for this device to -1 which is the sentinel value use by the # Linux kernel for a device with no NUMA affinity. @ddt.data((-1, 'ACTIVE'), (0, 'ERROR'), (1, 'ACTIVE')) def test_create_server_with_sriov_numa_affinity_policy_legacy( self, pci_numa_node, status): """Validate behavior of 'legacy' PCI NUMA affinity policy. This test ensures that it *is* possible to allocate CPU and memory resources from one NUMA node and a PCI device from another *if* the port NUMA affinity policy is set to legacy and the device does not report NUMA information. """ self._test_policy(pci_numa_node, status, 'legacy') @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) # The required policy requires a PCI device to both report a NUMA # and for the guest cpus and ram to be affinitized to the same # NUMA node so we create 1 pci device in the first NUMA node. @ddt.data((-1, 'ERROR'), (0, 'ERROR'), (1, 'ACTIVE')) def test_create_server_with_sriov_numa_affinity_policy_required( self, pci_numa_node, status): """Validate behavior of 'required' PCI NUMA affinity policy. This test ensures that it *is not* possible to allocate CPU and memory resources from one NUMA node and a PCI device from another *if* the port NUMA affinity policy is set to required and the device does reports NUMA information. """ # we set the numa_affinity policy to preferred to allow the PCI device # to be selected from any numa node so we can prove the flavor # overrides the alias. alias = [jsonutils.dumps( { 'vendor_id': fakelibvirt.PCI_VEND_ID, 'product_id': fakelibvirt.PCI_PROD_ID, 'name': self.ALIAS_NAME, 'device_type': fields.PciDeviceType.STANDARD, 'numa_policy': fields.PCINUMAAffinityPolicy.PREFERRED, } )] self.flags(passthrough_whitelist=self.PCI_PASSTHROUGH_WHITELIST, alias=alias, group='pci') self._test_policy(pci_numa_node, status, 'required') def test_socket_policy_pass(self): # With 1 socket containing 2 NUMA nodes, make the first node's CPU # available for pinning, but affine the PCI device to the second node. # This should pass. host_info = fakelibvirt.HostInfo( cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=(16 * units.Gi) // units.Ki) pci_info = fakelibvirt.HostPCIDevicesInfo( num_pfs=1, num_vfs=1, numa_node=1) self.flags(cpu_dedicated_set='0-3', group='compute') self.start_compute(host_info=host_info, pci_info=pci_info) extra_spec = { 'hw:cpu_policy': 'dedicated', } flavor_id = self._create_flavor(extra_spec=extra_spec) port_id = self._create_port_with_policy('socket') # create a server using the VF via neutron self._create_server( flavor_id=flavor_id, networks=[ {'port': port_id}, ], ) self.assertTrue(self.mock_filter.called) def test_socket_policy_fail(self): # With 2 sockets containing 1 NUMA node each, make the first socket's # CPUs available for pinning, but affine the PCI device to the second # NUMA node in the second socket. This should fail. host_info = fakelibvirt.HostInfo( cpu_nodes=1, cpu_sockets=2, cpu_cores=2, cpu_threads=2, kB_mem=(16 * units.Gi) // units.Ki) pci_info = fakelibvirt.HostPCIDevicesInfo( num_pfs=1, num_vfs=1, numa_node=1) self.flags(cpu_dedicated_set='0-3', group='compute') self.start_compute(host_info=host_info, pci_info=pci_info) extra_spec = { 'hw:cpu_policy': 'dedicated', } flavor_id = self._create_flavor(extra_spec=extra_spec) port_id = self._create_port_with_policy('socket') # create a server using the VF via neutron server = self._create_server( flavor_id=flavor_id, networks=[ {'port': port_id}, ], expected_state='ERROR' ) self.assertIn('fault', server) self.assertIn('No valid host', server['fault']['message']) self.assertFalse(self.mock_filter.called) def test_socket_policy_multi_numa_pass(self): # 2 sockets, 2 NUMA nodes each, with the PCI device on NUMA 0 and # socket 0. If we restrict cpu_dedicated_set to NUMA 1, 2 and 3, we # should still be able to boot an instance with hw:numa_nodes=3 and the # `socket` policy, because one of the instance's NUMA nodes will be on # the same socket as the PCI device (even if there is no direct NUMA # node affinity). host_info = fakelibvirt.HostInfo( cpu_nodes=2, cpu_sockets=2, cpu_cores=2, cpu_threads=1, kB_mem=(16 * units.Gi) // units.Ki) pci_info = fakelibvirt.HostPCIDevicesInfo( num_pfs=1, num_vfs=1, numa_node=0) self.flags(cpu_dedicated_set='2-7', group='compute') self.start_compute(host_info=host_info, pci_info=pci_info) extra_spec = { 'hw:numa_nodes': '3', 'hw:cpu_policy': 'dedicated', } flavor_id = self._create_flavor(vcpu=6, memory_mb=3144, extra_spec=extra_spec) port_id = self._create_port_with_policy('socket') # create a server using the VF via neutron self._create_server( flavor_id=flavor_id, networks=[ {'port': port_id}, ], ) self.assertTrue(self.mock_filter.called) class RemoteManagedServersTest(_PCIServersWithMigrationTestBase): ADMIN_API = True microversion = 'latest' PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in ( # A PF with access to physnet4. { 'vendor_id': '15b3', 'product_id': 'a2dc', 'physical_network': 'physnet4', 'remote_managed': 'false', }, # A VF with access to physnet4. { 'vendor_id': '15b3', 'product_id': '1021', 'physical_network': 'physnet4', 'remote_managed': 'true', }, # A PF programmed to forward traffic to an overlay network. { 'vendor_id': '15b3', 'product_id': 'a2d6', 'physical_network': None, 'remote_managed': 'false', }, # A VF programmed to forward traffic to an overlay network. { 'vendor_id': '15b3', 'product_id': '101e', 'physical_network': None, 'remote_managed': 'true', }, )] PCI_ALIAS = [] NUM_PFS = 1 NUM_VFS = 4 vf_ratio = NUM_VFS // NUM_PFS # Min Libvirt version that supports working with PCI VPD. FAKE_LIBVIRT_VERSION = 7_009_000 # 7.9.0 FAKE_QEMU_VERSION = 5_001_000 # 5.1.0 def setUp(self): super().setUp() self.neutron = self.useFixture(base.LibvirtNeutronFixture(self)) self.useFixture(fixtures.MockPatch( 'nova.pci.utils.get_vf_num_by_pci_address', new=mock.MagicMock( side_effect=lambda addr: self._get_pci_function_number(addr)))) self.useFixture(fixtures.MockPatch( 'nova.pci.utils.get_mac_by_pci_address', new=mock.MagicMock( side_effect=( lambda addr: { "0000:80:00.0": "52:54:00:1e:59:42", "0000:81:00.0": "52:54:00:1e:59:01", "0000:82:00.0": "52:54:00:1e:59:02", }.get(addr) ) ) )) @classmethod def _get_pci_function_number(cls, pci_addr: str): """Get a VF function number based on a PCI address. Assume that the PCI ARI capability is enabled (slot bits become a part of a function number). """ _, _, slot, function = parse_address(pci_addr) # The number of PFs is extracted to get a VF number. return int(slot, 16) + int(function, 16) - cls.NUM_PFS def start_compute( self, hostname='test_compute0', host_info=None, pci_info=None, mdev_info=None, vdpa_info=None, libvirt_version=None, qemu_version=None): if not pci_info: pci_info = fakelibvirt.HostPCIDevicesInfo( num_pci=0, num_pfs=0, num_vfs=0) pci_info.add_device( dev_type='PF', bus=0x81, slot=0x0, function=0, iommu_group=42, numa_node=0, vf_ratio=self.vf_ratio, vend_id='15b3', vend_name='Mellanox Technologies', prod_id='a2dc', prod_name='BlueField-3 integrated ConnectX-7 controller', driver_name='mlx5_core', vpd_fields={ 'name': 'MT43244 BlueField-3 integrated ConnectX-7', 'readonly': { 'serial_number': 'MT0000X00001', }, } ) for idx in range(self.NUM_VFS): pci_info.add_device( dev_type='VF', bus=0x81, slot=0x0, function=idx + 1, iommu_group=idx + 43, numa_node=0, vf_ratio=self.vf_ratio, parent=(0x81, 0x0, 0), vend_id='15b3', vend_name='Mellanox Technologies', prod_id='1021', prod_name='MT2910 Family [ConnectX-7]', driver_name='mlx5_core', vpd_fields={ 'name': 'MT2910 Family [ConnectX-7]', 'readonly': { 'serial_number': 'MT0000X00001', }, } ) pci_info.add_device( dev_type='PF', bus=0x82, slot=0x0, function=0, iommu_group=84, numa_node=0, vf_ratio=self.vf_ratio, vend_id='15b3', vend_name='Mellanox Technologies', prod_id='a2d6', prod_name='MT42822 BlueField-2 integrated ConnectX-6', driver_name='mlx5_core', vpd_fields={ 'name': 'MT42822 BlueField-2 integrated ConnectX-6', 'readonly': { 'serial_number': 'MT0000X00002', }, } ) for idx in range(self.NUM_VFS): pci_info.add_device( dev_type='VF', bus=0x82, slot=0x0, function=idx + 1, iommu_group=idx + 85, numa_node=0, vf_ratio=self.vf_ratio, parent=(0x82, 0x0, 0), vend_id='15b3', vend_name='Mellanox Technologies', prod_id='101e', prod_name='ConnectX Family mlx5Gen Virtual Function', driver_name='mlx5_core') return super().start_compute( hostname=hostname, host_info=host_info, pci_info=pci_info, mdev_info=mdev_info, vdpa_info=vdpa_info, libvirt_version=libvirt_version or self.FAKE_LIBVIRT_VERSION, qemu_version=qemu_version or self.FAKE_QEMU_VERSION) def create_remote_managed_tunnel_port(self): dpu_tunnel_port = { 'id': uuids.dpu_tunnel_port, 'network_id': self.neutron.network_3['id'], 'status': 'ACTIVE', 'mac_address': 'fa:16:3e:f0:a4:bb', 'fixed_ips': [ { 'ip_address': '192.168.2.8', 'subnet_id': self.neutron.subnet_3['id'] } ], 'binding:vif_details': {}, 'binding:vif_type': 'ovs', 'binding:vnic_type': 'remote-managed', } self.neutron.create_port({'port': dpu_tunnel_port}) return dpu_tunnel_port def create_remote_managed_physnet_port(self): dpu_physnet_port = { 'id': uuids.dpu_physnet_port, 'network_id': self.neutron.network_4['id'], 'status': 'ACTIVE', 'mac_address': 'd2:0b:fd:99:89:8b', 'fixed_ips': [ { 'ip_address': '192.168.4.10', 'subnet_id': self.neutron.subnet_4['id'] } ], 'binding:vif_details': {}, 'binding:vif_type': 'ovs', 'binding:vnic_type': 'remote-managed', } self.neutron.create_port({'port': dpu_physnet_port}) return dpu_physnet_port def test_create_server_physnet(self): """Create an instance with a tunnel remote-managed port.""" hostname = self.start_compute() num_pci = (self.NUM_PFS + self.NUM_VFS) * 2 self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci) dpu_port = self.create_remote_managed_physnet_port() port = self.neutron.show_port(dpu_port['id'])['port'] self.assertNotIn('binding:profile', port) self._create_server(networks=[{'port': dpu_port['id']}]) # Ensure there is one less VF available and that the PF # is no longer usable. self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2) # Ensure the binding:profile details sent to Neutron are correct after # a port update. port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual({ 'card_serial_number': 'MT0000X00001', 'pci_slot': '0000:81:00.4', 'pci_vendor_info': '15b3:1021', 'pf_mac_address': '52:54:00:1e:59:01', 'physical_network': 'physnet4', 'vf_num': 3 }, port['binding:profile']) def test_create_server_tunnel(self): """Create an instance with a tunnel remote-managed port.""" hostname = self.start_compute() num_pci = (self.NUM_PFS + self.NUM_VFS) * 2 self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci) dpu_port = self.create_remote_managed_tunnel_port() port = self.neutron.show_port(dpu_port['id'])['port'] self.assertNotIn('binding:profile', port) self._create_server(networks=[{'port': dpu_port['id']}]) # Ensure there is one less VF available and that the PF # is no longer usable. self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2) # Ensure the binding:profile details sent to Neutron are correct after # a port update. port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual({ 'card_serial_number': 'MT0000X00002', 'pci_slot': '0000:82:00.4', 'pci_vendor_info': '15b3:101e', 'pf_mac_address': '52:54:00:1e:59:02', 'physical_network': None, 'vf_num': 3 }, port['binding:profile']) def _test_common(self, op, *args, **kwargs): self.start_compute() dpu_port = self.create_remote_managed_tunnel_port() server = self._create_server(networks=[{'port': dpu_port['id']}]) op(server, *args, **kwargs) def test_attach_interface(self): self.start_compute() dpu_port = self.create_remote_managed_tunnel_port() server = self._create_server(networks='none') self._attach_interface(server, dpu_port['id']) port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:82:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:02', 'vf_num': 3, 'card_serial_number': 'MT0000X00002', }, port['binding:profile'], ) def test_detach_interface(self): self._test_common(self._detach_interface, uuids.dpu_tunnel_port) port = self.neutron.show_port(uuids.dpu_tunnel_port)['port'] self.assertIn('binding:profile', port) self.assertEqual({}, port['binding:profile']) def test_shelve(self): self._test_common(self._shelve_server) port = self.neutron.show_port(uuids.dpu_tunnel_port)['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:82:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:02', 'vf_num': 3, 'card_serial_number': 'MT0000X00002', }, port['binding:profile'], ) def test_suspend(self): self.start_compute() dpu_port = self.create_remote_managed_tunnel_port() server = self._create_server(networks=[{'port': dpu_port['id']}]) self._suspend_server(server) # TODO(dmitriis): detachDevice does not properly handle hostdevs # so full suspend/resume testing is problematic. def _test_move_operation_with_neutron(self, move_operation, dpu_port): """Test a move operation with a remote-managed port. """ compute1_pci_info = fakelibvirt.HostPCIDevicesInfo( num_pfs=0, num_vfs=0) compute1_pci_info.add_device( dev_type='PF', bus=0x80, slot=0x0, function=0, iommu_group=84, numa_node=1, vf_ratio=self.vf_ratio, vend_id='15b3', vend_name='Mellanox Technologies', prod_id='a2d6', prod_name='MT42822 BlueField-2 integrated ConnectX-6', driver_name='mlx5_core', vpd_fields={ 'name': 'MT42822 BlueField-2 integrated ConnectX-6', 'readonly': { 'serial_number': 'MT0000X00042', }, } ) for idx in range(self.NUM_VFS): compute1_pci_info.add_device( dev_type='VF', bus=0x80, slot=0x0, function=idx + 1, iommu_group=idx + 85, numa_node=1, vf_ratio=self.vf_ratio, parent=(0x80, 0x0, 0), vend_id='15b3', vend_name='Mellanox Technologies', prod_id='101e', prod_name='ConnectX Family mlx5Gen Virtual Function', driver_name='mlx5_core', vpd_fields={ 'name': 'MT42822 BlueField-2 integrated ConnectX-6', 'readonly': { 'serial_number': 'MT0000X00042', }, } ) self.start_compute(hostname='test_compute0') self.start_compute(hostname='test_compute1', pci_info=compute1_pci_info) port = self.neutron.show_port(dpu_port['id'])['port'] self.assertNotIn('binding:profile', port) flavor_id = self._create_flavor(vcpu=4) server = self._create_server( flavor_id=flavor_id, networks=[{'port': dpu_port['id']}], host='test_compute0', ) self.assertEqual('test_compute0', server['OS-EXT-SRV-ATTR:host']) self.assertPCIDeviceCounts('test_compute0', total=10, free=8) self.assertPCIDeviceCounts('test_compute1', total=5, free=5) port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:82:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:02', 'vf_num': 3, 'card_serial_number': 'MT0000X00002', }, port['binding:profile'], ) move_operation(server) def test_unshelve_server_with_neutron(self): def move_operation(source_server): self._shelve_server(source_server) # Disable the source compute, to force unshelving on the dest. self.api.put_service( self.computes['test_compute0'].service_ref.uuid, {'status': 'disabled'}) self._unshelve_server(source_server) dpu_port = self.create_remote_managed_tunnel_port() self._test_move_operation_with_neutron(move_operation, dpu_port) self.assertPCIDeviceCounts('test_compute0', total=10, free=10) self.assertPCIDeviceCounts('test_compute1', total=5, free=3) # Ensure the binding:profile details got updated, including the # fields relevant to remote-managed ports. port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:80:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:42', 'vf_num': 3, 'card_serial_number': 'MT0000X00042', }, port['binding:profile'], ) def test_cold_migrate_server_with_neutron(self): def move_operation(source_server): with mock.patch('nova.virt.libvirt.driver.LibvirtDriver' '.migrate_disk_and_power_off', return_value='{}'): server = self._migrate_server(source_server) self._confirm_resize(server) self.assertPCIDeviceCounts('test_compute0', total=10, free=10) self.assertPCIDeviceCounts('test_compute1', total=5, free=3) # Ensure the binding:profile details got updated, including the # fields relevant to remote-managed ports. port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:80:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:42', 'vf_num': 3, 'card_serial_number': 'MT0000X00042', }, port['binding:profile'], ) dpu_port = self.create_remote_managed_tunnel_port() self._test_move_operation_with_neutron(move_operation, dpu_port) def test_cold_migrate_server_with_neutron_revert(self): def move_operation(source_server): with mock.patch('nova.virt.libvirt.driver.LibvirtDriver' '.migrate_disk_and_power_off', return_value='{}'): server = self._migrate_server(source_server) self.assertPCIDeviceCounts('test_compute0', total=10, free=8) self.assertPCIDeviceCounts('test_compute1', total=5, free=3) self._revert_resize(server) self.assertPCIDeviceCounts('test_compute0', total=10, free=8) self.assertPCIDeviceCounts('test_compute1', total=5, free=5) port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:82:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:02', 'vf_num': 3, 'card_serial_number': 'MT0000X00002', }, port['binding:profile'], ) dpu_port = self.create_remote_managed_tunnel_port() self._test_move_operation_with_neutron(move_operation, dpu_port) def test_evacuate_server_with_neutron(self): def move_operation(source_server): # Down the source compute to enable the evacuation self.api.put_service( self.computes['test_compute0'].service_ref.uuid, {'forced_down': True}) self.computes['test_compute0'].stop() self._evacuate_server(source_server) dpu_port = self.create_remote_managed_tunnel_port() self._test_move_operation_with_neutron(move_operation, dpu_port) self.assertPCIDeviceCounts('test_compute0', total=10, free=8) self.assertPCIDeviceCounts('test_compute1', total=5, free=3) # Ensure the binding:profile details got updated, including the # fields relevant to remote-managed ports. port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:80:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:42', 'vf_num': 3, 'card_serial_number': 'MT0000X00042', }, port['binding:profile'], ) def test_live_migrate_server_with_neutron(self): """Live migrate an instance using a remote-managed port. This should succeed since we support this via detach and attach of the PCI device similar to how this is done for SR-IOV ports. """ def move_operation(source_server): self._live_migrate(source_server, 'completed') dpu_port = self.create_remote_managed_tunnel_port() self._test_move_operation_with_neutron(move_operation, dpu_port) self.assertPCIDeviceCounts('test_compute0', total=10, free=10) self.assertPCIDeviceCounts('test_compute1', total=5, free=3) # Ensure the binding:profile details got updated, including the # fields relevant to remote-managed ports. port = self.neutron.show_port(dpu_port['id'])['port'] self.assertIn('binding:profile', port) self.assertEqual( { 'pci_vendor_info': '15b3:101e', 'pci_slot': '0000:80:00.4', 'physical_network': None, 'pf_mac_address': '52:54:00:1e:59:42', 'vf_num': 3, 'card_serial_number': 'MT0000X00042', }, port['binding:profile'], )