223 lines
8.8 KiB
Python
223 lines
8.8 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import copy
|
|
import fixtures
|
|
import time
|
|
|
|
from oslo_config import cfg
|
|
|
|
from nova import context
|
|
from nova import objects
|
|
from nova import test
|
|
from nova.tests import fixtures as nova_fixtures
|
|
from nova.tests.functional import fixtures as func_fixtures
|
|
from nova.tests.functional import integrated_helpers
|
|
from nova import utils
|
|
from nova.virt import fake
|
|
|
|
|
|
CONF = cfg.CONF
|
|
|
|
|
|
class TestEvacuateResourceTrackerRace(
|
|
test.TestCase, integrated_helpers.InstanceHelperMixin,
|
|
):
|
|
"""Demonstrate bug #1896463.
|
|
|
|
Trigger a race condition between an almost finished evacuation that is
|
|
dropping the migration context, and the _update_available_resource()
|
|
periodic task that already loaded the instance list but haven't loaded the
|
|
migration list yet. The result is that the PCI allocation made by the
|
|
evacuation is deleted by the overlapping periodic task run and the instance
|
|
will not have PCI allocation after the evacuation.
|
|
"""
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.neutron = self.useFixture(nova_fixtures.NeutronFixture(self))
|
|
self.glance = self.useFixture(nova_fixtures.GlanceFixture(self))
|
|
self.placement = self.useFixture(func_fixtures.PlacementFixture()).api
|
|
|
|
self.api_fixture = self.useFixture(nova_fixtures.OSAPIFixture(
|
|
api_version='v2.1'))
|
|
|
|
self.admin_api = self.api_fixture.admin_api
|
|
self.admin_api.microversion = 'latest'
|
|
self.api = self.admin_api
|
|
|
|
self.start_service('conductor')
|
|
self.start_service('scheduler')
|
|
|
|
self.flags(compute_driver='fake.FakeDriverWithPciResources')
|
|
self.useFixture(
|
|
fake.FakeDriverWithPciResources.
|
|
FakeDriverWithPciResourcesConfigFixture())
|
|
|
|
self.compute1 = self._start_compute('host1')
|
|
self.compute1_id = self._get_compute_node_id_by_host('host1')
|
|
self.compute1_service_id = self.admin_api.get_services(
|
|
host='host1', binary='nova-compute')[0]['id']
|
|
|
|
self.compute2 = self._start_compute('host2')
|
|
self.compute2_id = self._get_compute_node_id_by_host('host2')
|
|
self.compute2_service_id = self.admin_api.get_services(
|
|
host='host2', binary='nova-compute')[0]['id']
|
|
|
|
# add extra ports and the related network to the neutron fixture
|
|
# specifically for these tests. It cannot be added globally in the
|
|
# fixture init as it adds a second network that makes auto allocation
|
|
# based test to fail due to ambiguous networks.
|
|
self.neutron._ports[self.neutron.sriov_port['id']] = \
|
|
copy.deepcopy(self.neutron.sriov_port)
|
|
self.neutron._networks[
|
|
self.neutron.network_2['id']] = self.neutron.network_2
|
|
self.neutron._subnets[
|
|
self.neutron.subnet_2['id']] = self.neutron.subnet_2
|
|
|
|
self.ctxt = context.get_admin_context()
|
|
|
|
def _get_compute_node_id_by_host(self, host):
|
|
# we specifically need the integer id of the node not the UUID so we
|
|
# need to use the old microversion
|
|
with utils.temporary_mutation(self.admin_api, microversion='2.52'):
|
|
hypers = self.admin_api.api_get(
|
|
'os-hypervisors').body['hypervisors']
|
|
for hyper in hypers:
|
|
if hyper['hypervisor_hostname'] == host:
|
|
return hyper['id']
|
|
|
|
self.fail('Hypervisor with hostname=%s not found' % host)
|
|
|
|
def _assert_pci_device_allocated(
|
|
self, instance_uuid, compute_node_id, num=1):
|
|
"""Assert that a given number of PCI devices are allocated to the
|
|
instance on the given host.
|
|
"""
|
|
|
|
devices = objects.PciDeviceList.get_by_instance_uuid(
|
|
self.ctxt, instance_uuid)
|
|
devices_on_host = [dev for dev in devices
|
|
if dev.compute_node_id == compute_node_id]
|
|
self.assertEqual(num, len(devices_on_host))
|
|
|
|
def test_evacuate_races_with_update_available_resource(self):
|
|
# Create a server with a direct port to have PCI allocation
|
|
server = self._create_server(
|
|
name='test-server-for-bug-1896463',
|
|
networks=[{'port': self.neutron.sriov_port['id']}],
|
|
host='host1'
|
|
)
|
|
|
|
self._assert_pci_device_allocated(server['id'], self.compute1_id)
|
|
self._assert_pci_device_allocated(
|
|
server['id'], self.compute2_id, num=0)
|
|
|
|
# stop and force down the compute the instance is on to allow
|
|
# evacuation
|
|
self.compute1.stop()
|
|
self.admin_api.put_service(
|
|
self.compute1_service_id, {'forced_down': 'true'})
|
|
|
|
# Inject some sleeps both in the Instance.drop_migration_context and
|
|
# the MigrationList.get_in_progress_and_error code to make them
|
|
# overlap.
|
|
# We want to create the following execution scenario:
|
|
# 1) The evacuation makes a move claim on the dest including the PCI
|
|
# claim. This means there is a migration context. But the evacuation
|
|
# is not complete yet so the instance.host does not point to the
|
|
# dest host.
|
|
# 2) The dest resource tracker starts an _update_available_resource()
|
|
# periodic task and this task loads the list of instances on its
|
|
# host from the DB. Our instance is not in this list due to #1.
|
|
# 3) The evacuation finishes, the instance.host is set to the dest host
|
|
# and the migration context is deleted.
|
|
# 4) The periodic task now loads the list of in-progress migration from
|
|
# the DB to check for incoming our outgoing migrations. However due
|
|
# to #3 our instance is not in this list either.
|
|
# 5) The periodic task cleans up every lingering PCI claim that is not
|
|
# connected to any instance collected above from the instance list
|
|
# and from the migration list. As our instance is not in either of
|
|
# the lists, the resource tracker cleans up the PCI allocation for
|
|
# the already finished evacuation of our instance.
|
|
#
|
|
# Unfortunately we cannot reproduce the above situation without sleeps.
|
|
# We need that the evac starts first then the periodic starts, but not
|
|
# finishes, then evac finishes, then periodic finishes. If I trigger
|
|
# and run the whole periodic in a wrapper of drop_migration_context
|
|
# then I could not reproduce the situation described at #4). In general
|
|
# it is not
|
|
#
|
|
# evac
|
|
# |
|
|
# |
|
|
# | periodic
|
|
# | |
|
|
# | |
|
|
# | x
|
|
# |
|
|
# |
|
|
# x
|
|
#
|
|
# but
|
|
#
|
|
# evac
|
|
# |
|
|
# |
|
|
# | periodic
|
|
# | |
|
|
# | |
|
|
# | |
|
|
# x |
|
|
# |
|
|
# x
|
|
#
|
|
# what is needed need.
|
|
#
|
|
# Starting the periodic from the test in a separate thread at
|
|
# drop_migration_context() might work but that is an extra complexity
|
|
# in the test code. Also it might need a sleep still to make the
|
|
# reproduction stable but only one sleep instead of two.
|
|
orig_drop = objects.Instance.drop_migration_context
|
|
|
|
def slow_drop(*args, **kwargs):
|
|
time.sleep(1)
|
|
return orig_drop(*args, **kwargs)
|
|
|
|
self.useFixture(
|
|
fixtures.MockPatch(
|
|
'nova.objects.instance.Instance.drop_migration_context',
|
|
new=slow_drop))
|
|
|
|
orig_get_mig = objects.MigrationList.get_in_progress_and_error
|
|
|
|
def slow_get_mig(*args, **kwargs):
|
|
time.sleep(2)
|
|
return orig_get_mig(*args, **kwargs)
|
|
|
|
self.useFixture(
|
|
fixtures.MockPatch(
|
|
'nova.objects.migration.MigrationList.'
|
|
'get_in_progress_and_error',
|
|
new=slow_get_mig))
|
|
|
|
self.admin_api.post_server_action(server['id'], {'evacuate': {}})
|
|
# we trigger the _update_available_resource periodic to overlap with
|
|
# the already started evacuation
|
|
self._run_periodics()
|
|
|
|
self._wait_for_server_parameter(
|
|
server, {'OS-EXT-SRV-ATTR:host': 'host2', 'status': 'ACTIVE'})
|
|
|
|
self._assert_pci_device_allocated(server['id'], self.compute1_id)
|
|
self._assert_pci_device_allocated(server['id'], self.compute2_id)
|