# Copyright 2020 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import collections import fixtures import mock import os.path from oslo_utils import fileutils from oslo_utils.fixture import uuidsentinel as uuids from oslo_utils import units from nova import conf from nova import context from nova import exception from nova import objects from nova import test from nova.tests import fixtures as nova_fixtures from nova.tests.fixtures import libvirt as fakelibvirt from nova.tests.functional import fixtures as func_fixtures from nova.tests.functional import integrated_helpers from nova.tests.unit import fake_network from nova.virt.libvirt import config as libvirt_config CONF = conf.CONF FLAVOR_FIXTURES = [ {'flavorid': 'root_only', 'name': 'root_only', 'vcpus': 1, 'memory_mb': 512, 'root_gb': 1, 'ephemeral_gb': 0, 'swap': 0}, {'flavorid': 'with_ephemeral', 'name': 'with_ephemeral', 'vcpus': 1, 'memory_mb': 512, 'root_gb': 1, 'ephemeral_gb': 1, 'swap': 0}, {'flavorid': 'with_swap', 'name': 'with_swap', 'vcpus': 1, 'memory_mb': 512, 'root_gb': 1, 'ephemeral_gb': 0, 'swap': 1}, ] # Choice of image id is arbitrary, but fixed for consistency. IMAGE_ID = nova_fixtures.GlanceFixture.auto_disk_config_enabled_image['id'] # NOTE(mdbooth): Change I76448196 tests for creation of any local disk, and # short-circuits as soon as it sees one created. Disks are created in order: # root disk, ephemeral disks, swap disk. Therefore to test correct handling of # ephemeral disks we must ensure there is no root disk, and to test swap disks # we must ensure there is no root or ephemeral disks. Each of the following # fixtures intentionally has only a single local disk (or none for bfv), # ensuring we cover all local disks. SERVER_FIXTURES = [ # Local root disk only {'name': 'local_root', 'imageRef': IMAGE_ID, 'flavorRef': 'root_only', }, # No local disks {'name': 'bfv', 'flavorRef': 'root_only', 'block_device_mapping_v2': [{ 'boot_index': 0, 'uuid': uuids.vol1, 'source_type': 'volume', 'destination_type': 'volume', }], }, # Local eph disk only {'name': 'bfv_with_eph', 'flavorRef': 'with_ephemeral', 'block_device_mapping_v2': [{ 'boot_index': 0, 'uuid': uuids.vol2, 'source_type': 'volume', 'destination_type': 'volume', }], }, # Local swap disk only {'name': 'bfv_with_swap', 'flavorRef': 'with_swap', 'block_device_mapping_v2': [{ 'boot_index': 0, 'uuid': uuids.vol3, 'source_type': 'volume', 'destination_type': 'volume', }], }, ] SERVER_DISKS = { 'local_root': 'disk', 'bfv': None, 'bfv_with_eph': 'disk.eph0', 'bfv_with_swap': 'disk.swap', } class _FileTest(object): """A base class for the _FlatTest and _Qcow2Test mixin test classes""" def setUp(self): super(_FileTest, self).setUp() def assert_disks_nonshared_instancedir(self, server): name = server['name'] disk = SERVER_DISKS[name] if not disk: return source_root_disk = os.path.join(self.source_instance_path(server), disk) dest_root_disk = os.path.join(self.dest_instance_path(server), disk) self.assertTrue(os.path.exists(source_root_disk), "Source root disk %s for server %s does not exist" % (source_root_disk, name)) self.assertFalse(os.path.exists(dest_root_disk), "Destination root disk %s for server %s exists" % (dest_root_disk, name)) def assert_disks_shared_instancedir(self, server): name = server['name'] disk = SERVER_DISKS[name] if not disk: return source_root_disk = os.path.join( self.source_instance_path(server), disk) self.assertTrue(os.path.exists(source_root_disk), "Source root disk %s for server %s does not exist" % (source_root_disk, name)) class _FlatTest(_FileTest): """A mixin which configures the flat imagebackend, and provides assertions for the expected state of the flat imagebackend after an evacuation. We mock create_image to touch a file so we can assert its existence/removal in tests. """ def setUp(self): super(_FlatTest, self).setUp() self.flags(group='libvirt', images_type='flat') def fake_create_image(_self, *args, **kwargs): # Simply ensure the file exists open(_self.path, 'a').close() self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.Flat.create_image', fake_create_image)) # Mocked to reduce runtime self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.imagebackend.Flat.correct_format')) class _Qcow2Test(_FileTest): """A mixin which configures the qcow2 imagebackend, and provides assertions for the expected state of the flat imagebackend after an evacuation. We mock create_image to touch a file so we can assert its existence/removal in tests. """ def setUp(self): super(_Qcow2Test, self).setUp() self.flags(group='libvirt', images_type='qcow2') def fake_create_image(_self, *args, **kwargs): # Simply ensure the file exists open(_self.path, 'a').close() self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.Qcow2.create_image', fake_create_image)) class _RbdTest(object): """A mixin which configures the rbd imagebackend, and provides assertions for the expected state of the rbd imagebackend after an evacuation. We mock RBDDriver so we don't need an actual ceph cluster. We mock create_image to store which rbd volumes would have been created, and exists to reference that store. """ def setUp(self): super(_RbdTest, self).setUp() self.flags(group='libvirt', images_type='rbd') self.flags(group='libvirt', rbd_secret_uuid='1234') self.created = set() def fake_create_image(_self, *args, **kwargs): self.created.add(_self.rbd_name) def fake_exists(_self): return _self.rbd_name in self.created self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.Rbd.create_image', fake_create_image)) self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.Rbd.exists', fake_exists)) # We never want to actually touch rbd self.mock_rbd_driver = self.useFixture(fixtures.MockPatch( 'nova.storage.rbd_utils.RBDDriver')).mock.return_value self.mock_rbd_driver.get_mon_addrs.return_value = ([], []) self.mock_rbd_driver.size.return_value = 10 * units.Gi self.mock_rbd_driver.rbd_user = 'rbd' def _assert_disks(self, server): name = server['name'] disk = SERVER_DISKS[name] if not disk: return # Check that we created a root disk and haven't called _cleanup_rbd at # all self.assertIn("%s_%s" % (server['id'], disk), self.created) self.mock_rbd_driver.cleanup_volumes.assert_not_called() # We never want to cleanup rbd disks during evacuate, regardless of # instance shared storage assert_disks_nonshared_instancedir = _assert_disks assert_disks_shared_instancedir = _assert_disks class FakeLVM(object): def __init__(self): self.volumes = set() def _exists(self, vg, lv): return any([v for v in self.volumes if v[0] == vg and v[1] == lv]) def _vg_exists(self, vg): return any([v for v in self.volumes if v[0] == vg]) def _find_vol_from_path(self, path): info = self.volume_info(path) for vol in self.volumes: if vol[0] == info['VG'] and vol[1] == info['LV']: return vol return None def create_volume(self, vg, lv, size, sparse=False): self.volumes.add((vg, lv, size)) def list_volumes(self, vg_path): _, vg = os.path.split(vg_path) return [vol[1] for vol in self.volumes if vol[0] == vg] def volume_info(self, path): path, lv = os.path.split(path) path, vg = os.path.split(path) return {'VG': vg, 'LV': lv} def get_volume_size(self, path): vol = self._find_vol_from_path(path) if vol is not None: return vol[2] raise exception.VolumeBDMPathNotFound(path=path) def remove_volumes(self, paths): for path in paths: vol = self._find_vol_from_path(path) if vol is not None: self.volumes.remove(vol) class _LVMTest(object): """A mixin which configures the LVM imagebackend, and provides assertions for the expected state of the LVM imagebackend after an evacuation. We need to track logical volumes on each compute separately, which we do by mocking the nova.virt.libvirt.storage.lvm module immediately before starting a new compute. """ def setUp(self): super(_LVMTest, self).setUp() self.flags(group='libvirt', images_type='lvm', images_volume_group='fake_vg') # A map of compute service name: fake libvirt module self.fake_lvms = collections.defaultdict(FakeLVM) # The fake libvirt module in use by the compute service which is # running currently self.fake_lvm = None def fake_create_image(_self, prepare_template, base, size, *args, **kwargs): self.fake_lvm.create_volume(_self.vg, _self.lv, size) def fake_exists(_self, *args, **kwargs): return self.fake_lvm._exists(_self.vg, _self.lv) self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.Lvm.create_image', fake_create_image)) self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.Lvm.exists', fake_exists)) orig_path_exists = os.path.exists def fake_path_exists(path): if path.startswith('/dev/'): paths = path.split(os.sep)[2:] if len(paths) == 0: # For completeness: /dev exists return True if len(paths) == 1: return self.fake_lvm._vg_exists(*paths) if len(paths) == 2: return self.fake_lvm._exists(*paths) return False else: return orig_path_exists(path) self.useFixture(fixtures.MonkeyPatch( 'os.path.exists', fake_path_exists)) def _start_compute(self, name): compute = super(_LVMTest, self)._start_compute(name) # We need each compute to have its own fake LVM. These mocks replace # the previous mocks, globally. This only works because in this test we # only ever run one of the computes at a time. self.fake_lvm = self.fake_lvms[name] self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.driver.lvm', self.fake_lvm)) self.useFixture(fixtures.MonkeyPatch( 'nova.virt.libvirt.imagebackend.lvm', self.fake_lvm)) return compute def _assert_disks(self, server): name = server['name'] disk = SERVER_DISKS[name] if not disk: return vg = CONF.libvirt.images_volume_group lv = '{uuid}_{disk}'.format(uuid=server['id'], disk=disk) compute0 = self.fake_lvms['compute0'] compute1 = self.fake_lvms['compute1'] self.assertTrue(compute0._exists(vg, lv), 'Disk "{disk}" of server {server} does not exist on ' 'source'.format(disk=disk, server=name)) self.assertFalse(compute1._exists(vg, lv), 'Disk "{disk}" of server {server} still exists on ' 'destination'.format(disk=disk, server=name)) # We always want to cleanup LVM disks on failure, regardless of shared # instance directory assert_disks_nonshared_instancedir = _assert_disks assert_disks_shared_instancedir = _assert_disks class _LibvirtEvacuateTest(integrated_helpers.InstanceHelperMixin): """The main libvirt evacuate test. This configures a set of stub services with 2 computes and defines 2 tests, both of which create a server on compute0 and then evacuate it to compute1. test_evacuate_failure_nonshared_instancedir does this with a non-shared instance directory, and test_evacuate_failure_shared_instancedir does this with a shared instance directory. This class requires one of the mixins _FlatTest, _RbdTest, _LVMTest, or _Qcow2Test to execute. These configure an imagebackend, and define the assertions assert_disks_nonshared_instancedir and assert_disks_shared_instancedir to assert the expected state of that imagebackend after an evacuation. By combining shared and non-shared instance directory tests in this class with these mixins we get test coverage of all combinations of shared/nonshared instanace directories and block storage. """ def _start_compute(self, name): # NOTE(mdbooth): fakelibvirt's getHostname currently returns a # hardcoded 'compute1', which is undesirable if we want multiple fake # computes. There's no good way to pre-initialise get_connection() to # return a fake libvirt with a custom return for getHostname. # # Here we mock the class during service creation to return our custom # hostname, but we can't leave this in place because then both computes # will still get the same value from their libvirt Connection. Once the # service has started, we poke a custom getHostname into the # instantiated object to do the same thing, but only for that object. with mock.patch.object(fakelibvirt.Connection, 'getHostname', return_value=name): compute = self.start_service('compute', host=name) compute.driver._host.get_connection().getHostname = lambda: name return compute def setUp(self): super(_LibvirtEvacuateTest, self).setUp() self.useFixture(nova_fixtures.CinderFixture(self)) self.useFixture(nova_fixtures.NeutronFixture(self)) self.useFixture(nova_fixtures.GlanceFixture(self)) self.useFixture(func_fixtures.PlacementFixture()) fake_network.set_stub_network_methods(self) api_fixture = self.useFixture( nova_fixtures.OSAPIFixture(api_version='v2.1')) self.api = api_fixture.admin_api # force_down and evacuate without onSharedStorage self.api.microversion = '2.14' self.notifier = self.useFixture( nova_fixtures.NotificationFixture(self)) self.useFixture(nova_fixtures.LibvirtFixture()) # Fake out all the details of volume connection self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.driver.LibvirtDriver.get_volume_connector')) self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.driver.LibvirtDriver._connect_volume')) # For cleanup self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.driver.LibvirtDriver._disconnect_volume')) volume_config = libvirt_config.LibvirtConfigGuestDisk() volume_config.driver_name = 'fake-volume-driver' volume_config.source_path = 'fake-source-path' volume_config.target_dev = 'fake-target-dev' volume_config.target_bus = 'fake-target-bus' get_volume_config = self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.driver.LibvirtDriver._get_volume_config')).mock get_volume_config.return_value = volume_config # Ensure our computes report lots of available disk, vcpu, and ram lots = 10000000 get_local_gb_info = self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.driver.LibvirtDriver._get_local_gb_info')).mock get_local_gb_info.return_value = { 'total': lots, 'free': lots, 'used': 1} get_vcpu_available = self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.driver.LibvirtDriver._get_vcpu_available')).mock get_vcpu_available.return_value = set(cpu for cpu in range(24)) get_memory_mb_total = self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.host.Host.get_memory_mb_total')).mock get_memory_mb_total.return_value = lots # Mock out adding rng devices self.useFixture(fixtures.MockPatch( 'nova.virt.libvirt.driver.LibvirtDriver._add_rng_device')).mock self.start_service('conductor') self.start_service('scheduler') self.flags(compute_driver='libvirt.LibvirtDriver') ctxt = context.get_admin_context() for flavor in FLAVOR_FIXTURES: objects.Flavor(context=ctxt, **flavor).create() @staticmethod def source_instance_path(server): return os.path.join(CONF.instances_path, server['id']) @staticmethod def dest_instance_path(server): return os.path.join(CONF.instances_path, 'dest', server['id']) def _create_servers(self): def _create_server(server): # NOTE(mdbooth): We could do all the server creations concurrently # to improve throughput, but we have seen this result in timeouts # on a loaded CI worker. server = self.api.post_server({'server': server}) # Wait for server to become ACTIVE, and return its updated state # NOTE(mdbooth): Increase max_retries from default to reduce # chances of timeout. return self._wait_for_state_change(server, 'ACTIVE', max_retries=30) return [_create_server(server) for server in SERVER_FIXTURES] def _swap_computes(self, compute0): # Force compute0 down compute0.stop() self.api.force_down_service('compute0', 'nova-compute', True) # Start compute1 return self._start_compute('compute1') def _evacuate_with_failure(self, server, compute1): # Perform an evacuation during which we experience a failure on the # destination host with mock.patch.object(compute1.driver, 'plug_vifs') as plug_vifs: plug_vifs.side_effect = test.TestingException server = self._evacuate_server( server, {'host': 'compute1'}, expected_state='ERROR', expected_task_state=None, expected_migration_status='failed') # Wait for the rebuild to start, then complete self.notifier.wait_for_versioned_notifications( 'instance.rebuild.start') # Meta-test plug_vifs.assert_called() plug_vifs.reset_mock() # Return fresh server state after evacuate return server def test_evacuate_failure_nonshared_instancedir(self): """Assert the failure cleanup behaviour of non-shared instance storage If we fail during evacuate and the instance directory didn't previously exist on the destination, we should delete it """ # Create instances on compute0 compute0 = self._start_compute('compute0') servers = self._create_servers() compute1 = self._swap_computes(compute0) # Create a 'pass-through' mock for ensure_tree so we can log its calls orig_ensure_tree = fileutils.ensure_tree mock_ensure_tree = self.useFixture(fixtures.MockPatch( 'oslo_utils.fileutils.ensure_tree', side_effect=orig_ensure_tree)).mock for server in servers: name = server['name'] source_instance_path = self.source_instance_path(server) dest_instance_path = self.dest_instance_path(server) # Check that we've got an instance directory on the source and not # on the dest self.assertTrue(os.path.exists(source_instance_path), "Source instance directory %s for server %s does " "not exist" % (source_instance_path, name)) self.assertFalse(os.path.exists(dest_instance_path), "Destination instance directory %s for server %s " "exists" % (dest_instance_path, name)) # By default our 2 compute hosts share the same instance directory # on the test runner. Force a different directory while running # evacuate on compute1 so we don't have shared storage. def dest_get_instance_path(instance, relative=False): if relative: return instance.uuid return dest_instance_path with mock.patch('nova.virt.libvirt.utils.get_instance_path') \ as get_instance_path: get_instance_path.side_effect = dest_get_instance_path server = self._evacuate_with_failure(server, compute1) # Check that we've got an instance directory on the source and not # on the dest, but that the dest was created self.assertTrue(os.path.exists(source_instance_path), "Source instance directory %s for server %s does " "not exist" % (source_instance_path, name)) self.assertFalse(os.path.exists(dest_instance_path), "Destination instance directory %s for server %s " "exists" % (dest_instance_path, name)) mock_ensure_tree.assert_called_with(dest_instance_path) self.assert_disks_nonshared_instancedir(server) # Check we're still on the failed source host self.assertEqual('compute0', server['OS-EXT-SRV-ATTR:host']) def test_evacuate_failure_shared_instancedir(self): """Assert the failure cleanup behaviour of shared instance storage If we fail during evacuate and the instance directory was already present on the destination, we should leave it there By default our 2 compute hosts share the same instance directory on the test runner. """ # Create test instances on compute0 compute0 = self._start_compute('compute0') servers = self._create_servers() compute1 = self._swap_computes(compute0) for server in servers: name = server['name'] shared_instance_path = self.source_instance_path(server) # Check that we've got an instance directory on the source self.assertTrue(os.path.exists(shared_instance_path), "Shared instance directory %s for server %s does " "not exist" % (shared_instance_path, name)) server = self._evacuate_with_failure(server, compute1) # Check that the instance directory still exists self.assertTrue(os.path.exists(shared_instance_path), "Shared instance directory %s for server %s " "does not exist" % (shared_instance_path, name)) self.assert_disks_shared_instancedir(server) # Check we're still on the failed source host self.assertEqual('compute0', server['OS-EXT-SRV-ATTR:host']) class LibvirtFlatEvacuateTest(_FlatTest, _LibvirtEvacuateTest, test.TestCase): pass class LibvirtQcowEvacuateTest(_Qcow2Test, _LibvirtEvacuateTest, test.TestCase): pass class LibvirtRbdEvacuateTest(_RbdTest, _LibvirtEvacuateTest, test.TestCase): pass class LibvirtLVMEvacuateTest(_LVMTest, _LibvirtEvacuateTest, test.TestCase): pass