nova/nova/tests/unit/compute/test_shelve.py

1101 lines
51 KiB
Python

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import eventlet
import mock
from oslo_utils import fixture as utils_fixture
from oslo_utils.fixture import uuidsentinel as uuids
from oslo_utils import timeutils
from nova.compute import api as compute_api
from nova.compute import claims
from nova.compute import instance_actions
from nova.compute import power_state
from nova.compute import task_states
from nova.compute import utils as compute_utils
from nova.compute import vm_states
import nova.conf
from nova.db import api as db
from nova import exception
from nova.network import neutron as neutron_api
from nova import objects
from nova import test
from nova.tests import fixtures
from nova.tests.unit.compute import test_compute
CONF = nova.conf.CONF
def _fake_resources():
resources = {
'memory_mb': 2048,
'memory_mb_used': 0,
'free_ram_mb': 2048,
'local_gb': 20,
'local_gb_used': 0,
'free_disk_gb': 20,
'vcpus': 2,
'vcpus_used': 0
}
return objects.ComputeNode(**resources)
class ShelveComputeManagerTestCase(test_compute.BaseTestCase):
@mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_terminate_volume_connections')
@mock.patch.object(nova.virt.fake.SmallFakeDriver, 'power_off')
@mock.patch.object(nova.virt.fake.SmallFakeDriver, 'snapshot')
@mock.patch.object(nova.compute.manager.ComputeManager, '_get_power_state')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_notify_about_instance_usage')
@mock.patch('nova.compute.utils.notify_about_instance_action')
def _shelve_instance(self, shelved_offload_time, mock_notify,
mock_notify_instance_usage, mock_get_power_state,
mock_snapshot, mock_power_off, mock_terminate,
mock_get_bdms, clean_shutdown=True,
guest_power_state=power_state.RUNNING,
accel_uuids=None):
mock_get_power_state.return_value = 123
CONF.set_override('shelved_offload_time', shelved_offload_time)
host = 'fake-mini'
instance = self._create_fake_instance_obj(
params={'host': host, 'power_state': guest_power_state})
image_id = 'fake_image_id'
host = 'fake-mini'
self.useFixture(utils_fixture.TimeFixture())
instance.task_state = task_states.SHELVING
instance.save()
fake_bdms = None
if shelved_offload_time == 0:
fake_bdms = objects.BlockDeviceMappingList()
mock_get_bdms.return_value = fake_bdms
tracking = {'last_state': instance.vm_state}
def check_save(expected_task_state=None):
self.assertEqual(123, instance.power_state)
if tracking['last_state'] == vm_states.ACTIVE:
if CONF.shelved_offload_time == 0:
self.assertEqual(task_states.SHELVING_OFFLOADING,
instance.task_state)
else:
self.assertIsNone(instance.task_state)
self.assertEqual(vm_states.SHELVED, instance.vm_state)
self.assertEqual([task_states.SHELVING,
task_states.SHELVING_IMAGE_UPLOADING],
expected_task_state)
self.assertIn('shelved_at', instance.system_metadata)
self.assertEqual(image_id,
instance.system_metadata['shelved_image_id'])
self.assertEqual(host,
instance.system_metadata['shelved_host'])
tracking['last_state'] = instance.vm_state
elif (tracking['last_state'] == vm_states.SHELVED and
CONF.shelved_offload_time == 0):
self.assertIsNone(instance.task_state)
self.assertEqual(vm_states.SHELVED_OFFLOADED,
instance.vm_state)
self.assertEqual([task_states.SHELVING,
task_states.SHELVING_OFFLOADING],
expected_task_state)
tracking['last_state'] = instance.vm_state
elif (tracking['last_state'] == vm_states.SHELVED_OFFLOADED and
CONF.shelved_offload_time == 0):
self.assertIsNone(instance.host)
self.assertIsNone(instance.node)
self.assertIsNone(expected_task_state)
else:
self.fail('Unexpected save!')
with test.nested(
mock.patch.object(instance, 'save'),
mock.patch.object(self.compute.network_api,
'cleanup_instance_network_on_host')) as (
mock_save, mock_cleanup
):
mock_save.side_effect = check_save
self.compute.shelve_instance(self.context, instance,
image_id=image_id,
clean_shutdown=clean_shutdown,
accel_uuids=accel_uuids)
mock_notify.assert_has_calls([
mock.call(self.context, instance, 'fake-mini',
action='shelve', phase='start', bdms=fake_bdms),
mock.call(self.context, instance, 'fake-mini',
action='shelve', phase='end', bdms=fake_bdms)])
# prepare expect call lists
mock_notify_instance_usage_call_list = [
mock.call(self.context, instance, 'shelve.start'),
mock.call(self.context, instance, 'shelve.end')]
mock_power_off_call_list = []
mock_get_power_state_call_list = [mock.call(instance)]
if clean_shutdown:
if guest_power_state == power_state.PAUSED:
mock_power_off_call_list.append(mock.call(instance, 0, 0))
else:
mock_power_off_call_list.append(
mock.call(instance, CONF.shutdown_timeout,
CONF.compute.shutdown_retry_interval))
else:
mock_power_off_call_list.append(mock.call(instance, 0, 0))
if CONF.shelved_offload_time == 0:
mock_notify_instance_usage_call_list.extend([
mock.call(self.context, instance, 'shelve_offload.start'),
mock.call(self.context, instance, 'shelve_offload.end')])
mock_power_off_call_list.append(mock.call(instance, 0, 0))
mock_get_power_state_call_list.append(mock.call(instance))
mock_notify_instance_usage.assert_has_calls(
mock_notify_instance_usage_call_list)
mock_power_off.assert_has_calls(mock_power_off_call_list)
mock_cleanup.assert_not_called()
mock_snapshot.assert_called_once_with(self.context, instance,
'fake_image_id', mock.ANY)
mock_get_power_state.assert_has_calls(mock_get_power_state_call_list)
if CONF.shelved_offload_time == 0:
self.assertTrue(mock_terminate.called)
def test_shelve(self):
self._shelve_instance(-1)
def test_shelve_forced_shutdown(self):
self._shelve_instance(-1, clean_shutdown=False)
def test_shelve_and_offload(self):
self._shelve_instance(0)
def test_shelve_paused_instance(self):
self._shelve_instance(-1, guest_power_state=power_state.PAUSED)
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'delete_arqs_for_instance')
def test_shelve_and_offload_with_accel_uuids(self, mock_del_arqs):
self._shelve_instance(0, accel_uuids=[uuids.fake])
mock_del_arqs.assert_called_once()
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'delete_arqs_for_instance')
def test_shelve_and_with_accel_uuids(self, mock_del_arqs):
self._shelve_instance(-1, accel_uuids=[uuids.fake])
mock_del_arqs.assert_not_called()
@mock.patch.object(nova.virt.fake.SmallFakeDriver, 'power_off')
def test_shelve_offload(self, mock_power_off):
instance = self._shelve_offload()
mock_power_off.assert_called_once_with(instance,
CONF.shutdown_timeout, CONF.compute.shutdown_retry_interval)
@mock.patch.object(nova.virt.fake.SmallFakeDriver, 'power_off')
def test_shelve_offload_forced_shutdown(self, mock_power_off):
instance = self._shelve_offload(clean_shutdown=False)
mock_power_off.assert_called_once_with(instance, 0, 0)
@mock.patch.object(compute_utils, 'EventReporter')
@mock.patch.object(objects.BlockDeviceMappingList, 'get_by_instance_uuid')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_terminate_volume_connections')
@mock.patch('nova.compute.resource_tracker.ResourceTracker.'
'delete_allocation_for_shelve_offloaded_instance')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_update_resource_tracker')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_get_power_state', return_value=123)
@mock.patch.object(nova.compute.manager.ComputeManager,
'_notify_about_instance_usage')
@mock.patch('nova.compute.utils.notify_about_instance_action')
def _shelve_offload(self, mock_notify, mock_notify_instance_usage,
mock_get_power_state, mock_update_resource_tracker,
mock_delete_alloc, mock_terminate, mock_get_bdms,
mock_event, clean_shutdown=True):
host = 'fake-mini'
instance = self._create_fake_instance_obj(params={'host': host})
instance.task_state = task_states.SHELVING
instance.save()
self.useFixture(utils_fixture.TimeFixture())
fake_bdms = objects.BlockDeviceMappingList()
mock_get_bdms.return_value = fake_bdms
def stub_instance_save(inst, *args, **kwargs):
# If the vm_state is changed to SHELVED_OFFLOADED make sure we
# have already freed up allocations in placement.
if inst.vm_state == vm_states.SHELVED_OFFLOADED:
self.assertTrue(mock_delete_alloc.called,
'Allocations must be deleted before the '
'vm_status can change to shelved_offloaded.')
self.stub_out('nova.objects.Instance.save', stub_instance_save)
self.compute.shelve_offload_instance(self.context, instance,
clean_shutdown=clean_shutdown,
accel_uuids=[])
mock_notify.assert_has_calls([
mock.call(self.context, instance, 'fake-mini',
action='shelve_offload', phase='start',
bdms=fake_bdms),
mock.call(self.context, instance, 'fake-mini',
action='shelve_offload', phase='end',
bdms=fake_bdms)])
self.assertEqual(vm_states.SHELVED_OFFLOADED, instance.vm_state)
self.assertIsNone(instance.task_state)
self.assertTrue(mock_terminate.called)
# prepare expect call lists
mock_notify_instance_usage_call_list = [
mock.call(self.context, instance, 'shelve_offload.start'),
mock.call(self.context, instance, 'shelve_offload.end')]
mock_notify_instance_usage.assert_has_calls(
mock_notify_instance_usage_call_list)
# instance.host is replaced with host because
# original instance.host is clear after
# ComputeManager.shelve_offload_instance execute
mock_get_power_state.assert_called_once_with(instance)
mock_update_resource_tracker.assert_called_once_with(self.context,
instance)
mock_delete_alloc.assert_called_once_with(self.context, instance)
mock_event.assert_called_once_with(self.context,
'compute_shelve_offload_instance',
CONF.host,
instance.uuid,
graceful_exit=False)
return instance
@mock.patch('nova.compute.utils.'
'update_pci_request_spec_with_allocated_interface_name',
new=mock.NonCallableMock())
@mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid')
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_notify_about_instance_usage')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_prep_block_device', return_value='fake_bdm')
@mock.patch.object(nova.virt.fake.SmallFakeDriver, 'spawn')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_get_power_state', return_value=123)
@mock.patch.object(neutron_api.API, 'setup_instance_network_on_host')
def test_unshelve(self, mock_setup_network,
mock_get_power_state, mock_spawn,
mock_prep_block_device, mock_notify_instance_usage,
mock_notify_instance_action,
mock_get_bdms,
accel_uuids=None,
instance=None):
mock_bdms = mock.Mock()
mock_get_bdms.return_value = mock_bdms
instance = instance or self._create_fake_instance_obj()
instance.task_state = task_states.UNSHELVING
instance.save()
image = {'id': uuids.image_id}
node = test_compute.NODENAME
limits = {}
filter_properties = {'limits': limits}
host = 'fake-mini'
cur_time = timeutils.utcnow()
# Adding shelved_* keys in system metadata to verify
# whether those are deleted after unshelve call.
sys_meta = dict(instance.system_metadata)
sys_meta['shelved_at'] = cur_time.isoformat()
sys_meta['shelved_image_id'] = image['id']
sys_meta['shelved_host'] = host
instance.system_metadata = sys_meta
self.deleted_image_id = None
def fake_delete(self2, ctxt, image_id):
self.deleted_image_id = image_id
def fake_claim(context, instance, node, allocations, limits):
instance.host = self.compute.host
requests = objects.InstancePCIRequests(requests=[])
return claims.Claim(context, instance, test_compute.NODENAME,
self.rt, _fake_resources(),
requests)
tracking = {
'last_state': instance.task_state,
'spawned': False,
}
def check_save(expected_task_state=None):
if tracking['last_state'] == task_states.UNSHELVING:
if tracking['spawned']:
self.assertIsNone(instance.task_state)
else:
self.assertEqual(task_states.SPAWNING, instance.task_state)
tracking['spawned'] = True
tracking['last_state'] == instance.task_state
elif tracking['last_state'] == task_states.SPAWNING:
self.assertEqual(vm_states.ACTIVE, instance.vm_state)
tracking['last_state'] == instance.task_state
else:
self.fail('Unexpected save!')
self.useFixture(fixtures.GlanceFixture(self))
self.stub_out('nova.tests.fixtures.GlanceFixture.delete', fake_delete)
with mock.patch.object(self.rt, 'instance_claim',
side_effect=fake_claim), \
mock.patch.object(instance, 'save') as mock_save:
mock_save.side_effect = check_save
self.compute.unshelve_instance(
self.context, instance, image=image,
filter_properties=filter_properties,
node=node, request_spec=objects.RequestSpec(),
accel_uuids=accel_uuids)
mock_notify_instance_action.assert_has_calls([
mock.call(self.context, instance, 'fake-mini',
action='unshelve', phase='start', bdms=mock_bdms),
mock.call(self.context, instance, 'fake-mini',
action='unshelve', phase='end', bdms=mock_bdms)])
# prepare expect call lists
mock_notify_instance_usage_call_list = [
mock.call(self.context, instance, 'unshelve.start'),
mock.call(self.context, instance, 'unshelve.end')]
mock_notify_instance_usage.assert_has_calls(
mock_notify_instance_usage_call_list)
mock_prep_block_device.assert_called_once_with(self.context,
instance, mock.ANY)
mock_setup_network.assert_called_once_with(
self.context, instance, self.compute.host, provider_mappings=None)
mock_spawn.assert_called_once_with(self.context, instance,
test.MatchType(objects.ImageMeta), injected_files=[],
admin_password=None, allocations={}, network_info=[],
block_device_info='fake_bdm', accel_info=mock.ANY)
self.mock_get_allocations.assert_called_once_with(self.context,
instance.uuid)
mock_get_power_state.assert_called_once_with(instance)
self.assertNotIn('shelved_at', instance.system_metadata)
self.assertNotIn('shelved_image_id', instance.system_metadata)
self.assertNotIn('shelved_host', instance.system_metadata)
self.assertEqual(image['id'], self.deleted_image_id)
self.assertEqual(instance.host, self.compute.host)
self.assertEqual(123, instance.power_state)
self.assertEqual(vm_states.ACTIVE, instance.vm_state)
self.assertIsNone(instance.task_state)
self.assertIsNone(instance.key_data)
self.assertEqual(self.compute.host, instance.host)
self.assertFalse(instance.auto_disk_config)
@mock.patch.object(nova.compute.manager.ComputeManager,
'_get_bound_arq_resources')
def test_unshelve_with_arqs(self, mock_get_arqs):
self.test_unshelve(accel_uuids=[uuids.fake])
mock_get_arqs.assert_called_once()
@mock.patch.object(nova.compute.manager.ComputeManager,
'_nil_out_instance_obj_host_and_node')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_terminate_volume_connections')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_build_resources_cleanup')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_get_bound_arq_resources')
def test_unshelve_with_arqs_failes_cleanup_resources(
self, mock_get_arqs, mock_cleanup_resources,
mock_bdms, mock_nil):
instance = self._create_fake_instance_obj()
mock_get_arqs.side_effect = exception.AcceleratorRequestOpFailed(
instance_uuid=instance.uuid, op='get',
msg='Failure getting accelerator requests.')
exc = self.assertRaises(exception.AcceleratorRequestOpFailed,
self.test_unshelve,
accel_uuids=[uuids.fake],
instance=instance)
self.assertIn('Failure getting accelerator requests.', str(exc))
mock_cleanup_resources.assert_called_once()
@mock.patch.object(nova.compute.manager.ComputeManager,
'_nil_out_instance_obj_host_and_node')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_terminate_volume_connections')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_build_resources_cleanup')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_get_bound_arq_resources')
def test_unshelve_with_arqs_failes_cleanup_resources_timeout(
self, mock_get_arqs, mock_cleanup_resources,
mock_bdms, mock_nil):
instance = self._create_fake_instance_obj()
mock_get_arqs.side_effect = eventlet.timeout.Timeout()
self.assertRaises(eventlet.timeout.Timeout,
self.test_unshelve,
accel_uuids=[uuids.fake],
instance=instance)
mock_cleanup_resources.assert_called_once()
@mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid')
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch.object(nova.compute.resource_tracker.ResourceTracker,
'instance_claim')
@mock.patch.object(neutron_api.API, 'setup_instance_network_on_host')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_get_power_state', return_value=123)
@mock.patch.object(nova.virt.fake.SmallFakeDriver, 'spawn')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_prep_block_device', return_value='fake_bdm')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_notify_about_instance_usage')
@mock.patch('nova.utils.get_image_from_system_metadata')
def test_unshelve_volume_backed(self, mock_image_meta,
mock_notify_instance_usage,
mock_prep_block_device, mock_spawn,
mock_get_power_state,
mock_setup_network, mock_instance_claim,
mock_notify_instance_action,
mock_get_bdms):
mock_bdms = mock.Mock()
mock_get_bdms.return_value = mock_bdms
instance = self._create_fake_instance_obj()
node = test_compute.NODENAME
limits = {}
filter_properties = {'limits': limits}
instance.task_state = task_states.UNSHELVING
instance.save()
image_meta = {'properties': {'base_image_ref': uuids.image_id}}
mock_image_meta.return_value = image_meta
tracking = {'last_state': instance.task_state}
def fake_claim(context, instance, node, allocations, limits):
instance.host = self.compute.host
requests = objects.InstancePCIRequests(requests=[])
return claims.Claim(context, instance, test_compute.NODENAME,
self.rt, _fake_resources(),
requests)
mock_instance_claim.side_effect = fake_claim
def check_save(expected_task_state=None):
if tracking['last_state'] == task_states.UNSHELVING:
self.assertEqual(task_states.SPAWNING, instance.task_state)
tracking['last_state'] = instance.task_state
elif tracking['last_state'] == task_states.SPAWNING:
self.assertEqual(123, instance.power_state)
self.assertEqual(vm_states.ACTIVE, instance.vm_state)
self.assertIsNone(instance.task_state)
self.assertIsNone(instance.key_data)
self.assertFalse(instance.auto_disk_config)
self.assertIsNone(instance.task_state)
tracking['last_state'] = instance.task_state
else:
self.fail('Unexpected save!')
with mock.patch.object(instance, 'save') as mock_save:
mock_save.side_effect = check_save
self.compute.unshelve_instance(self.context, instance, image=None,
filter_properties=filter_properties, node=node,
request_spec=objects.RequestSpec(), accel_uuids=[])
mock_notify_instance_action.assert_has_calls([
mock.call(self.context, instance, 'fake-mini',
action='unshelve', phase='start', bdms=mock_bdms),
mock.call(self.context, instance, 'fake-mini',
action='unshelve', phase='end', bdms=mock_bdms)])
# prepare expect call lists
mock_notify_instance_usage_call_list = [
mock.call(self.context, instance, 'unshelve.start'),
mock.call(self.context, instance, 'unshelve.end')]
mock_notify_instance_usage.assert_has_calls(
mock_notify_instance_usage_call_list)
mock_prep_block_device.assert_called_once_with(self.context, instance,
mock.ANY)
mock_setup_network.assert_called_once_with(
self.context, instance, self.compute.host, provider_mappings=None)
mock_instance_claim.assert_called_once_with(self.context, instance,
test_compute.NODENAME,
{}, limits)
mock_spawn.assert_called_once_with(self.context, instance,
test.MatchType(objects.ImageMeta),
injected_files=[], admin_password=None, accel_info=[],
allocations={}, network_info=[], block_device_info='fake_bdm')
self.mock_get_allocations.assert_called_once_with(self.context,
instance.uuid)
mock_get_power_state.assert_called_once_with(instance)
@mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid')
@mock.patch('nova.compute.utils.notify_about_instance_action')
@mock.patch.object(nova.compute.resource_tracker.ResourceTracker,
'instance_claim')
@mock.patch.object(neutron_api.API, 'setup_instance_network_on_host')
@mock.patch.object(nova.virt.fake.SmallFakeDriver, 'spawn',
side_effect=test.TestingException('oops!'))
@mock.patch.object(nova.compute.manager.ComputeManager,
'_prep_block_device', return_value='fake_bdm')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_notify_about_instance_usage')
@mock.patch('nova.utils.get_image_from_system_metadata')
@mock.patch.object(nova.compute.manager.ComputeManager,
'_terminate_volume_connections')
def test_unshelve_spawn_fails_cleanup_volume_connections(
self, mock_terminate_volume_connections, mock_image_meta,
mock_notify_instance_usage, mock_prep_block_device, mock_spawn,
mock_setup_network, mock_instance_claim,
mock_notify_instance_action, mock_get_bdms):
"""Tests error handling when a instance fails to unshelve and makes
sure that volume connections are cleaned up from the host
and that the host/node values are unset on the instance.
"""
mock_bdms = mock.Mock()
mock_get_bdms.return_value = mock_bdms
instance = self._create_fake_instance_obj()
node = test_compute.NODENAME
limits = {}
filter_properties = {'limits': limits}
instance.task_state = task_states.UNSHELVING
instance.save()
image_meta = {'properties': {'base_image_ref': uuids.image_id}}
mock_image_meta.return_value = image_meta
tracking = {'last_state': instance.task_state}
def fake_claim(context, instance, node, allocations, limits):
instance.host = self.compute.host
instance.node = node
requests = objects.InstancePCIRequests(requests=[])
return claims.Claim(context, instance, node,
self.rt, _fake_resources(),
requests, limits=limits)
mock_instance_claim.side_effect = fake_claim
def check_save(expected_task_state=None):
if tracking['last_state'] == task_states.UNSHELVING:
# This is before we've failed.
self.assertEqual(task_states.SPAWNING, instance.task_state)
tracking['last_state'] = instance.task_state
elif tracking['last_state'] == task_states.SPAWNING:
# This is after we've failed.
self.assertIsNone(instance.host)
self.assertIsNone(instance.node)
self.assertIsNone(instance.task_state)
tracking['last_state'] = instance.task_state
else:
self.fail('Unexpected save!')
with mock.patch.object(instance, 'save') as mock_save:
mock_save.side_effect = check_save
self.assertRaises(test.TestingException,
self.compute.unshelve_instance,
self.context, instance, image=None,
filter_properties=filter_properties, node=node,
request_spec=objects.RequestSpec(),
accel_uuids=[])
mock_notify_instance_action.assert_called_once_with(
self.context, instance, 'fake-mini', action='unshelve',
phase='start', bdms=mock_bdms)
mock_notify_instance_usage.assert_called_once_with(
self.context, instance, 'unshelve.start')
mock_prep_block_device.assert_called_once_with(
self.context, instance, mock_bdms)
mock_setup_network.assert_called_once_with(
self.context, instance, self.compute.host, provider_mappings=None)
mock_instance_claim.assert_called_once_with(self.context, instance,
test_compute.NODENAME,
{}, limits)
mock_spawn.assert_called_once_with(
self.context, instance, test.MatchType(objects.ImageMeta),
injected_files=[], admin_password=None, accel_info=[],
allocations={}, network_info=[], block_device_info='fake_bdm')
mock_terminate_volume_connections.assert_called_once_with(
self.context, instance, mock_bdms)
@mock.patch('nova.network.neutron.API.setup_instance_network_on_host')
@mock.patch('nova.compute.utils.'
'update_pci_request_spec_with_allocated_interface_name')
def test_unshelve_with_resource_request(
self, mock_update_pci, mock_setup_network):
requested_res = [objects.RequestGroup(
requester_id=uuids.port_1,
provider_uuids=[uuids.rp1])]
request_spec = objects.RequestSpec(requested_resources=requested_res)
instance = self._create_fake_instance_obj()
self.compute.unshelve_instance(
self.context, instance, image=None,
filter_properties={}, node='fake-node', request_spec=request_spec,
accel_uuids=[])
mock_update_pci.assert_called_once_with(
self.context, self.compute.reportclient, [],
{uuids.port_1: [uuids.rp1]})
mock_setup_network.assert_called_once_with(
self.context, instance, self.compute.host,
provider_mappings={uuids.port_1: [uuids.rp1]})
@mock.patch('nova.network.neutron.API.setup_instance_network_on_host',
new=mock.NonCallableMock())
@mock.patch('nova.compute.utils.'
'update_pci_request_spec_with_allocated_interface_name')
def test_unshelve_with_resource_request_update_raises(
self, mock_update_pci):
requested_res = [objects.RequestGroup(
requester_id=uuids.port_1,
provider_uuids=[uuids.rp1])]
request_spec = objects.RequestSpec(requested_resources=requested_res)
instance = self._create_fake_instance_obj()
mock_update_pci.side_effect = (
exception.UnexpectedResourceProviderNameForPCIRequest(
provider=uuids.rp1,
requester=uuids.port1,
provider_name='unexpected'))
self.assertRaises(
exception.UnexpectedResourceProviderNameForPCIRequest,
self.compute.unshelve_instance, self.context, instance, image=None,
filter_properties={}, node='fake-node', request_spec=request_spec,
accel_uuids=[])
mock_update_pci.assert_called_once_with(
self.context, self.compute.reportclient, [],
{uuids.port_1: [uuids.rp1]})
@mock.patch.object(objects.InstanceList, 'get_by_filters')
def test_shelved_poll_none_offloaded(self, mock_get_by_filters):
# Test instances are not offloaded when shelved_offload_time is -1
self.flags(shelved_offload_time=-1)
self.compute._poll_shelved_instances(self.context)
self.assertEqual(0, mock_get_by_filters.call_count)
@mock.patch('oslo_utils.timeutils.is_older_than')
def test_shelved_poll_none_exist(self, mock_older):
self.flags(shelved_offload_time=1)
mock_older.return_value = False
with mock.patch.object(self.compute, 'shelve_offload_instance') as soi:
self.compute._poll_shelved_instances(self.context)
self.assertFalse(soi.called)
@mock.patch('oslo_utils.timeutils.is_older_than')
def test_shelved_poll_not_timedout(self, mock_older):
mock_older.return_value = False
self.flags(shelved_offload_time=1)
shelved_time = timeutils.utcnow()
time_fixture = self.useFixture(utils_fixture.TimeFixture(shelved_time))
time_fixture.advance_time_seconds(CONF.shelved_offload_time - 1)
instance = self._create_fake_instance_obj()
instance.vm_state = vm_states.SHELVED
instance.task_state = None
instance.host = self.compute.host
sys_meta = instance.system_metadata
sys_meta['shelved_at'] = shelved_time.isoformat()
instance.save()
with mock.patch.object(self.compute, 'shelve_offload_instance') as soi:
self.compute._poll_shelved_instances(self.context)
self.assertFalse(soi.called)
self.assertTrue(mock_older.called)
def test_shelved_poll_timedout(self):
self.flags(shelved_offload_time=1)
shelved_time = timeutils.utcnow()
time_fixture = self.useFixture(utils_fixture.TimeFixture(shelved_time))
time_fixture.advance_time_seconds(CONF.shelved_offload_time + 1)
instance = self._create_fake_instance_obj()
instance.vm_state = vm_states.SHELVED
instance.task_state = None
instance.host = self.compute.host
sys_meta = instance.system_metadata
sys_meta['shelved_at'] = shelved_time.isoformat()
instance.save()
data = []
def fake_soi(context, instance, accel_uuids, **kwargs):
data.append(instance.uuid)
with mock.patch.object(self.compute, 'shelve_offload_instance') as soi:
soi.side_effect = fake_soi
self.compute._poll_shelved_instances(self.context)
self.assertTrue(soi.called)
self.assertEqual(instance.uuid, data[0])
@mock.patch('oslo_utils.timeutils.is_older_than')
@mock.patch('oslo_utils.timeutils.parse_strtime')
def test_shelved_poll_filters_task_state(self, mock_parse, mock_older):
self.flags(shelved_offload_time=1)
mock_older.return_value = True
instance1 = self._create_fake_instance_obj()
instance1.task_state = task_states.SPAWNING
instance1.vm_state = vm_states.SHELVED
instance1.host = self.compute.host
instance1.system_metadata = {'shelved_at': ''}
instance1.save()
instance2 = self._create_fake_instance_obj()
instance2.task_state = None
instance2.vm_state = vm_states.SHELVED
instance2.host = self.compute.host
instance2.system_metadata = {'shelved_at': ''}
instance2.save()
data = []
def fake_soi(context, instance, accel_uuids, **kwargs):
data.append(instance.uuid)
with mock.patch.object(self.compute, 'shelve_offload_instance') as soi:
soi.side_effect = fake_soi
self.compute._poll_shelved_instances(self.context)
self.assertTrue(soi.called)
self.assertEqual([instance2.uuid], data)
@mock.patch('nova.accelerator.cyborg._CyborgClient.'
'get_arq_uuids_for_instance')
@mock.patch('oslo_utils.timeutils.is_older_than')
@mock.patch('oslo_utils.timeutils.parse_strtime')
def test_shelved_poll_with_accel_uuids(self, mock_parse, mock_older,
mock_get_arq_uuids):
self.flags(shelved_offload_time=1)
mock_older.return_value = True
instance = self._create_fake_instance_obj()
instance.task_state = None
instance.vm_state = vm_states.SHELVED
instance.host = self.compute.host
instance.system_metadata = {'shelved_at': ''}
instance.flavor.extra_specs['accel:device_profile'] = 'dp_test'
mock_get_arq_uuids.return_value = [uuids.fake]
instance.save()
data = []
def fake_soi(context, instance, **kwargs):
data.append(instance.uuid)
with mock.patch.object(self.compute, 'shelve_offload_instance') as soi:
soi.side_effect = fake_soi
self.compute._poll_shelved_instances(self.context)
self.assertTrue(soi.called)
self.assertEqual([instance.uuid], data)
@mock.patch('oslo_utils.timeutils.is_older_than')
@mock.patch('oslo_utils.timeutils.parse_strtime')
def test_shelved_poll_checks_task_state_on_save(self, mock_parse,
mock_older):
self.flags(shelved_offload_time=1)
mock_older.return_value = True
instance = self._create_fake_instance_obj()
instance.task_state = None
instance.vm_state = vm_states.SHELVED
instance.host = self.compute.host
instance.system_metadata = {'shelved_at': ''}
instance.save()
def fake_parse_hook(timestring):
instance.task_state = task_states.SPAWNING
instance.save()
mock_parse.side_effect = fake_parse_hook
with mock.patch.object(self.compute, 'shelve_offload_instance') as soi:
self.compute._poll_shelved_instances(self.context)
self.assertFalse(soi.called)
class ShelveComputeAPITestCase(test_compute.BaseTestCase):
def _get_vm_states(self, exclude_states=None):
vm_state = set([vm_states.ACTIVE, vm_states.BUILDING, vm_states.PAUSED,
vm_states.SUSPENDED, vm_states.RESCUED, vm_states.STOPPED,
vm_states.RESIZED, vm_states.SOFT_DELETED,
vm_states.DELETED, vm_states.ERROR, vm_states.SHELVED,
vm_states.SHELVED_OFFLOADED])
if not exclude_states:
exclude_states = set()
return vm_state - exclude_states
def _test_shelve(self, vm_state=vm_states.ACTIVE, boot_from_volume=False,
clean_shutdown=True):
# Ensure instance can be shelved.
params = dict(task_state=None, vm_state=vm_state, display_name='vm01')
fake_instance = self._create_fake_instance_obj(params=params)
instance = fake_instance
self.assertIsNone(instance['task_state'])
with test.nested(
mock.patch.object(compute_utils, 'is_volume_backed_instance',
return_value=boot_from_volume),
mock.patch.object(compute_utils, 'create_image',
return_value=dict(id='fake-image-id')),
mock.patch.object(instance, 'save'),
mock.patch.object(self.compute_api, '_record_action_start'),
mock.patch.object(self.compute_api.compute_rpcapi,
'shelve_instance'),
mock.patch.object(self.compute_api.compute_rpcapi,
'shelve_offload_instance')
) as (
volume_backed_inst, create_image, instance_save,
record_action_start, rpcapi_shelve_instance,
rpcapi_shelve_offload_instance
):
self.compute_api.shelve(self.context, instance,
clean_shutdown=clean_shutdown)
self.assertEqual(instance.task_state, task_states.SHELVING)
# assert our mock calls
volume_backed_inst.assert_called_once_with(
self.context, instance)
instance_save.assert_called_once_with(expected_task_state=[None])
record_action_start.assert_called_once_with(
self.context, instance, instance_actions.SHELVE)
if boot_from_volume:
rpcapi_shelve_offload_instance.assert_called_once_with(
self.context, instance=instance,
clean_shutdown=clean_shutdown, accel_uuids=[])
else:
rpcapi_shelve_instance.assert_called_once_with(
self.context, instance=instance, image_id='fake-image-id',
clean_shutdown=clean_shutdown, accel_uuids=[])
db.instance_destroy(self.context, instance['uuid'])
def test_shelve(self):
self._test_shelve()
def test_shelves_stopped(self):
self._test_shelve(vm_state=vm_states.STOPPED)
def test_shelves_paused(self):
self._test_shelve(vm_state=vm_states.PAUSED)
def test_shelves_suspended(self):
self._test_shelve(vm_state=vm_states.SUSPENDED)
def test_shelves_boot_from_volume(self):
self._test_shelve(boot_from_volume=True)
def test_shelve_forced_shutdown(self):
self._test_shelve(clean_shutdown=False)
def test_shelve_boot_from_volume_forced_shutdown(self):
self._test_shelve(boot_from_volume=True,
clean_shutdown=False)
def _test_shelve_invalid_state(self, vm_state):
params = dict(vm_state=vm_state)
fake_instance = self._create_fake_instance_obj(params=params)
self.assertRaises(exception.InstanceInvalidState,
self.compute_api.shelve,
self.context, fake_instance)
def test_shelve_fails_invalid_states(self):
invalid_vm_states = self._get_vm_states(set([vm_states.ACTIVE,
vm_states.STOPPED,
vm_states.PAUSED,
vm_states.SUSPENDED]))
for state in invalid_vm_states:
self._test_shelve_invalid_state(state)
def _test_shelve_offload(self, clean_shutdown=True):
params = dict(task_state=None, vm_state=vm_states.SHELVED)
fake_instance = self._create_fake_instance_obj(params=params)
with test.nested(
mock.patch.object(fake_instance, 'save'),
mock.patch.object(self.compute_api.compute_rpcapi,
'shelve_offload_instance'),
mock.patch('nova.compute.api.API._record_action_start')
) as (
instance_save, rpcapi_shelve_offload_instance, record
):
self.compute_api.shelve_offload(self.context, fake_instance,
clean_shutdown=clean_shutdown)
# assert field values set on the instance object
self.assertEqual(task_states.SHELVING_OFFLOADING,
fake_instance.task_state)
instance_save.assert_called_once_with(expected_task_state=[None])
rpcapi_shelve_offload_instance.assert_called_once_with(
self.context, instance=fake_instance,
clean_shutdown=clean_shutdown, accel_uuids=[])
record.assert_called_once_with(self.context, fake_instance,
instance_actions.SHELVE_OFFLOAD)
def test_shelve_offload(self):
self._test_shelve_offload()
def test_shelve_offload_forced_shutdown(self):
self._test_shelve_offload(clean_shutdown=False)
def _test_shelve_offload_invalid_state(self, vm_state):
params = dict(vm_state=vm_state)
fake_instance = self._create_fake_instance_obj(params=params)
self.assertRaises(exception.InstanceInvalidState,
self.compute_api.shelve_offload,
self.context, fake_instance)
def test_shelve_offload_fails_invalid_states(self):
invalid_vm_states = self._get_vm_states(set([vm_states.SHELVED]))
for state in invalid_vm_states:
self._test_shelve_offload_invalid_state(state)
def _get_specify_state_instance(self, vm_state):
# Ensure instance can be unshelved.
instance = self._create_fake_instance_obj()
self.assertIsNone(instance['task_state'])
self.compute_api.shelve(self.context, instance)
instance.task_state = None
instance.vm_state = vm_state
instance.save()
return instance
@mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid')
def test_unshelve(self, get_by_instance_uuid):
# Ensure instance can be unshelved.
instance = self._get_specify_state_instance(vm_states.SHELVED)
fake_spec = objects.RequestSpec()
get_by_instance_uuid.return_value = fake_spec
with mock.patch.object(self.compute_api.compute_task_api,
'unshelve_instance') as unshelve:
self.compute_api.unshelve(self.context, instance)
get_by_instance_uuid.assert_called_once_with(self.context,
instance.uuid)
unshelve.assert_called_once_with(self.context, instance, fake_spec)
self.assertEqual(instance.task_state, task_states.UNSHELVING)
db.instance_destroy(self.context, instance['uuid'])
@mock.patch('nova.availability_zones.get_availability_zones',
return_value=['az1', 'az2'])
@mock.patch.object(objects.RequestSpec, 'save')
@mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid')
def test_specified_az_ushelve_invalid_request(self,
get_by_instance_uuid,
mock_save,
mock_availability_zones):
# Ensure instance can be unshelved.
instance = self._get_specify_state_instance(
vm_states.SHELVED_OFFLOADED)
new_az = "fake-new-az"
fake_spec = objects.RequestSpec()
fake_spec.availability_zone = "fake-old-az"
get_by_instance_uuid.return_value = fake_spec
exc = self.assertRaises(exception.InvalidRequest,
self.compute_api.unshelve,
self.context, instance, new_az=new_az)
self.assertEqual("The requested availability zone is not available",
exc.format_message())
@mock.patch('nova.availability_zones.get_availability_zones',
return_value=['az1', 'az2'])
@mock.patch.object(objects.RequestSpec, 'save')
@mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid')
def test_specified_az_unshelve_invalid_state(self, get_by_instance_uuid,
mock_save,
mock_availability_zones):
# Ensure instance can be unshelved.
instance = self._get_specify_state_instance(vm_states.SHELVED)
new_az = "az1"
fake_spec = objects.RequestSpec()
fake_spec.availability_zone = "fake-old-az"
get_by_instance_uuid.return_value = fake_spec
self.assertRaises(exception.UnshelveInstanceInvalidState,
self.compute_api.unshelve,
self.context, instance, new_az=new_az)
@mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid',
new_callable=mock.NonCallableMock)
@mock.patch('nova.availability_zones.get_availability_zones')
def test_validate_unshelve_az_cross_az_attach_true(
self, mock_get_azs, mock_get_bdms):
"""Tests a case where the new AZ to unshelve does not match the volume
attached to the server but cross_az_attach=True so it's not an error.
"""
# Ensure instance can be unshelved.
instance = self._create_fake_instance_obj(
params=dict(vm_state=vm_states.SHELVED_OFFLOADED))
new_az = "west_az"
mock_get_azs.return_value = ["west_az", "east_az"]
self.flags(cross_az_attach=True, group='cinder')
self.compute_api._validate_unshelve_az(self.context, instance, new_az)
mock_get_azs.assert_called_once_with(
self.context, self.compute_api.host_api, get_only_available=True)
@mock.patch('nova.volume.cinder.API.get')
@mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid')
@mock.patch('nova.availability_zones.get_availability_zones')
def test_validate_unshelve_az_cross_az_attach_false(
self, mock_get_azs, mock_get_bdms, mock_get):
"""Tests a case where the new AZ to unshelve does not match the volume
attached to the server and cross_az_attach=False so it's an error.
"""
# Ensure instance can be unshelved.
instance = self._create_fake_instance_obj(
params=dict(vm_state=vm_states.SHELVED_OFFLOADED))
new_az = "west_az"
mock_get_azs.return_value = ["west_az", "east_az"]
bdms = [objects.BlockDeviceMapping(destination_type='volume',
volume_id=uuids.volume_id)]
mock_get_bdms.return_value = bdms
volume = {'id': uuids.volume_id, 'availability_zone': 'east_az'}
mock_get.return_value = volume
self.flags(cross_az_attach=False, group='cinder')
self.assertRaises(exception.MismatchVolumeAZException,
self.compute_api._validate_unshelve_az,
self.context, instance, new_az)
mock_get_azs.assert_called_once_with(
self.context, self.compute_api.host_api, get_only_available=True)
mock_get_bdms.assert_called_once_with(self.context, instance.uuid)
mock_get.assert_called_once_with(self.context, uuids.volume_id)
@mock.patch.object(compute_api.API, '_validate_unshelve_az')
@mock.patch.object(objects.RequestSpec, 'save')
@mock.patch.object(objects.RequestSpec, 'get_by_instance_uuid')
def test_specified_az_unshelve(self, get_by_instance_uuid,
mock_save, mock_validate_unshelve_az):
# Ensure instance can be unshelved.
instance = self._get_specify_state_instance(
vm_states.SHELVED_OFFLOADED)
new_az = "west_az"
fake_spec = objects.RequestSpec()
fake_spec.availability_zone = "fake-old-az"
get_by_instance_uuid.return_value = fake_spec
self.compute_api.unshelve(self.context, instance, new_az=new_az)
mock_save.assert_called_once_with()
self.assertEqual(new_az, fake_spec.availability_zone)
mock_validate_unshelve_az.assert_called_once_with(
self.context, instance, new_az)