Merge "Add scheduler retries for prep_resize operations."
This commit is contained in:
commit
82a84d3b85
@ -212,7 +212,7 @@ def _get_image_meta(context, image_ref):
|
||||
class ComputeManager(manager.SchedulerDependentManager):
|
||||
"""Manages the running instances from creation to destruction."""
|
||||
|
||||
RPC_API_VERSION = '2.9'
|
||||
RPC_API_VERSION = '2.10'
|
||||
|
||||
def __init__(self, compute_driver=None, *args, **kwargs):
|
||||
"""Load configuration options and connect to the hypervisor."""
|
||||
@ -531,32 +531,39 @@ class ComputeManager(manager.SchedulerDependentManager):
|
||||
with excutils.save_and_reraise_exception():
|
||||
self._set_instance_error_state(context, instance['uuid'])
|
||||
|
||||
def _log_original_error(self, exc_info, instance_uuid):
|
||||
type_, value, tb = exc_info
|
||||
LOG.error(_('Error: %s') %
|
||||
traceback.format_exception(type_, value, tb),
|
||||
instance_uuid=instance_uuid)
|
||||
|
||||
def _reschedule_or_reraise(self, context, instance, requested_networks,
|
||||
admin_password, injected_files, is_first_time,
|
||||
request_spec, filter_properties):
|
||||
"""Try to re-schedule the build or re-raise the original build error to
|
||||
error out the instance.
|
||||
"""
|
||||
type_, value, tb = sys.exc_info() # save original exception
|
||||
rescheduled = False
|
||||
exc_info = sys.exc_info()
|
||||
instance_uuid = instance['uuid']
|
||||
|
||||
def _log_original_error():
|
||||
LOG.error(_('Build error: %s') %
|
||||
traceback.format_exception(type_, value, tb),
|
||||
instance_uuid=instance_uuid)
|
||||
rescheduled = False
|
||||
|
||||
try:
|
||||
self._deallocate_network(context, instance)
|
||||
except Exception:
|
||||
# do not attempt retry if network de-allocation failed:
|
||||
_log_original_error()
|
||||
self._log_original_error(exc_info, instance_uuid)
|
||||
raise
|
||||
|
||||
try:
|
||||
rescheduled = self._reschedule(context, instance_uuid,
|
||||
requested_networks, admin_password, injected_files,
|
||||
is_first_time, request_spec, filter_properties)
|
||||
method_args = (request_spec, admin_password, injected_files,
|
||||
requested_networks, is_first_time, filter_properties)
|
||||
task_state = task_states.SCHEDULING
|
||||
|
||||
rescheduled = self._reschedule(context, request_spec,
|
||||
instance['uuid'], filter_properties,
|
||||
self.scheduler_rpcapi.run_instance, method_args,
|
||||
task_state)
|
||||
|
||||
except Exception:
|
||||
rescheduled = False
|
||||
LOG.exception(_("Error trying to reschedule"),
|
||||
@ -564,14 +571,14 @@ class ComputeManager(manager.SchedulerDependentManager):
|
||||
|
||||
if rescheduled:
|
||||
# log the original build error
|
||||
_log_original_error()
|
||||
self._log_original_error(exc_info, instance_uuid)
|
||||
else:
|
||||
# not re-scheduling
|
||||
raise type_, value, tb
|
||||
raise exc_info[0], exc_info[1], exc_info[2]
|
||||
|
||||
def _reschedule(self, context, instance_uuid, requested_networks,
|
||||
admin_password, injected_files, is_first_time, request_spec,
|
||||
filter_properties):
|
||||
def _reschedule(self, context, request_spec, filter_properties,
|
||||
instance_uuid, scheduler_method, method_args, task_state):
|
||||
"""Attempt to re-schedule a compute operation."""
|
||||
|
||||
retry = filter_properties.get('retry', None)
|
||||
if not retry:
|
||||
@ -587,16 +594,14 @@ class ComputeManager(manager.SchedulerDependentManager):
|
||||
|
||||
request_spec['instance_uuids'] = [instance_uuid]
|
||||
|
||||
LOG.debug(_("Re-scheduling instance: attempt %d"),
|
||||
retry['num_attempts'], instance_uuid=instance_uuid)
|
||||
LOG.debug(_("Re-scheduling %(method)s: attempt %(num)d") %
|
||||
{'method': scheduler_method.func_name,
|
||||
'num': retry['num_attempts']}, instance_uuid=instance_uuid)
|
||||
|
||||
# reset the task state:
|
||||
self._instance_update(context, instance_uuid,
|
||||
task_state=task_states.SCHEDULING)
|
||||
self._instance_update(context, instance_uuid, task_state=task_state)
|
||||
|
||||
self.scheduler_rpcapi.run_instance(context,
|
||||
request_spec, admin_password, injected_files,
|
||||
requested_networks, is_first_time, filter_properties)
|
||||
scheduler_method(context, *method_args)
|
||||
return True
|
||||
|
||||
@manager.periodic_task
|
||||
@ -1574,7 +1579,8 @@ class ComputeManager(manager.SchedulerDependentManager):
|
||||
@reverts_task_state
|
||||
@wrap_instance_fault
|
||||
def prep_resize(self, context, image, instance, instance_type,
|
||||
reservations=None):
|
||||
reservations=None, request_spec=None,
|
||||
filter_properties=None):
|
||||
"""Initiates the process of moving a running instance to another host.
|
||||
|
||||
Possibly changes the RAM and disk size in the process.
|
||||
@ -1587,38 +1593,82 @@ class ComputeManager(manager.SchedulerDependentManager):
|
||||
self._notify_about_instance_usage(
|
||||
context, instance, "resize.prep.start")
|
||||
|
||||
same_host = instance['host'] == self.host
|
||||
if same_host and not FLAGS.allow_resize_to_same_host:
|
||||
self._set_instance_error_state(context, instance['uuid'])
|
||||
msg = _('destination same as source!')
|
||||
raise exception.MigrationError(msg)
|
||||
try:
|
||||
same_host = instance['host'] == self.host
|
||||
if same_host and not FLAGS.allow_resize_to_same_host:
|
||||
self._set_instance_error_state(context, instance['uuid'])
|
||||
msg = _('destination same as source!')
|
||||
raise exception.MigrationError(msg)
|
||||
|
||||
# TODO(russellb): no-db-compute: Send the old instance type info
|
||||
# that is needed via rpc so db access isn't required here.
|
||||
old_instance_type_id = instance['instance_type_id']
|
||||
old_instance_type = instance_types.get_instance_type(
|
||||
old_instance_type_id)
|
||||
# TODO(russellb): no-db-compute: Send the old instance type
|
||||
# info that is needed via rpc so db access isn't required
|
||||
# here.
|
||||
old_instance_type_id = instance['instance_type_id']
|
||||
old_instance_type = instance_types.get_instance_type(
|
||||
old_instance_type_id)
|
||||
|
||||
migration_ref = self.db.migration_create(context.elevated(),
|
||||
{'instance_uuid': instance['uuid'],
|
||||
'source_compute': instance['host'],
|
||||
'dest_compute': self.host,
|
||||
'dest_host': self.driver.get_host_ip_addr(),
|
||||
'old_instance_type_id': old_instance_type['id'],
|
||||
'new_instance_type_id': instance_type['id'],
|
||||
'status': 'pre-migrating'})
|
||||
migration_ref = self.db.migration_create(context.elevated(),
|
||||
{'instance_uuid': instance['uuid'],
|
||||
'source_compute': instance['host'],
|
||||
'dest_compute': self.host,
|
||||
'dest_host': self.driver.get_host_ip_addr(),
|
||||
'old_instance_type_id': old_instance_type['id'],
|
||||
'new_instance_type_id': instance_type['id'],
|
||||
'status': 'pre-migrating'})
|
||||
|
||||
LOG.audit(_('Migrating'), context=context, instance=instance)
|
||||
self.compute_rpcapi.resize_instance(context, instance,
|
||||
migration_ref, image, reservations)
|
||||
LOG.audit(_('Migrating'), context=context,
|
||||
instance=instance)
|
||||
self.compute_rpcapi.resize_instance(context, instance,
|
||||
migration_ref, image, reservations)
|
||||
|
||||
extra_usage_info = dict(
|
||||
new_instance_type=instance_type['name'],
|
||||
new_instance_type_id=instance_type['id'])
|
||||
except Exception:
|
||||
# try to re-schedule the resize elsewhere:
|
||||
self._reschedule_resize_or_reraise(context, image, instance,
|
||||
instance_type, reservations, request_spec,
|
||||
filter_properties)
|
||||
finally:
|
||||
extra_usage_info = dict(
|
||||
new_instance_type=instance_type['name'],
|
||||
new_instance_type_id=instance_type['id'])
|
||||
|
||||
self._notify_about_instance_usage(
|
||||
context, instance, "resize.prep.end",
|
||||
extra_usage_info=extra_usage_info)
|
||||
self._notify_about_instance_usage(
|
||||
context, instance, "resize.prep.end",
|
||||
extra_usage_info=extra_usage_info)
|
||||
|
||||
def _reschedule_resize_or_reraise(self, context, image, instance,
|
||||
instance_type, reservations, request_spec, filter_properties):
|
||||
"""Try to re-schedule the resize or re-raise the original error to
|
||||
error out the instance.
|
||||
"""
|
||||
if not request_spec:
|
||||
request_spec = {}
|
||||
if not filter_properties:
|
||||
filter_properties = {}
|
||||
|
||||
exc_info = sys.exc_info()
|
||||
rescheduled = False
|
||||
instance_uuid = instance['uuid']
|
||||
|
||||
try:
|
||||
scheduler_method = self.scheduler_rpcapi.prep_resize
|
||||
method_args = (instance, instance_type, image, request_spec,
|
||||
filter_properties, reservations)
|
||||
task_state = task_states.RESIZE_PREP
|
||||
|
||||
rescheduled = self._reschedule(context, request_spec,
|
||||
filter_properties, instance_uuid, scheduler_method,
|
||||
method_args, task_state)
|
||||
except Exception:
|
||||
rescheduled = False
|
||||
LOG.exception(_("Error trying to reschedule"),
|
||||
instance_uuid=instance_uuid)
|
||||
|
||||
if rescheduled:
|
||||
# log the original build error
|
||||
self._log_original_error(exc_info, instance_uuid)
|
||||
else:
|
||||
# not re-scheduling
|
||||
raise exc_info[0], exc_info[1], exc_info[2]
|
||||
|
||||
@exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
|
||||
@reverts_task_state
|
||||
|
@ -137,6 +137,7 @@ class ComputeAPI(nova.openstack.common.rpc.proxy.RpcProxy):
|
||||
2.7 - Remove migration_id, add migration to confirm_resize
|
||||
2.8 - Remove migration_id, add migration to finish_resize
|
||||
2.9 - Add publish_service_capabilities()
|
||||
2.10 - Adds filter_properties and request_spec to prep_resize()
|
||||
'''
|
||||
|
||||
#
|
||||
@ -343,13 +344,17 @@ class ComputeAPI(nova.openstack.common.rpc.proxy.RpcProxy):
|
||||
disk=disk), _compute_topic(self.topic, ctxt, host, None))
|
||||
|
||||
def prep_resize(self, ctxt, image, instance, instance_type, host,
|
||||
reservations=None):
|
||||
reservations=None, request_spec=None,
|
||||
filter_properties=None):
|
||||
instance_p = jsonutils.to_primitive(instance)
|
||||
instance_type_p = jsonutils.to_primitive(instance_type)
|
||||
self.cast(ctxt, self.make_msg('prep_resize',
|
||||
instance=instance_p, instance_type=instance_type_p,
|
||||
image=image, reservations=reservations),
|
||||
_compute_topic(self.topic, ctxt, host, None))
|
||||
image=image, reservations=reservations,
|
||||
request_spec=request_spec,
|
||||
filter_properties=filter_properties),
|
||||
_compute_topic(self.topic, ctxt, host, None),
|
||||
version='2.10')
|
||||
|
||||
def reboot_instance(self, ctxt, instance,
|
||||
block_device_info, network_info, reboot_type):
|
||||
|
@ -121,20 +121,21 @@ class FilterScheduler(driver.Scheduler):
|
||||
raise exception.NoValidHost(reason="")
|
||||
host = hosts.pop(0)
|
||||
|
||||
self._post_select_populate_filter_properties(filter_properties,
|
||||
host.host_state)
|
||||
|
||||
# context is not serializable
|
||||
filter_properties.pop('context', None)
|
||||
|
||||
# Forward off to the host
|
||||
self.compute_rpcapi.prep_resize(context, image, instance,
|
||||
instance_type, host.host_state.host, reservations)
|
||||
instance_type, host.host_state.host, reservations,
|
||||
request_spec=request_spec, filter_properties=filter_properties)
|
||||
|
||||
def _provision_resource(self, context, weighted_host, request_spec,
|
||||
filter_properties, requested_networks, injected_files,
|
||||
admin_password, is_first_time, instance_uuid=None):
|
||||
"""Create the requested resource in this Zone."""
|
||||
# Add a retry entry for the selected compute host:
|
||||
self._add_retry_host(filter_properties, weighted_host.host_state.host)
|
||||
|
||||
self._add_oversubscription_policy(filter_properties,
|
||||
weighted_host.host_state)
|
||||
|
||||
payload = dict(request_spec=request_spec,
|
||||
weighted_host=weighted_host.to_dict(),
|
||||
instance_id=instance_uuid)
|
||||
@ -144,6 +145,9 @@ class FilterScheduler(driver.Scheduler):
|
||||
|
||||
updated_instance = driver.instance_update_db(context, instance_uuid)
|
||||
|
||||
self._post_select_populate_filter_properties(filter_properties,
|
||||
weighted_host.host_state)
|
||||
|
||||
self.compute_rpcapi.run_instance(context, instance=updated_instance,
|
||||
host=weighted_host.host_state.host,
|
||||
request_spec=request_spec, filter_properties=filter_properties,
|
||||
@ -151,6 +155,16 @@ class FilterScheduler(driver.Scheduler):
|
||||
injected_files=injected_files,
|
||||
admin_password=admin_password, is_first_time=is_first_time)
|
||||
|
||||
def _post_select_populate_filter_properties(self, filter_properties,
|
||||
host_state):
|
||||
"""Add additional information to the filter properties after a host has
|
||||
been selected by the scheduling process.
|
||||
"""
|
||||
# Add a retry entry for the selected compute host:
|
||||
self._add_retry_host(filter_properties, host_state.host)
|
||||
|
||||
self._add_oversubscription_policy(filter_properties, host_state)
|
||||
|
||||
def _add_retry_host(self, filter_properties, host):
|
||||
"""Add a retry entry for the selected compute host. In the event that
|
||||
the request gets re-scheduled, this entry will signal that the given
|
||||
|
@ -21,7 +21,6 @@
|
||||
import base64
|
||||
import copy
|
||||
import datetime
|
||||
import functools
|
||||
import sys
|
||||
import time
|
||||
|
||||
@ -88,6 +87,10 @@ class FakeSchedulerAPI(object):
|
||||
instance, dest):
|
||||
pass
|
||||
|
||||
def prep_resize(self, ctxt, instance, instance_type, image, request_spec,
|
||||
filter_properties, reservations):
|
||||
pass
|
||||
|
||||
|
||||
class BaseTestCase(test.TestCase):
|
||||
|
||||
@ -5539,28 +5542,36 @@ class DisabledInstanceTypesTestCase(BaseTestCase):
|
||||
|
||||
|
||||
class ComputeReschedulingTestCase(BaseTestCase):
|
||||
"""Tests related to re-scheduling build requests"""
|
||||
"""Tests re-scheduling logic for new build requests"""
|
||||
|
||||
def setUp(self):
|
||||
super(ComputeReschedulingTestCase, self).setUp()
|
||||
|
||||
self._reschedule = self._reschedule_partial()
|
||||
self.expected_task_state = task_states.SCHEDULING
|
||||
|
||||
def fake_update(*args, **kwargs):
|
||||
self.updated_task_state = kwargs.get('task_state')
|
||||
self.stubs.Set(self.compute, '_instance_update', fake_update)
|
||||
|
||||
def _reschedule_partial(self):
|
||||
uuid = "12-34-56-78-90"
|
||||
def _reschedule(self, request_spec=None, filter_properties=None):
|
||||
if not filter_properties:
|
||||
filter_properties = {}
|
||||
|
||||
instance_uuid = "12-34-56-78-90"
|
||||
|
||||
requested_networks = None
|
||||
admin_password = None
|
||||
injected_files = None
|
||||
requested_networks = None
|
||||
is_first_time = False
|
||||
|
||||
return functools.partial(self.compute._reschedule, self.context, uuid,
|
||||
requested_networks, admin_password, injected_files,
|
||||
is_first_time, request_spec=None, filter_properties={})
|
||||
scheduler_method = self.compute.scheduler_rpcapi.run_instance
|
||||
method_args = (request_spec, admin_password, injected_files,
|
||||
requested_networks, is_first_time, filter_properties)
|
||||
task_state = task_states.SCHEDULING
|
||||
|
||||
return self.compute._reschedule(self.context, request_spec,
|
||||
filter_properties, instance_uuid, scheduler_method,
|
||||
method_args, self.expected_task_state)
|
||||
|
||||
def test_reschedule_no_filter_properties(self):
|
||||
"""no filter_properties will disable re-scheduling"""
|
||||
@ -5584,61 +5595,254 @@ class ComputeReschedulingTestCase(BaseTestCase):
|
||||
self.assertTrue(self._reschedule(filter_properties=filter_properties,
|
||||
request_spec=request_spec))
|
||||
self.assertEqual(1, len(request_spec['instance_uuids']))
|
||||
self.assertEqual(self.updated_task_state, task_states.SCHEDULING)
|
||||
self.assertEqual(self.updated_task_state, self.expected_task_state)
|
||||
|
||||
|
||||
class ComputeReschedulingExceptionTestCase(BaseTestCase):
|
||||
"""Tests for re-scheduling exception handling logic"""
|
||||
class ComputeReschedulingResizeTestCase(ComputeReschedulingTestCase):
|
||||
"""Test re-scheduling logic for prep_resize requests"""
|
||||
|
||||
def setUp(self):
|
||||
super(ComputeReschedulingExceptionTestCase, self).setUp()
|
||||
super(ComputeReschedulingResizeTestCase, self).setUp()
|
||||
self.expected_task_state = task_states.RESIZE_PREP
|
||||
|
||||
# cause _spawn to raise an exception to test the exception logic:
|
||||
def exploding_spawn(*args, **kwargs):
|
||||
raise test.TestingException()
|
||||
self.stubs.Set(self.compute, '_spawn',
|
||||
exploding_spawn)
|
||||
def _reschedule(self, request_spec=None, filter_properties=None):
|
||||
if not filter_properties:
|
||||
filter_properties = {}
|
||||
|
||||
self.fake_instance = jsonutils.to_primitive(
|
||||
self._create_fake_instance())
|
||||
self.instance_uuid = self.fake_instance['uuid']
|
||||
instance_uuid = "12-34-56-78-90"
|
||||
|
||||
def test_exception_with_rescheduling_disabled(self):
|
||||
"""Spawn fails and re-scheduling is disabled."""
|
||||
# this won't be re-scheduled:
|
||||
self.assertRaises(test.TestingException,
|
||||
self.compute._run_instance, self.context,
|
||||
None, {}, None, None, None, None, self.fake_instance)
|
||||
instance = {'uuid': instance_uuid}
|
||||
instance_type = {}
|
||||
image = None
|
||||
reservations = None
|
||||
|
||||
def test_exception_with_rescheduling_enabled(self):
|
||||
"""Spawn fails and re-scheduling is enabled. Original exception
|
||||
should *not* be re-raised.
|
||||
scheduler_method = self.compute.scheduler_rpcapi.prep_resize
|
||||
method_args = (instance, instance_type, image, request_spec,
|
||||
filter_properties, reservations)
|
||||
|
||||
return self.compute._reschedule(self.context, request_spec,
|
||||
filter_properties, instance_uuid, scheduler_method,
|
||||
method_args, self.expected_task_state)
|
||||
|
||||
|
||||
class InnerTestingException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ComputeRescheduleOrReraiseTestCase(BaseTestCase):
|
||||
"""Test logic and exception handling around rescheduling or re-raising
|
||||
original exceptions when builds fail.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(ComputeRescheduleOrReraiseTestCase, self).setUp()
|
||||
self.instance = self._create_fake_instance()
|
||||
|
||||
def test_reschedule_or_reraise_called(self):
|
||||
"""Basic sanity check to make sure _reschedule_or_reraise is called
|
||||
when a build fails.
|
||||
"""
|
||||
# provide the expected status so that this one will be re-scheduled:
|
||||
retry = dict(num_attempts=1)
|
||||
filter_properties = dict(retry=retry)
|
||||
request_spec = dict(num_attempts=1)
|
||||
# Assert that test.TestingException is not raised
|
||||
self.compute._run_instance(self.context, request_spec,
|
||||
filter_properties, None, None, None,
|
||||
True, self.fake_instance)
|
||||
self.mox.StubOutWithMock(self.compute, '_spawn')
|
||||
self.mox.StubOutWithMock(self.compute, '_reschedule_or_reraise')
|
||||
|
||||
def test_exception_context_cleared(self):
|
||||
"""Test with no rescheduling and an additional exception occurs
|
||||
clearing the original build error's exception context.
|
||||
self.compute._spawn(mox.IgnoreArg(), self.instance, None, None, None,
|
||||
False, None).AndRaise(test.TestingException("BuildError"))
|
||||
self.compute._reschedule_or_reraise(mox.IgnoreArg(), self.instance,
|
||||
None, None, None, False, None, {})
|
||||
|
||||
self.mox.ReplayAll()
|
||||
self.compute._run_instance(self.context, None, {}, None, None, None,
|
||||
False, self.instance)
|
||||
|
||||
def test_deallocate_network_fail(self):
|
||||
"""Test de-allocation of network failing before re-scheduling logic
|
||||
can even run.
|
||||
"""
|
||||
# clears the original exception context:
|
||||
class FleshWoundException(Exception):
|
||||
pass
|
||||
instance_uuid = self.instance['uuid']
|
||||
self.mox.StubOutWithMock(self.compute, '_deallocate_network')
|
||||
|
||||
def reschedule_explode(*args, **kwargs):
|
||||
raise FleshWoundException()
|
||||
self.stubs.Set(self.compute, '_reschedule', reschedule_explode)
|
||||
try:
|
||||
raise test.TestingException("Original")
|
||||
except Exception:
|
||||
exc_info = sys.exc_info()
|
||||
|
||||
# the original exception should now be raised:
|
||||
self.assertRaises(test.TestingException,
|
||||
self.compute._run_instance, self.context,
|
||||
None, {}, None, None, None, None, self.fake_instance)
|
||||
self.compute._deallocate_network(self.context,
|
||||
self.instance).AndRaise(InnerTestingException("Error"))
|
||||
self.compute._log_original_error(exc_info, instance_uuid)
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
# should raise the deallocation exception, not the original build
|
||||
# error:
|
||||
self.assertRaises(InnerTestingException,
|
||||
self.compute._reschedule_or_reraise, self.context,
|
||||
self.instance, None, None, None, False, None, {})
|
||||
|
||||
def test_reschedule_fail(self):
|
||||
"""Test handling of exception from _reschedule"""
|
||||
instance_uuid = self.instance['uuid']
|
||||
method_args = (None, None, None, None, False, {})
|
||||
self.mox.StubOutWithMock(self.compute, '_deallocate_network')
|
||||
self.mox.StubOutWithMock(self.compute, '_reschedule')
|
||||
|
||||
self.compute._deallocate_network(self.context,
|
||||
self.instance)
|
||||
self.compute._reschedule(self.context, None, instance_uuid,
|
||||
{}, self.compute.scheduler_rpcapi.run_instance,
|
||||
method_args, task_states.SCHEDULING).AndRaise(
|
||||
InnerTestingException("Inner"))
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
try:
|
||||
raise test.TestingException("Original")
|
||||
except Exception:
|
||||
# not re-scheduling, should raise the original build error:
|
||||
self.assertRaises(test.TestingException,
|
||||
self.compute._reschedule_or_reraise, self.context,
|
||||
self.instance, None, None, None, False, None, {})
|
||||
|
||||
def test_reschedule_false(self):
|
||||
"""Test not-rescheduling, but no nested exception"""
|
||||
instance_uuid = self.instance['uuid']
|
||||
method_args = (None, None, None, None, False, {})
|
||||
self.mox.StubOutWithMock(self.compute, '_deallocate_network')
|
||||
self.mox.StubOutWithMock(self.compute, '_reschedule')
|
||||
|
||||
self.compute._deallocate_network(self.context,
|
||||
self.instance)
|
||||
self.compute._reschedule(self.context, None, instance_uuid,
|
||||
{}, self.compute.scheduler_rpcapi.run_instance, method_args,
|
||||
task_states.SCHEDULING).AndReturn(False)
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
try:
|
||||
raise test.TestingException("Original")
|
||||
except Exception:
|
||||
# re-scheduling is False, the original build error should be
|
||||
# raised here:
|
||||
self.assertRaises(test.TestingException,
|
||||
self.compute._reschedule_or_reraise, self.context,
|
||||
self.instance, None, None, None, False, None, {})
|
||||
|
||||
def test_reschedule_true(self):
|
||||
"""Test behavior when re-scheduling happens"""
|
||||
instance_uuid = self.instance['uuid']
|
||||
method_args = (None, None, None, None, False, {})
|
||||
self.mox.StubOutWithMock(self.compute, '_deallocate_network')
|
||||
self.mox.StubOutWithMock(self.compute, '_reschedule')
|
||||
|
||||
try:
|
||||
raise test.TestingException("Original")
|
||||
except Exception:
|
||||
exc_info = sys.exc_info()
|
||||
|
||||
self.compute._deallocate_network(self.context,
|
||||
self.instance)
|
||||
self.compute._reschedule(self.context, None, instance_uuid,
|
||||
{}, self.compute.scheduler_rpcapi.run_instance,
|
||||
method_args, task_states.SCHEDULING).AndReturn(True)
|
||||
self.compute._log_original_error(exc_info, instance_uuid)
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
# re-scheduling is True, original error is logged, but nothing
|
||||
# is raised:
|
||||
self.compute._reschedule_or_reraise(self.context, self.instance,
|
||||
None, None, None, False, None, {})
|
||||
|
||||
|
||||
class ComputeRescheduleResizeOrReraiseTestCase(BaseTestCase):
|
||||
"""Test logic and exception handling around rescheduling prep resize
|
||||
requests
|
||||
"""
|
||||
def setUp(self):
|
||||
super(ComputeRescheduleResizeOrReraiseTestCase, self).setUp()
|
||||
self.instance = self._create_fake_instance()
|
||||
self.instance_uuid = self.instance['uuid']
|
||||
self.instance_type = instance_types.get_instance_type_by_name(
|
||||
"m1.tiny")
|
||||
|
||||
def test_reschedule_resize_or_reraise_called(self):
|
||||
"""Verify the rescheduling logic gets called when there is an error
|
||||
during prep_resize.
|
||||
"""
|
||||
self.mox.StubOutWithMock(self.compute.db, 'migration_create')
|
||||
self.mox.StubOutWithMock(self.compute, '_reschedule_resize_or_reraise')
|
||||
|
||||
self.compute.db.migration_create(mox.IgnoreArg(),
|
||||
mox.IgnoreArg()).AndRaise(test.TestingException("Original"))
|
||||
|
||||
self.compute._reschedule_resize_or_reraise(mox.IgnoreArg(), None,
|
||||
self.instance, self.instance_type, None, None, None)
|
||||
|
||||
self.mox.ReplayAll()
|
||||
|
||||
self.compute.prep_resize(self.context, None, self.instance,
|
||||
self.instance_type)
|
||||
|
||||
def test_reschedule_fails_with_exception(self):
|
||||
"""Original exception should be raised if the _reschedule method
|
||||
raises another exception
|
||||
"""
|
||||
method_args = (None, self.instance, self.instance_type, None, None,
|
||||
None)
|
||||
self.mox.StubOutWithMock(self.compute, "_reschedule")
|
||||
|
||||
self.compute._reschedule(self.context, None, None, self.instance_uuid,
|
||||
self.compute.scheduler_rpcapi.prep_resize, method_args,
|
||||
task_states.RESIZE_PREP).AndRaise(
|
||||
InnerTestingException("Inner"))
|
||||
self.mox.ReplayAll()
|
||||
|
||||
try:
|
||||
raise test.TestingException("Original")
|
||||
except Exception:
|
||||
self.assertRaises(test.TestingException,
|
||||
self.compute._reschedule_resize_or_reraise, self.context,
|
||||
None, self.instance, self.instance_type, None, {}, {})
|
||||
|
||||
def test_reschedule_false(self):
|
||||
"""Original exception should be raised if the resize is not
|
||||
rescheduled.
|
||||
"""
|
||||
method_args = (None, self.instance, self.instance_type, None, None,
|
||||
None)
|
||||
self.mox.StubOutWithMock(self.compute, "_reschedule")
|
||||
|
||||
self.compute._reschedule(self.context, None, None, self.instance_uuid,
|
||||
self.compute.scheduler_rpcapi.prep_resize, method_args,
|
||||
task_states.RESIZE_PREP).AndReturn(False)
|
||||
self.mox.ReplayAll()
|
||||
|
||||
try:
|
||||
raise test.TestingException("Original")
|
||||
except Exception:
|
||||
self.assertRaises(test.TestingException,
|
||||
self.compute._reschedule_resize_or_reraise, self.context,
|
||||
None, self.instance, self.instance_type, None, {}, {})
|
||||
|
||||
def test_reschedule_true(self):
|
||||
"""If rescheduled, the original resize exception should be logged"""
|
||||
method_args = (self.instance, self.instance_type, None, {}, {}, None)
|
||||
try:
|
||||
raise test.TestingException("Original")
|
||||
except Exception:
|
||||
exc_info = sys.exc_info()
|
||||
|
||||
self.mox.StubOutWithMock(self.compute, "_reschedule")
|
||||
self.mox.StubOutWithMock(self.compute, "_log_original_error")
|
||||
self.compute._reschedule(self.context, {}, {},
|
||||
self.instance_uuid,
|
||||
self.compute.scheduler_rpcapi.prep_resize, method_args,
|
||||
task_states.RESIZE_PREP).AndReturn(True)
|
||||
|
||||
self.compute._log_original_error(exc_info, self.instance_uuid)
|
||||
self.mox.ReplayAll()
|
||||
|
||||
self.compute._reschedule_resize_or_reraise(self.context, None,
|
||||
self.instance, self.instance_type, None, {}, {})
|
||||
|
||||
|
||||
class ComputeInactiveImageTestCase(BaseTestCase):
|
||||
|
@ -213,7 +213,10 @@ class ComputeRpcAPITestCase(test.TestCase):
|
||||
self._test_compute_api('prep_resize', 'cast',
|
||||
instance=self.fake_instance, instance_type='fake_type',
|
||||
image='fake_image', host='host',
|
||||
reservations=list('fake_res'))
|
||||
reservations=list('fake_res'),
|
||||
request_spec='fake_spec',
|
||||
filter_properties={'fakeprop': 'fakeval'},
|
||||
version='2.10')
|
||||
|
||||
def test_reboot_instance(self):
|
||||
self.maxDiff = None
|
||||
|
@ -18,6 +18,7 @@ Tests For Filter Scheduler.
|
||||
|
||||
import mox
|
||||
|
||||
from nova.compute import instance_types
|
||||
from nova.compute import utils as compute_utils
|
||||
from nova.compute import vm_states
|
||||
from nova import context
|
||||
@ -310,3 +311,52 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
|
||||
hosts = filter_properties['retry']['hosts']
|
||||
self.assertEqual(1, len(hosts))
|
||||
self.assertEqual(host, hosts[0])
|
||||
|
||||
def test_post_select_populate(self):
|
||||
"""Test addition of certain filter props after a host is selected"""
|
||||
retry = {'hosts': [], 'num_attempts': 1}
|
||||
filter_properties = {'retry': retry}
|
||||
sched = fakes.FakeFilterScheduler()
|
||||
|
||||
host_state = host_manager.HostState('host', 'compute')
|
||||
host_state.limits['vcpus'] = 5
|
||||
sched._post_select_populate_filter_properties(filter_properties,
|
||||
host_state)
|
||||
|
||||
self.assertEqual('host', filter_properties['retry']['hosts'][0])
|
||||
|
||||
self.assertEqual({'vcpus': 5}, host_state.limits)
|
||||
|
||||
def test_prep_resize_post_populates_retry(self):
|
||||
"""Prep resize should add a 'host' entry to the retry dict"""
|
||||
sched = fakes.FakeFilterScheduler()
|
||||
|
||||
image = 'image'
|
||||
instance = db.instance_create(self.context, {})
|
||||
|
||||
instance_properties = {'project_id': 'fake', 'os_type': 'Linux'}
|
||||
instance_type = instance_types.get_instance_type_by_name("m1.tiny")
|
||||
request_spec = {'instance_properties': instance_properties,
|
||||
'instance_type': instance_type}
|
||||
retry = {'hosts': [], 'num_attempts': 1}
|
||||
filter_properties = {'retry': retry}
|
||||
reservations = None
|
||||
|
||||
host = fakes.FakeHostState('host', 'compute', {})
|
||||
weighted_host = least_cost.WeightedHost(1, host)
|
||||
hosts = [weighted_host]
|
||||
|
||||
self.mox.StubOutWithMock(sched, '_schedule')
|
||||
self.mox.StubOutWithMock(sched.compute_rpcapi, 'prep_resize')
|
||||
|
||||
sched._schedule(self.context, 'compute', request_spec,
|
||||
filter_properties, [instance['uuid']]).AndReturn(hosts)
|
||||
sched.compute_rpcapi.prep_resize(self.context, image, instance,
|
||||
instance_type, 'host', reservations, request_spec=request_spec,
|
||||
filter_properties=filter_properties)
|
||||
|
||||
self.mox.ReplayAll()
|
||||
sched.schedule_prep_resize(self.context, image, request_spec,
|
||||
filter_properties, instance, instance_type, reservations)
|
||||
|
||||
self.assertEqual(['host'], filter_properties['retry']['hosts'])
|
||||
|
Loading…
Reference in New Issue
Block a user