OpenStack Compute (Nova)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

1969 lines
84 KiB

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
import mock
from nova import exception
from nova.tests import fixtures
from nova.tests.functional.api import client
from nova.tests.functional.notification_sample_tests \
import notification_sample_base
class TestInstanceNotificationSampleWithMultipleCompute(
notification_sample_base.NotificationSampleTestBase):
def setUp(self):
self.flags(compute_driver='fake.FakeLiveMigrateDriver')
self.flags(bdms_in_notifications='True', group='notifications')
super(TestInstanceNotificationSampleWithMultipleCompute, self).setUp()
self.neutron = fixtures.NeutronFixture(self)
self.useFixture(self.neutron)
self.cinder = fixtures.CinderFixture(self)
self.useFixture(self.cinder)
self.useFixture(fixtures.AllServicesCurrent())
def test_multiple_compute_actions(self):
# There are not going to be real network-vif-plugged events coming
# so don't wait for them.
self.flags(live_migration_wait_for_vif_plug=False, group='compute')
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._wait_for_notification('instance.create.end')
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
# server will boot on the 'compute' host
self.compute2 = self.start_service('compute', host='host2')
actions = [
self._test_live_migration_rollback,
self._test_live_migration_abort,
self._test_live_migration_success,
self._test_evacuate_server,
self._test_live_migration_force_complete
]
for action in actions:
self.notifier.reset()
action(server)
# Ensure that instance is in active state after an action
self._wait_for_state_change(server, 'ACTIVE')
@mock.patch('nova.compute.manager.ComputeManager.'
'_live_migration_cleanup_flags', return_value=[True, False])
@mock.patch('nova.compute.rpcapi.ComputeAPI.pre_live_migration',
side_effect=exception.DestinationDiskExists(path='path'))
def _test_live_migration_rollback(self, server, mock_migration,
mock_flags):
post = {
'os-migrateLive': {
'host': 'host2',
'block_migration': True,
}
}
self.admin_api.post_server_action(server['id'], post)
self._wait_for_notification(
'instance.live_migration_rollback_dest.end')
# 0. scheduler.select_destinations.start
# 1. scheduler.select_destinations.end
# 2. instance.live_migration_rollback.start
# 3. instance.live_migration_rollback.end
# 4. instance.live_migration_rollback_dest.start
# 5. instance.live_migration_rollback_dest.end
self.assertEqual(6, len(self.notifier.versioned_notifications),
[x['event_type'] for x in
self.notifier.versioned_notifications])
self._verify_notification(
'instance-live_migration_rollback-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-live_migration_rollback-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
self._verify_notification(
'instance-live_migration_rollback_dest-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[4])
self._verify_notification(
'instance-live_migration_rollback_dest-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[5])
def _test_live_migration_success(self, server):
post = {
'os-migrateLive': {
'host': 'host2',
'block_migration': True,
}
}
self.admin_api.post_server_action(server['id'], post)
self._wait_for_notification('instance.live_migration_pre.end')
# 0. scheduler.select_destinations.start
# 1. scheduler.select_destinations.end
# 2. instance.live_migration_pre.start
# 3. instance.live_migration_pre.end
self.assertEqual(4, len(self.notifier.versioned_notifications),
[x['event_type'] for x in
self.notifier.versioned_notifications])
self._verify_notification(
'instance-live_migration_pre-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-live_migration_pre-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
migrations = self.admin_api.get_active_migrations(server['id'])
self.assertEqual(1, len(migrations))
self._wait_for_notification('instance.live_migration_post.end')
# 0. scheduler.select_destinations.start
# 1. scheduler.select_destinations.end
# 2. instance.live_migration_pre.start
# 3. instance.live_migration_pre.end
# 4. instance.live_migration_post.start
# 5. instance.live_migration_post_dest.start
# 6. instance.live_migration_post_dest.end
# 7. instance.live_migration_post.end
self.assertEqual(8, len(self.notifier.versioned_notifications),
[x['event_type'] for x in
self.notifier.versioned_notifications])
self._verify_notification(
'instance-live_migration_post-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[4])
self._verify_notification(
'instance-live_migration_post_dest-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[5])
self._verify_notification(
'instance-live_migration_post_dest-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[6])
self._verify_notification(
'instance-live_migration_post-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[7])
def _test_live_migration_abort(self, server):
post = {
"os-migrateLive": {
"host": "host2",
"block_migration": False,
}
}
self.admin_api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'MIGRATING')
migrations = self._wait_and_get_migrations(server)
self.admin_api.delete_migration(server['id'], migrations[0]['id'])
self._wait_for_notification('instance.live_migration_abort.start')
self._wait_for_state_change(server, 'ACTIVE')
# NOTE(gibi): the intance.live_migration_rollback notification emitted
# after the instance.live_migration_abort notification so we have to
# wait for the rollback to ensure we can assert both notifications
# below
self._wait_for_notification('instance.live_migration_rollback.end')
# 0. scheduler.select_destinations.start
# 1. scheduler.select_destinations.end
# 2. instance.live_migration_pre.start
# 3. instance.live_migration_pre.end
# 4. instance.live_migration_abort.start
# 5. instance.live_migration_abort.end
# 6. instance.live_migration_rollback.start
# 7. instance.live_migration_rollback.end
self.assertEqual(8, len(self.notifier.versioned_notifications),
[x['event_type'] for x in
self.notifier.versioned_notifications])
self._verify_notification(
'instance-live_migration_pre-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-live_migration_pre-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
self._verify_notification(
'instance-live_migration_abort-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[4])
self._verify_notification(
'instance-live_migration_abort-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[5])
self._verify_notification(
'instance-live_migration_rollback-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[6])
self._verify_notification(
'instance-live_migration_rollback-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[7])
def _test_evacuate_server(self, server):
services = self.admin_api.get_services(host='host2',
binary='nova-compute')
service_id = services[0]['id']
self.compute2.stop()
self.admin_api.put_service(service_id, {'forced_down': True})
evacuate = {
'evacuate': {
'host': 'compute',
}
}
self.admin_api.post_server_action(server['id'], evacuate)
self._wait_for_state_change(server, expected_status='REBUILD')
self._wait_for_state_change(server, expected_status='ACTIVE')
notifications = self._get_notifications('instance.evacuate')
self.assertEqual(1, len(notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-evacuate',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=notifications[0])
self.compute2.start()
self._wait_for_migration_status(server, ['completed'])
self.admin_api.put_service(service_id, {'forced_down': False})
def _test_live_migration_force_complete(self, server):
post = {
'os-migrateLive': {
'host': 'host2',
'block_migration': True,
}
}
self.admin_api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'MIGRATING')
migrations = self._wait_and_get_migrations(server)
migration_id = migrations[0]['id']
self.admin_api.force_complete_migration(server['id'], migration_id)
# Note that we wait for instance.live_migration_force_complete.end but
# by the time we check versioned notifications received we could have
# entered ComputeManager._post_live_migration which could emit up to
# four other notifications:
# - instance.live_migration_post.start
# - instance.live_migration_post_dest.start
# - instance.live_migration_post_dest.end
# - instance.live_migration_post.end
# We are not concerned about those in this test so that's why we stop
# once we get instance.live_migration_force_complete.end and assert
# we got at least 6 notifications.
self._wait_for_notification(
'instance.live_migration_force_complete.end')
# 0. scheduler.select_destinations.start
# 1. scheduler.select_destinations.end
# 2. instance.live_migration_pre.start
# 3. instance.live_migration_pre.end
# 4. instance.live_migration_force_complete.start
# 5. instance.live_migration_force_complete.end
self.assertGreaterEqual(len(self.notifier.versioned_notifications), 6,
self.notifier.versioned_notifications)
self._verify_notification(
'instance-live_migration_force_complete-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[4])
self._verify_notification(
'instance-live_migration_force_complete-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[5])
class TestInstanceNotificationSample(
notification_sample_base.NotificationSampleTestBase):
def setUp(self):
self.flags(bdms_in_notifications='True', group='notifications')
super(TestInstanceNotificationSample, self).setUp()
self.neutron = fixtures.NeutronFixture(self)
self.useFixture(self.neutron)
self.cinder = fixtures.CinderFixture(self)
self.useFixture(self.cinder)
def _wait_until_swap_volume(self, server, volume_id):
for i in range(50):
volume_attachments = self.api.get_server_volumes(server['id'])
if len(volume_attachments) > 0:
for volume_attachment in volume_attachments:
if volume_attachment['volumeId'] == volume_id:
return
time.sleep(0.5)
self.fail('Volume swap operation failed.')
def test_instance_action(self):
# A single test case is used to test most of the instance action
# notifications to avoid booting up an instance for every action
# separately.
# Every instance action test function shall make sure that after the
# function the instance is in active state and usable by other actions.
# Therefore some action especially delete cannot be used here as
# recovering from that action would mean to recreate the instance and
# that would go against the whole purpose of this optimization
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
actions = [
self._test_power_off_on_server,
self._test_restore_server,
self._test_suspend_resume_server,
self._test_pause_unpause_server,
self._test_shelve_and_shelve_offload_server,
self._test_unshelve_server,
self._test_resize_and_revert_server,
self._test_snapshot_server,
self._test_reboot_server,
self._test_reboot_server_error,
self._test_trigger_crash_dump,
self._test_volume_detach_attach_server,
self._test_rescue_unrescue_server,
self._test_soft_delete_server,
self._test_attach_volume_error,
self._test_interface_attach_and_detach,
self._test_interface_attach_error,
self._test_lock_unlock_instance,
self._test_lock_unlock_instance_with_reason,
]
for action in actions:
self.notifier.reset()
action(server)
# Ensure that instance is in active state after an action
self._wait_for_state_change(server, 'ACTIVE')
# if the test step did not raised then we consider the step as
# succeeded. We drop the logs to avoid causing subunit parser
# errors due to logging too much at the end of the test case.
self.stdlog.delete_stored_logs()
def test_create_delete_server(self):
fake_trusted_certs = ['cert-id-1', 'cert-id-2']
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}],
'tags': ['tag'],
'trusted_image_certificates': fake_trusted_certs})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
self._delete_server(server)
# NOTE(gibi): The wait_unit_deleted() call polls the REST API to see if
# the instance is disappeared however the _delete_instance() in
# compute/manager destroys the instance first then send the
# instance.delete.end notification. So to avoid race condition the test
# needs to wait for the notification as well here.
self._wait_for_notification('instance.delete.end')
self.assertEqual(9, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
# This list needs to be in order.
expected_notifications = [
'instance-create-start',
'instance-create-end',
'instance-update-tags-action',
'instance-volume_attach-start',
'instance-volume_attach-end',
'instance-delete-start',
'instance-shutdown-start',
'instance-shutdown-end',
'instance-delete-end'
]
for idx, notification in enumerate(expected_notifications):
self._verify_notification(
notification,
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[idx])
@mock.patch('nova.compute.manager.ComputeManager._build_resources')
def test_create_server_error(self, mock_build):
def _build_resources(*args, **kwargs):
raise exception.FlavorDiskTooSmall()
mock_build.side_effect = _build_resources
fake_trusted_certs = ['cert-id-1', 'cert-id-2']
server = self._boot_a_server(
expected_status='ERROR',
extra_params={'networks': [{'port': self.neutron.port_1['id']}],
'tags': ['tag'],
'trusted_image_certificates': fake_trusted_certs})
# 0. scheduler.select_destinations.start
# 1. scheduler.select_destinations.end
# 2. instance-create-start
# 3. instance-create-error
self.assertEqual(4, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
tb = self.notifier.versioned_notifications[3]['payload'][
'nova_object.data']['fault']['nova_object.data']['traceback']
self.assertIn('raise exception.FlavorDiskTooSmall()', tb)
self._verify_notification(
'instance-create-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-create-error',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'fault.traceback': self.ANY},
actual=self.notifier.versioned_notifications[3])
self.notifier.reset()
self._delete_server(server)
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-delete-start_not_scheduled',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-delete-end_not_scheduled',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def test_instance_exists_usage_audit(self):
# TODO(xavvior): Should create a functional test for the
# "instance_usage_audit" periodic task. We didn't find usable
# solution for this problem, however we tried to test it in
# several ways.
pass
def test_instance_exists(self):
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
self.notifier.reset()
post = {
'rebuild': {
'imageRef': 'a2459075-d96c-40d5-893e-577ff92e721c',
'metadata': {}
}
}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, expected_status='REBUILD')
self._wait_for_state_change(server, expected_status='ACTIVE')
notifications = self._get_notifications('instance.exists')
self._verify_notification(
'instance-exists',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']
},
actual=notifications[0])
def test_delete_server_while_compute_is_down(self):
server = self._boot_a_server(
expected_status='ACTIVE',
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
service_id = self.api.get_service_id('nova-compute')
self.admin_api.put_service_force_down(service_id, True)
self.notifier.reset()
self._delete_server(server)
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-delete-start_compute_down',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-delete-end_compute_down',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self.admin_api.put_service_force_down(service_id, False)
def _verify_instance_update_steps(self, steps, notifications,
initial=None):
replacements = {}
if initial:
replacements = initial
for i, step in enumerate(steps):
replacements.update(step)
self._verify_notification(
'instance-update',
replacements=replacements,
actual=notifications[i])
return replacements
def test_create_delete_server_with_instance_update(self):
# This makes server network creation synchronous which is necessary
# for notification samples that expect instance.info_cache.network_info
# to be set.
self.useFixture(fixtures.SpawnIsSynchronousFixture())
self.flags(notify_on_state_change='vm_and_task_state',
group='notifications')
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
instance_updates = self._wait_for_notifications('instance.update', 8)
# The first notification comes from the nova-conductor, the
# eighth notification comes from nova-api the
# rest is from the nova-compute. To keep the test simpler
# assert this fact and then modify the publisher_id of the
# first and eighth notification to match the template
self.assertEqual('nova-conductor:fake-mini',
instance_updates[0]['publisher_id'])
self.assertEqual('nova-api:fake-mini',
instance_updates[7]['publisher_id'])
instance_updates[0]['publisher_id'] = 'nova-compute:fake-mini'
instance_updates[7]['publisher_id'] = 'nova-compute:fake-mini'
create_steps = [
# nothing -> scheduling
{'reservation_id': server['reservation_id'],
'uuid': server['id'],
'host': None,
'node': None,
'state_update.new_task_state': 'scheduling',
'state_update.old_task_state': 'scheduling',
'state_update.state': 'building',
'state_update.old_state': 'building',
'state': 'building'},
# scheduling -> building
{
'state_update.new_task_state': None,
'state_update.old_task_state': 'scheduling',
'task_state': None},
# scheduled
{'host': 'compute',
'node': 'fake-mini',
'state_update.old_task_state': None,
'updated_at': '2012-10-29T13:42:11Z'},
# building -> networking
{'state_update.new_task_state': 'networking',
'state_update.old_task_state': 'networking',
'task_state': 'networking'},
# networking -> block_device_mapping
{'state_update.new_task_state': 'block_device_mapping',
'state_update.old_task_state': 'networking',
'task_state': 'block_device_mapping',
'ip_addresses': [{
"nova_object.name": "IpPayload",
"nova_object.namespace": "nova",
"nova_object.version": "1.0",
"nova_object.data": {
"mac": "fa:16:3e:4c:2c:30",
"address": "192.168.1.3",
"port_uuid": "ce531f90-199f-48c0-816c-13e38010b442",
"meta": {},
"version": 4,
"label": "private",
"device_name": "tapce531f90-19"
}}]
},
# block_device_mapping -> spawning
{'state_update.new_task_state': 'spawning',
'state_update.old_task_state': 'block_device_mapping',
'task_state': 'spawning',
},
# spawning -> active
{'state_update.new_task_state': None,
'state_update.old_task_state': 'spawning',
'state_update.state': 'active',
'launched_at': '2012-10-29T13:42:11Z',
'state': 'active',
'task_state': None,
'power_state': 'running'},
# tag added
{'state_update.old_task_state': None,
'state_update.old_state': 'active',
'tags': ['tag1']},
]
replacements = self._verify_instance_update_steps(
create_steps, instance_updates)
self.notifier.reset()
self._delete_server(server)
instance_updates = self._get_notifications('instance.update')
self.assertEqual(2, len(instance_updates),
self.notifier.versioned_notifications)
delete_steps = [
# active -> deleting
{'state_update.new_task_state': 'deleting',
'state_update.old_task_state': 'deleting',
'state_update.old_state': 'active',
'state': 'active',
'task_state': 'deleting',
'bandwidth': [],
'tags': ["tag1"],
'block_devices': [{
"nova_object.data": {
"boot_index": None,
"delete_on_termination": False,
"device_name": "/dev/sdb",
"tag": None,
"volume_id": "a07f71dc-8151-4e7d-a0cc-cd24a3f11113"
},
"nova_object.name": "BlockDevicePayload",
"nova_object.namespace": "nova",
"nova_object.version": "1.0"
}]
},
# deleting -> deleted
{'state_update.new_task_state': None,
'state_update.old_task_state': 'deleting',
'state_update.old_state': 'active',
'state_update.state': 'deleted',
'state': 'deleted',
'task_state': None,
'terminated_at': '2012-10-29T13:42:11Z',
'ip_addresses': [],
'power_state': 'pending',
'bandwidth': [],
'tags': ["tag1"],
'block_devices': [{
"nova_object.data": {
"boot_index": None,
"delete_on_termination": False,
"device_name": "/dev/sdb",
"tag": None,
"volume_id": "a07f71dc-8151-4e7d-a0cc-cd24a3f11113"
},
"nova_object.name": "BlockDevicePayload",
"nova_object.namespace": "nova",
"nova_object.version": "1.0"
}]
},
]
self._verify_instance_update_steps(delete_steps, instance_updates,
initial=replacements)
def _test_power_off_on_server(self, server):
self.api.post_server_action(server['id'], {'os-stop': {}})
self._wait_for_state_change(server, expected_status='SHUTOFF')
self.api.post_server_action(server['id'], {'os-start': {}})
self._wait_for_state_change(server, expected_status='ACTIVE')
self.assertEqual(4, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-power_off-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-power_off-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self._verify_notification(
'instance-power_on-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-power_on-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
def _test_shelve_and_shelve_offload_server(self, server):
self.flags(shelved_offload_time=-1)
self.api.post_server_action(server['id'], {'shelve': {}})
self._wait_for_state_change(server, expected_status='SHELVED')
self.assertEqual(3, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-shelve-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self._verify_notification(
'instance-shelve-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self.notifier.reset()
self.api.post_server_action(server['id'], {'shelveOffload': {}})
# we need to wait for the instance.host to become None as well before
# we can unshelve to make sure that the unshelve.start notification
# payload is stable as the compute manager first sets the instance
# state then a bit later sets the instance.host to None.
self._wait_for_server_parameter(server,
{'status': 'SHELVED_OFFLOADED',
'OS-EXT-SRV-ATTR:host': None})
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-shelve_offload-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-shelve_offload-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self.api.post_server_action(server['id'], {'unshelve': None})
self._wait_for_state_change(server, 'ACTIVE')
self._wait_for_notification('instance.unshelve.end')
def _test_unshelve_server(self, server):
# setting the shelved_offload_time to 0 should set the
# instance status to 'SHELVED_OFFLOADED'
self.flags(shelved_offload_time = 0)
self.api.post_server_action(server['id'], {'shelve': {}})
# we need to wait for the instance.host to become None as well before
# we can unshelve to make sure that the unshelve.start notification
# payload is stable as the compute manager first sets the instance
# state then a bit later sets the instance.host to None.
self._wait_for_server_parameter(server,
{'status': 'SHELVED_OFFLOADED',
'OS-EXT-SRV-ATTR:host': None})
post = {'unshelve': None}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'ACTIVE')
self._wait_for_notification('instance.unshelve.end')
self.assertEqual(9, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-unshelve-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[7])
self._verify_notification(
'instance-unshelve-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[8])
def _test_suspend_resume_server(self, server):
post = {'suspend': {}}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'SUSPENDED')
post = {'resume': None}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'ACTIVE')
# Four versioned notification are generated.
# 0. instance-suspend-start
# 1. instance-suspend-end
# 2. instance-resume-start
# 3. instance-resume-end
self.assertEqual(4, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-suspend-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-suspend-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self._verify_notification(
'instance-resume-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-resume-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
self.flags(reclaim_instance_interval=0)
def _test_pause_unpause_server(self, server):
self.api.post_server_action(server['id'], {'pause': {}})
self._wait_for_state_change(server, 'PAUSED')
self.api.post_server_action(server['id'], {'unpause': {}})
self._wait_for_state_change(server, 'ACTIVE')
# Four versioned notifications are generated
# 0. instance-pause-start
# 1. instance-pause-end
# 2. instance-unpause-start
# 3. instance-unpause-end
self.assertEqual(4, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-pause-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-pause-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self._verify_notification(
'instance-unpause-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-unpause-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
def _build_destination_payload(self):
cell1 = self.cell_mappings.get('cell1')
return {
'nova_object.version': '1.0',
'nova_object.namespace': 'nova',
'nova_object.name': 'DestinationPayload',
'nova_object.data': {
'aggregates': None,
'cell': {
'nova_object.version': '2.0',
'nova_object.namespace': 'nova',
'nova_object.name': 'CellMappingPayload',
'nova_object.data': {
'disabled': False,
'name': u'cell1',
'uuid': cell1.uuid
}
}
}
}
def _test_resize_and_revert_server(self, server):
self.flags(allow_resize_to_same_host=True)
other_flavor_body = {
'flavor': {
'name': 'other_flavor',
'ram': 256,
'vcpus': 1,
'disk': 1,
'id': 'd5a8bb54-365a-45ae-abdb-38d249df7845'
}
}
other_flavor_id = self.api.post_flavor(other_flavor_body)['id']
extra_specs = {
"extra_specs": {
"hw:watchdog_action": "reset"}}
self.admin_api.post_extra_spec(other_flavor_id, extra_specs)
# Ignore the create flavor notification
self.notifier.reset()
post = {
'resize': {
'flavorRef': other_flavor_id
}
}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'VERIFY_RESIZE')
self._pop_and_verify_dest_select_notification(server['id'],
replacements={
'ignore_hosts': [],
'flavor.memory_mb': other_flavor_body['flavor']['ram'],
'flavor.name': other_flavor_body['flavor']['name'],
'flavor.flavorid': other_flavor_id,
'flavor.extra_specs': extra_specs['extra_specs'],
'requested_destination': self._build_destination_payload()})
self.assertEqual(7, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
# ignore instance.exists
self.notifier.versioned_notifications.pop(0)
# This list needs to be in order.
expected_notifications = [
'instance-resize_prep-start',
'instance-resize_prep-end',
'instance-resize-start',
'instance-resize-end',
'instance-resize_finish-start',
'instance-resize_finish-end'
]
for idx, notification in enumerate(expected_notifications):
self._verify_notification(
notification,
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[idx])
self.notifier.reset()
# the following is the revert server request
post = {'revertResize': None}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'ACTIVE')
self.assertEqual(3, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
# ignore instance.exists
self.notifier.versioned_notifications.pop(0)
self._verify_notification(
'instance-resize_revert-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-resize_revert-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
@mock.patch('nova.compute.manager.ComputeManager._prep_resize')
def test_resize_server_error_but_reschedule_was_success(
self, mock_prep_resize):
"""Test it, when the prep_resize method raise an exception,
but the reschedule_resize_or_reraise was successful and
scheduled the resize. In this case we get a notification
about the exception, which caused the prep_resize error.
"""
def _build_resources(*args, **kwargs):
raise exception.FlavorDiskTooSmall()
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self.flags(allow_resize_to_same_host=True)
other_flavor_body = {
'flavor': {
'name': 'other_flavor_error',
'ram': 512,
'vcpus': 1,
'disk': 1,
'id': 'a22d5517-147c-4147-a0d1-e698df5cd4e9'
}
}
other_flavor_id = self.api.post_flavor(other_flavor_body)['id']
post = {
'resize': {
'flavorRef': other_flavor_id
}
}
self.notifier.reset()
mock_prep_resize.side_effect = _build_resources
# NOTE(gibi): the first resize_instance call (from the API) should be
# unaffected so that we can reach _prep_resize at all. But the
# subsequent resize_instance call (from _reschedule_resize_or_reraise)
# needs to be mocked as there is no alternative host to resize to.
patcher = mock.patch.object(self.compute.manager.compute_task_api,
'resize_instance')
self.addCleanup(patcher.stop)
patcher.start()
self.api.post_server_action(server['id'], post)
self._wait_for_notification('instance.resize.error')
self._pop_and_verify_dest_select_notification(server['id'],
replacements={
'ignore_hosts': [],
'flavor.name': other_flavor_body['flavor']['name'],
'flavor.flavorid': other_flavor_id,
'flavor.extra_specs': {},
'requested_destination': self._build_destination_payload()})
# 0: instance-exists
# 1: instance-resize_prep-start
# 2: instance-resize-error
# 3: instance-resize_prep-end
self.assertLessEqual(2, len(self.notifier.versioned_notifications),
'Unexpected number of notifications: %s' %
self.notifier.versioned_notifications)
# Note(gibi): There is also an instance.exists notification emitted
# during the rescheduling
tb = self.notifier.versioned_notifications[2]['payload'][
'nova_object.data']['fault']['nova_object.data']['traceback']
self.assertIn("raise exception.FlavorDiskTooSmall()", tb)
self._verify_notification('instance-resize-error',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'fault.traceback': self.ANY
},
actual=self.notifier.versioned_notifications[2])
@mock.patch('nova.compute.manager.ComputeManager._prep_resize')
def test_resize_server_error_and_reschedule_was_failed(
self, mock_prep_resize):
"""Test it, when the prep_resize method raise an exception,
after trying again with the reschedule_resize_or_reraise method
call, but the rescheduled also was unsuccessful. In this
case called the exception block.
In the exception block send a notification about error.
At end called raising an exception based on *exc_info,
which not send another error.
"""
def _build_resources(*args, **kwargs):
raise exception.FlavorDiskTooSmall()
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self.flags(allow_resize_to_same_host=True)
other_flavor_body = {
'flavor': {
'name': 'other_flavor_error',
'ram': 512,
'vcpus': 1,
'disk': 1,
'id': 'a22d5517-147c-4147-a0d1-e698df5cd4e9'
}
}
other_flavor_id = self.api.post_flavor(other_flavor_body)['id']
post = {
'resize': {
'flavorRef': other_flavor_id
}
}
self.notifier.reset()
mock_prep_resize.side_effect = _build_resources
# NOTE(gibi): the first resize_instance call (from the API) should be
# unaffected so that we can reach _prep_resize at all. But the
# subsequent resize_instance call (from _reschedule_resize_or_reraise)
# needs to fail. It isn't realistic that resize_instance would raise
# FlavorDiskTooSmall, but it's needed for the notification sample
# to work.
patcher = mock.patch.object(self.compute.manager.compute_task_api,
'resize_instance',
side_effect=_build_resources)
self.addCleanup(patcher.stop)
patcher.start()
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, expected_status='ERROR')
self._wait_for_notification('compute.exception')
# There should be the following notifications after scheduler's
# select_destination notifications:
# 0: instance-exists
# 1: instance-resize_prep-start
# 2: instance-resize-error
# 3: instance-resize_prep-end
# 4: compute.exception
# (via the wrap_exception decorator on
# the ComputeManager.prep_resize method.)
self._pop_and_verify_dest_select_notification(server['id'],
replacements={
'ignore_hosts': [],
'flavor.name': other_flavor_body['flavor']['name'],
'flavor.flavorid': other_flavor_id,
'flavor.extra_specs': {},
'requested_destination': self._build_destination_payload()})
self.assertEqual(5, len(self.notifier.versioned_notifications),
'Unexpected number of notifications: %s' %
self.notifier.versioned_notifications)
tb = self.notifier.versioned_notifications[2]['payload'][
'nova_object.data']['fault']['nova_object.data']['traceback']
self.assertIn("raise exception.FlavorDiskTooSmall()", tb)
self._verify_notification('instance-resize-error',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'fault.traceback': self.ANY
},
actual=self.notifier.versioned_notifications[2])
def _test_snapshot_server(self, server):
post = {'createImage': {'name': 'test-snap'}}
response = self.api.post_server_action(server['id'], post)
self._wait_for_notification('instance.snapshot.end')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-snapshot-start',
replacements={
'snapshot_image_id': response['image_id'],
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-snapshot-end',
replacements={
'snapshot_image_id': response['image_id'],
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def test_rebuild_server(self):
# NOTE(gabor_antal): Rebuild changes the image used by the instance,
# therefore the actions tested in test_instance_action had to be in
# specific order. To avoid this problem, rebuild was moved from
# test_instance_action to its own method.
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
self.notifier.reset()
image_ref = 'a2459075-d96c-40d5-893e-577ff92e721c'
post = {
'rebuild': {
'imageRef': image_ref,
'metadata': {}
}
}
self.api.post_server_action(server['id'], post)
# Before going back to ACTIVE state
# server state need to be changed to REBUILD state
self._wait_for_state_change(server, expected_status='REBUILD')
self._wait_for_state_change(server, expected_status='ACTIVE')
self._pop_and_verify_dest_select_notification(server['id'],
replacements={
'image.container_format': 'ami',
'image.disk_format': 'ami',
'image.id': image_ref,
'image.properties': {
'nova_object.data': {},
'nova_object.name': 'ImageMetaPropsPayload',
'nova_object.namespace': 'nova',
'nova_object.version': '1.7',
},
'image.size': 58145823,
'image.tags': [],
'scheduler_hints': {'_nova_check_type': ['rebuild']},
'force_hosts': 'compute',
'force_nodes': 'fake-mini',
'requested_destination': self._build_destination_payload()})
# 0. instance.rebuild_scheduled
# 1. instance.exists
# 2. instance.rebuild.start
# 3. instance.detach.start
# 4. instance.detach.end
# 5. instance.rebuild.end
# The compute/manager will detach every volume during rebuild
self.assertEqual(6, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-rebuild_scheduled',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'trusted_image_certificates': None},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-rebuild-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'trusted_image_certificates': None},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-volume_detach-start',
replacements={
'reservation_id': server['reservation_id'],
'task_state': 'rebuilding',
'architecture': None,
'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
self._verify_notification(
'instance-volume_detach-end',
replacements={
'reservation_id': server['reservation_id'],
'task_state': 'rebuilding',
'architecture': None,
'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
'uuid': server['id']},
actual=self.notifier.versioned_notifications[4])
self._verify_notification(
'instance-rebuild-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'trusted_image_certificates': None},
actual=self.notifier.versioned_notifications[5])
def test_rebuild_server_with_trusted_cert(self):
# NOTE(gabor_antal): Rebuild changes the image used by the instance,
# therefore the actions tested in test_instance_action had to be in
# specific order. To avoid this problem, rebuild was moved from
# test_instance_action to its own method.
create_trusted_certs = ['cert-id-1', 'cert-id-2']
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}],
'trusted_image_certificates': create_trusted_certs})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
self.notifier.reset()
image_ref = 'a2459075-d96c-40d5-893e-577ff92e721c'
rebuild_trusted_certs = ['rebuild-cert-id-1', 'rebuild-cert-id-2']
post = {
'rebuild': {
'imageRef': image_ref,
'metadata': {},
'trusted_image_certificates': rebuild_trusted_certs,
}
}
self.api.post_server_action(server['id'], post)
# Before going back to ACTIVE state
# server state need to be changed to REBUILD state
self._wait_for_state_change(server, expected_status='REBUILD')
self._wait_for_state_change(server, expected_status='ACTIVE')
self._pop_and_verify_dest_select_notification(server['id'],
replacements={
'image.container_format': 'ami',
'image.disk_format': 'ami',
'image.id': image_ref,
'image.properties': {
'nova_object.data': {},
'nova_object.name': 'ImageMetaPropsPayload',
'nova_object.namespace': 'nova',
'nova_object.version': '1.7',
},
'image.size': 58145823,
'image.tags': [],
'scheduler_hints': {'_nova_check_type': ['rebuild']},
'force_hosts': 'compute',
'force_nodes': 'fake-mini',
'requested_destination': self._build_destination_payload()})
# 0. instance.rebuild_scheduled
# 1. instance.exists
# 2. instance.rebuild.start
# 3. instance.detach.start
# 4. instance.detach.end
# 5. instance.rebuild.end
# The compute/manager will detach every volume during rebuild
self.assertEqual(6, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-rebuild_scheduled',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-rebuild-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self._verify_notification(
'instance-volume_detach-start',
replacements={
'reservation_id': server['reservation_id'],
'task_state': 'rebuilding',
'architecture': None,
'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
'uuid': server['id']},
actual=self.notifier.versioned_notifications[3])
self._verify_notification(
'instance-volume_detach-end',
replacements={
'reservation_id': server['reservation_id'],
'task_state': 'rebuilding',
'architecture': None,
'image_uuid': 'a2459075-d96c-40d5-893e-577ff92e721c',
'uuid': server['id']},
actual=self.notifier.versioned_notifications[4])
self._verify_notification(
'instance-rebuild-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[5])
@mock.patch('nova.compute.manager.ComputeManager.'
'_do_rebuild_instance_with_claim')
def test_rebuild_server_exc(self, mock_rebuild):
def _virtual_interface_create_failed(*args, **kwargs):
# A real error that could come out of driver.spawn() during rebuild
raise exception.VirtualInterfaceCreateException()
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
self.notifier.reset()
post = {
'rebuild': {
'imageRef': 'a2459075-d96c-40d5-893e-577ff92e721c',
'metadata': {}
}
}
self.api.post_server_action(server['id'], post)
mock_rebuild.side_effect = _virtual_interface_create_failed
self._wait_for_state_change(server, expected_status='ERROR')
notification = self._get_notifications('instance.rebuild.error')
self.assertEqual(1, len(notification),
self.notifier.versioned_notifications)
tb = notification[0]['payload']['nova_object.data']['fault'][
'nova_object.data']['traceback']
self.assertIn('raise exception.VirtualInterfaceCreateException()', tb)
self._verify_notification(
'instance-rebuild-error',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'trusted_image_certificates': None,
'fault.traceback': self.ANY},
actual=notification[0])
def _test_restore_server(self, server):
self.flags(reclaim_instance_interval=30)
self.api.delete_server(server['id'])
self._wait_for_state_change(server, 'SOFT_DELETED')
# we don't want to test soft_delete here
self.notifier.reset()
self.api.post_server_action(server['id'], {'restore': {}})
self._wait_for_state_change(server, 'ACTIVE')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-restore-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-restore-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def _test_reboot_server(self, server):
post = {'reboot': {'type': 'HARD'}}
self.api.post_server_action(server['id'], post)
self._wait_for_notification('instance.reboot.start')
self._wait_for_notification('instance.reboot.end')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-reboot-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-reboot-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
@mock.patch('nova.virt.fake.SmallFakeDriver.reboot')
def _test_reboot_server_error(self, server, mock_reboot):
def _hard_reboot(*args, **kwargs):
raise exception.UnsupportedVirtType(virt="FakeVirt")
mock_reboot.side_effect = _hard_reboot
post = {'reboot': {'type': 'HARD'}}
self.api.post_server_action(server['id'], post)
self._wait_for_notification('instance.reboot.start')
self._wait_for_notification('instance.reboot.error')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
tb = self.notifier.versioned_notifications[1]['payload'][
'nova_object.data']['fault']['nova_object.data']['traceback']
self.assertIn("raise exception.UnsupportedVirtType", tb)
self._verify_notification(
'instance-reboot-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-reboot-error',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'fault.traceback': self.ANY},
actual=self.notifier.versioned_notifications[1])
def _detach_volume_from_server(self, server, volume_id):
self.api.delete_server_volume(server['id'], volume_id)
self._wait_for_notification('instance.volume_detach.end')
def _volume_swap_server(self, server, attachement_id, volume_id):
self.api.put_server_volume(server['id'], attachement_id, volume_id)
def test_volume_swap_server(self):
server = self._boot_a_server(
extra_params={'networks':
[{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
self.cinder.swap_volume_instance_uuid = server['id']
self._volume_swap_server(server, self.cinder.SWAP_OLD_VOL,
self.cinder.SWAP_NEW_VOL)
self._wait_until_swap_volume(server, self.cinder.SWAP_NEW_VOL)
# NOTE(gibi): the new volume id can appear on the API earlier than the
# volume_swap.end notification emitted. So to make the test stable
# we have to wait for the volume_swap.end notification directly.
self._wait_for_notification('instance.volume_swap.end')
self.assertEqual(7, len(self.notifier.versioned_notifications),
'Unexpected number of versioned notifications. '
'Got: %s' % self.notifier.versioned_notifications)
self._verify_notification(
'instance-volume_swap-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[5])
self._verify_notification(
'instance-volume_swap-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[6])
def _do_setup_server_and_error_flag(self):
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_ERR_OLD_VOL)
self.cinder.attachment_error_id = self.cinder.SWAP_ERR_ATTACH_ID
return server
def test_volume_swap_server_with_error(self):
server = self._do_setup_server_and_error_flag()
self._volume_swap_server(server, self.cinder.SWAP_ERR_OLD_VOL,
self.cinder.SWAP_ERR_NEW_VOL)
self._wait_for_notification('compute.exception')
# Eight versioned notifications are generated.
# 0. instance-create-start
# 1. instance-create-end
# 2. instance-update
# 3. instance-volume_attach-start
# 4. instance-volume_attach-end
# 5. instance-volume_swap-start
# 6. instance-volume_swap-error
# 7. compute.exception
self.assertLessEqual(7, len(self.notifier.versioned_notifications),
'Unexpected number of versioned notifications. '
'Got: %s' % self.notifier.versioned_notifications)
block_devices = [{
"nova_object.data": {
"boot_index": None,
"delete_on_termination": False,
"device_name": "/dev/sdb",
"tag": None,
"volume_id": self.cinder.SWAP_ERR_OLD_VOL
},
"nova_object.name": "BlockDevicePayload",
"nova_object.namespace": "nova",
"nova_object.version": "1.0"
}]
self._verify_notification(
'instance-volume_swap-start',
replacements={
'new_volume_id': self.cinder.SWAP_ERR_NEW_VOL,
'old_volume_id': self.cinder.SWAP_ERR_OLD_VOL,
'block_devices': block_devices,
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[5])
tb1 = self.notifier.versioned_notifications[6]['payload'][
'nova_object.data']['fault']['nova_object.data']['traceback']
self.assertIn("_swap_volume", tb1)
tb2 = self.notifier.versioned_notifications[7]['payload'][
'nova_object.data']['traceback']
self.assertIn("_swap_volume", tb2)
self._verify_notification(
'instance-volume_swap-error',
replacements={
'reservation_id': server['reservation_id'],
'block_devices': block_devices,
'uuid': server['id'],
'fault.traceback': self.ANY},
actual=self.notifier.versioned_notifications[6])
def test_resize_confirm_server(self):
server = self._boot_a_server(
extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
self.admin_api.post_extra_spec(
'2', {"extra_specs": {"hw:watchdog_action": "disabled"}})
self.flags(allow_resize_to_same_host=True)
post = {'resize': {'flavorRef': '2'}}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'VERIFY_RESIZE')
self.notifier.reset()
post = {'confirmResize': None}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'ACTIVE')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-resize_confirm-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-resize_confirm-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def _test_trigger_crash_dump(self, server):
post = {'trigger_crash_dump': None}
self.api.post_server_action(server['id'], post)
self._wait_for_notification('instance.trigger_crash_dump.end')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-trigger_crash_dump-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-trigger_crash_dump-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def _test_volume_detach_attach_server(self, server):
self._detach_volume_from_server(server, self.cinder.SWAP_OLD_VOL)
# 0. volume_detach-start
# 1. volume_detach-end
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-volume_detach-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-volume_detach-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self.notifier.reset()
self._attach_volume_to_server(server, self.cinder.SWAP_OLD_VOL)
# 0. volume_attach-start
# 1. volume_attach-end
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-volume_attach-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-volume_attach-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def _test_rescue_unrescue_server(self, server):
# Both "rescue" and "unrescue" notification asserts are made here
# rescue notification asserts
post = {
"rescue": {
"rescue_image_ref": 'a2459075-d96c-40d5-893e-577ff92e721c'
}
}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'RESCUE')
# 0. instance.rescue.start
# 1. instance.exists
# 2. instance.rescue.end
self.assertEqual(3, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-rescue-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-rescue-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[2])
self.notifier.reset()
# unrescue notification asserts
post = {
'unrescue': None
}
self.api.post_server_action(server['id'], post)
self._wait_for_state_change(server, 'ACTIVE')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-unrescue-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-unrescue-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def _test_soft_delete_server(self, server):
self.flags(reclaim_instance_interval=30)
self.api.delete_server(server['id'])
self._wait_for_state_change(server, 'SOFT_DELETED')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-soft_delete-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-soft_delete-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self.flags(reclaim_instance_interval=0)
# Leave instance in normal, active state
self.api.post_server_action(server['id'], {'restore': {}})
def _do_test_attach_volume_error(self, server, mock_attach):
def attach_volume(*args, **kwargs):
raise exception.CinderConnectionFailed(
reason="Connection timed out")
mock_attach.side_effect = attach_volume
post = {"volumeAttachment": {"volumeId": self.cinder.SWAP_NEW_VOL}}
self.api.post_server_volume(server['id'], post)
self._wait_for_notification('instance.volume_attach.error')
block_devices = [
# Add by default at boot
{'nova_object.data': {'boot_index': None,
'delete_on_termination': False,
'tag': None,
'device_name': '/dev/sdb',
'volume_id': self.cinder.SWAP_OLD_VOL},
'nova_object.name': 'BlockDevicePayload',
'nova_object.namespace': 'nova',
'nova_object.version': '1.0'},
# Attaching it right now
{'nova_object.data': {'boot_index': None,
'delete_on_termination': False,
'tag': None,
'device_name': '/dev/sdc',
'volume_id': self.cinder.SWAP_NEW_VOL},
'nova_object.name': 'BlockDevicePayload',
'nova_object.namespace': 'nova',
'nova_object.version': '1.0'}]
# 0. volume_attach-start
# 1. volume_attach-error
# 2. compute.exception
# We only rely on the first 2 notifications, in this case we don't
# care about the exception notification.
self.assertLessEqual(2, len(self.notifier.versioned_notifications))
self._verify_notification(
'instance-volume_attach-start',
replacements={
'reservation_id': server['reservation_id'],
'block_devices': block_devices,
'volume_id': self.cinder.SWAP_NEW_VOL,
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
tb = self.notifier.versioned_notifications[1]['payload'][
'nova_object.data']['fault']['nova_object.data']['traceback']
self.assertIn("CinderConnectionFailed:", tb)
self._verify_notification(
'instance-volume_attach-error',
replacements={
'reservation_id': server['reservation_id'],
'block_devices': block_devices,
'volume_id': self.cinder.SWAP_NEW_VOL,
'uuid': server['id'],
'fault.traceback': self.ANY},
actual=self.notifier.versioned_notifications[1])
@mock.patch('nova.volume.cinder.API.attachment_update')
def _test_attach_volume_error(self, server, mock_attach):
self._do_test_attach_volume_error(server, mock_attach)
def _test_interface_attach_and_detach(self, server):
post = {
'interfaceAttachment': {
'net_id': fixtures.NeutronFixture.network_1['id']
}
}
self.api.attach_interface(server['id'], post)
self._wait_for_notification('instance.interface_attach.end')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-interface_attach-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-interface_attach-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
self.notifier.reset()
self.assertEqual(0, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self.api.detach_interface(
server['id'],
fixtures.NeutronFixture.port_2['id'])
self._wait_for_notification('instance.interface_detach.end')
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-interface_detach-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-interface_detach-end',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
@mock.patch('nova.virt.fake.SmallFakeDriver.attach_interface')
def _test_interface_attach_error(self, server, mock_driver):
def _unsuccessful_attach_interface(*args, **kwargs):
raise exception.InterfaceAttachFailed("dummy")
mock_driver.side_effect = _unsuccessful_attach_interface
post = {
'interfaceAttachment': {
'net_id': fixtures.NeutronFixture.network_1['id']
}
}
self.assertRaises(
client.OpenStackApiException,
self.api.attach_interface, server['id'], post)
self._wait_for_notification('instance.interface_attach.error')
# 0. instance.interface_attach.start
# 1. instance.interface_attach.error
# 2. compute.exception
self.assertLessEqual(2, len(self.notifier.versioned_notifications))
self._verify_notification(
'instance-interface_attach-start',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
tb = self.notifier.versioned_notifications[1]['payload'][
'nova_object.data']['fault']['nova_object.data']['traceback']
self.assertIn("raise exception.InterfaceAttachFailed", tb)
self._verify_notification(
'instance-interface_attach-error',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id'],
'fault.traceback': self.ANY},
actual=self.notifier.versioned_notifications[1])
def _test_lock_unlock_instance(self, server):
self.api.post_server_action(server['id'], {'lock': {}})
self._wait_for_server_parameter(server, {'locked': True})
self.api.post_server_action(server['id'], {'unlock': {}})
self._wait_for_server_parameter(server, {'locked': False})
# Two versioned notifications are generated
# 0. instance-lock
# 1. instance-unlock
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-lock',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-unlock',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])
def _test_lock_unlock_instance_with_reason(self, server):
self.api.post_server_action(
server['id'], {'lock': {"locked_reason": "global warming"}})
self._wait_for_server_parameter(server, {'locked': True})
self.api.post_server_action(server['id'], {'unlock': {}})
self._wait_for_server_parameter(server, {'locked': False})
# Two versioned notifications are generated
# 0. instance-lock
# 1. instance-unlock
self.assertEqual(2, len(self.notifier.versioned_notifications),
self.notifier.versioned_notifications)
self._verify_notification(
'instance-lock-with-reason',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[0])
self._verify_notification(
'instance-unlock',
replacements={
'reservation_id': server['reservation_id'],
'uuid': server['id']},
actual=self.notifier.versioned_notifications[1])