Shelve/unshelve an instance

This adds the compute methods for shelving and unshelving.  Shelving
will power down the instance, and snapshot if not volume backed.
Unshelving will restore the instance, which may involve re-scheduling
and rebuilding it from a snapshot.

Part of bp shelve-instance

Co-Author: Dan Smith <danms@us.ibm.com> (Instance objects)
Change-Id: I908474915c5325e120a64af9b9af13fcbce1f01c

etc/nova/policy.json

@@ -12,6 +12,9 @@
     "compute:get_all": "",
     "compute:get_all_tenants": "",
 
+    "compute:shelve": "",
+    "compute:shelve_offload": "",
+    "compute:unshelve": "",
 
     "admin_api": "is_admin:True",
     "compute_extension:accounts": "rule:admin_api",

nova/compute/api.py

@@ -2237,6 +2237,58 @@ class API(base.Base):
 
         self.scheduler_rpcapi.prep_resize(context, **args)
 
+    @wrap_check_policy
+    @check_instance_lock
+    @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED,
+                                    vm_states.PAUSED, vm_states.SUSPENDED],
+                          task_state=[None])
+    def shelve(self, context, instance):
+        """Shelve an instance.
+
+        Shuts down an instance and frees it up to be removed from the
+        hypervisor.
+        """
+        instance.task_state = task_states.SHELVING
+        instance.save(expected_task_state=None)
+
+        self._record_action_start(context, instance, instance_actions.SHELVE)
+
+        image_id = None
+        bdms = self.get_instance_bdms(context, instance)
+        if not self.is_volume_backed_instance(context, instance, bdms):
+            name = '%s-shelved' % instance['name']
+            image_meta = self._create_image(context, instance, name,
+                    'snapshot')
+            image_id = image_meta['id']
+            self.compute_rpcapi.shelve_instance(context, instance=instance,
+                    image_id=image_id)
+        else:
+            self.compute_rpcapi.shelve_offload_instance(context,
+                    instance=instance)
+
+    @wrap_check_policy
+    @check_instance_lock
+    @check_instance_state(vm_state=[vm_states.SHELVED], task_state=[None])
+    def shelve_offload(self, context, instance):
+        """Remove a shelved instance from the hypervisor."""
+        instance.task_state = task_states.SHELVING_OFFLOADING
+        instance.save(expected_task_state=None)
+
+        self.compute_rpcapi.shelve_offload_instance(context, instance=instance)
+
+    @wrap_check_policy
+    @check_instance_lock
+    @check_instance_state(vm_state=[vm_states.SHELVED,
+        vm_states.SHELVED_OFFLOADED], task_state=[None])
+    def unshelve(self, context, instance):
+        """Restore a shelved instance."""
+        instance.task_state = task_states.UNSHELVING
+        instance.save(expected_task_state=None)
+
+        self._record_action_start(context, instance, instance_actions.UNSHELVE)
+
+        self.compute_task_api.unshelve_instance(context, instance)
+
     @wrap_check_policy
     @check_instance_lock
     def add_fixed_ip(self, context, instance, network_id):

nova/compute/cells_api.py

@@ -408,6 +408,26 @@ class ComputeCellsAPI(compute_api.API):
         super(ComputeCellsAPI, self).unrescue(context, instance)
         self._cast_to_cells(context, instance, 'unrescue')
 
+    @wrap_check_policy
+    @check_instance_cell
+    def shelve(self, context, instance):
+        """Shelve the given instance."""
+        self._cast_to_cells(context, instance, 'shelve')
+
+    @wrap_check_policy
+    @check_instance_cell
+    def shelve_offload(self, context, instance):
+        """Offload the shelved instance."""
+        super(ComputeCellsAPI, self).shelve_offload(context, instance)
+        self._cast_to_cells(context, instance, 'shelve_offload')
+
+    @wrap_check_policy
+    @check_instance_cell
+    def unshelve(self, context, instance):
+        """Unshelve the given instance."""
+        super(ComputeCellsAPI, self).unshelve(context, instance)
+        self._cast_to_cells(context, instance, 'unshelve')
+
     @check_instance_cell
     def set_admin_password(self, context, instance, password=None):
         """Set the root/admin password for the given instance."""

nova/compute/instance_actions.py

@@ -43,3 +43,5 @@ RESUME = 'resume'
 RESCUE = 'rescue'
 UNRESCUE = 'unrescue'
 CHANGE_PASSWORD = 'changePassword'
+SHELVE = 'shelve'
+UNSHELVE = 'unshelve'

nova/compute/manager.py

@@ -354,7 +354,7 @@ class ComputeVirtAPI(virtapi.VirtAPI):
 class ComputeManager(manager.SchedulerDependentManager):
     """Manages the running instances from creation to destruction."""
 
-    RPC_API_VERSION = '2.30'
+    RPC_API_VERSION = '2.31'
 
     def __init__(self, compute_driver=None, *args, **kwargs):
         """Load configuration options and connect to the hypervisor."""
@@ -3041,6 +3041,151 @@ class ComputeManager(manager.SchedulerDependentManager):
 
         self._notify_about_instance_usage(context, instance, 'resume')
 
+    @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
+    @reverts_task_state
+    @wrap_instance_event
+    @wrap_instance_fault
+    def shelve_instance(self, context, instance, image_id):
+        """Shelve an instance.
+
+        This should be used when you want to take a snapshot of the instance.
+        It also adds system_metadata that can be used by a periodic task to
+        offload the shelved instance after a period of time.
+
+        :param context: request context
+        :param instance: an Instance object
+        :param image_id: an image id to snapshot to.
+        """
+        self.conductor_api.notify_usage_exists(context, instance,
+                current_period=True)
+        self._notify_about_instance_usage(context, instance, 'shelve.start')
+
+        def update_task_state(task_state, expected_state=task_states.SHELVING):
+            shelving_state_map = {
+                    task_states.IMAGE_PENDING_UPLOAD:
+                        task_states.SHELVING_IMAGE_PENDING_UPLOAD,
+                    task_states.IMAGE_UPLOADING:
+                        task_states.SHELVING_IMAGE_UPLOADING,
+                    task_states.SHELVING: task_states.SHELVING}
+            task_state = shelving_state_map[task_state]
+            expected_state = shelving_state_map[expected_state]
+            instance.task_state = task_state
+            instance.save(expected_task_state=expected_state)
+
+        self.driver.power_off(instance)
+        current_power_state = self._get_power_state(context, instance)
+        self.driver.snapshot(context, instance, image_id, update_task_state)
+
+        sys_meta = instance.system_metadata
+        sys_meta['shelved_at'] = timeutils.strtime()
+        sys_meta['shelved_image_id'] = image_id
+        sys_meta['shelved_host'] = self.host
+        instance.system_metadata = sys_meta
+        instance.vm_state = vm_states.SHELVED
+        instance.task_state = None
+        instance.power_state = current_power_state
+        instance.save(expected_task_state=[
+                task_states.SHELVING,
+                task_states.SHELVING_IMAGE_UPLOADING])
+        self._notify_about_instance_usage(context, instance, 'shelve.end')
+
+    @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
+    @reverts_task_state
+    @wrap_instance_fault
+    def shelve_offload_instance(self, context, instance):
+        """Remove a shelved instance from the hypervisor.
+
+        This frees up those resources for use by other instances, but may lead
+        to slower unshelve times for this instance.  This method is used by
+        volume backed instances since restoring them doesn't involve the
+        potentially large download of an image.
+
+        :param context: request context
+        :param instance: an Instance dict
+        """
+        self._notify_about_instance_usage(context, instance,
+                'shelve_offload.start')
+
+        self.driver.power_off(instance)
+        current_power_state = self._get_power_state(context, instance)
+
+        network_info = self._get_instance_nw_info(context, instance)
+        block_device_info = self._get_instance_volume_block_device_info(
+                context, instance)
+        self.driver.destroy(instance, self._legacy_nw_info(network_info),
+                block_device_info)
+
+        instance.power_state = current_power_state
+        instance.host = None
+        instance.node = None
+        instance.vm_state = vm_states.SHELVED_OFFLOADED
+        instance.task_state = None
+        instance.save(expected_task_state=[task_states.SHELVING,
+                                           task_states.SHELVING_OFFLOADING])
+        self._notify_about_instance_usage(context, instance,
+                'shelve_offload.end')
+
+    @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
+    @reverts_task_state
+    @wrap_instance_event
+    @wrap_instance_fault
+    def unshelve_instance(self, context, instance, image=None):
+        """Unshelve the instance.
+
+        :param context: request context
+        :param instance: an Instance dict
+        :param image: an image to build from.  If None we assume a
+            volume backed instance.
+        """
+        @utils.synchronized(instance['uuid'])
+        def do_unshelve_instance():
+            self._unshelve_instance(context, instance, image)
+        do_unshelve_instance()
+
+    def _unshelve_instance_key_scrub(self, instance):
+        """Remove data from the instance that may cause side effects."""
+        cleaned_keys = dict(
+                key_data=instance.key_data,
+                auto_disk_config=instance.auto_disk_config)
+        instance.key_data = None
+        instance.auto_disk_config = False
+        return cleaned_keys
+
+    def _unshelve_instance_key_restore(self, instance, keys):
+        """Restore previously scrubbed keys before saving the instance."""
+        instance.update(keys)
+
+    def _unshelve_instance(self, context, instance, image):
+        self._notify_about_instance_usage(context, instance, 'unshelve.start')
+        instance.task_state = task_states.SPAWNING
+        instance.save()
+
+        network_info = self._get_instance_nw_info(context, instance)
+        bdms = self.conductor_api.block_device_mapping_get_all_by_instance(
+                context, instance)
+        block_device_info = self._prep_block_device(context, instance, bdms)
+        scrubbed_keys = self._unshelve_instance_key_scrub(instance)
+        try:
+            self.driver.spawn(context, instance, image, injected_files=[],
+                    admin_password=None,
+                    network_info=self._legacy_nw_info(network_info),
+                    block_device_info=block_device_info)
+        except Exception:
+            with excutils.save_and_reraise_exception():
+                LOG.exception(_('Instance failed to spawn'), instance=instance)
+
+        if image:
+            image_service = glance.get_default_image_service()
+            image_service.delete(context, image['id'])
+
+        self._unshelve_instance_key_restore(instance, scrubbed_keys)
+        instance.power_state = self._get_power_state(context, instance)
+        instance.vm_state = vm_states.ACTIVE
+        instance.task_state = None
+        instance.launched_at = timeutils.utcnow()
+        instance.save(expected_task_state=task_states.SPAWNING)
+        self._notify_about_instance_usage(context, instance, 'unshelve.end')
+
     @reverts_task_state
     @wrap_instance_fault
     def reset_network(self, context, instance):

nova/compute/rpcapi.py

@@ -181,6 +181,8 @@ class ComputeAPI(nova.openstack.common.rpc.proxy.RpcProxy):
         2.29 - Made start_instance() and stop_instance() take new-world
                instance objects
         2.30 - Adds live_snapshot_instance()
+        2.31 - Adds shelve_instance(), shelve_offload_instance, and
+               unshelve_instance()
     '''
 
     #
@@ -667,6 +669,24 @@ class ComputeAPI(nova.openstack.common.rpc.proxy.RpcProxy):
                 instance=instance_p),
                 topic=_compute_topic(self.topic, ctxt, None, instance))
 
+    def shelve_instance(self, ctxt, instance, image_id=None):
+        self.cast(ctxt, self.make_msg('shelve_instance',
+            instance=instance, image_id=image_id),
+            topic=_compute_topic(self.topic, ctxt, None, instance),
+            version='2.31')
+
+    def shelve_offload_instance(self, ctxt, instance):
+        self.cast(ctxt, self.make_msg('shelve_offload_instance',
+            instance=instance),
+            topic=_compute_topic(self.topic, ctxt, None, instance),
+            version='2.31')
+
+    def unshelve_instance(self, ctxt, instance, host, image=None):
+        self.cast(ctxt, self.make_msg('unshelve_instance',
+            instance=instance, image=image),
+            topic=_compute_topic(self.topic, ctxt, host, None),
+            version='2.31')
+
 
 class SecurityGroupAPI(nova.openstack.common.rpc.proxy.RpcProxy):
     '''Client side of the security group rpc API.

nova/compute/task_states.py

@@ -109,3 +109,14 @@ SOFT_DELETING = 'soft-deleting'
 
 # possible task states during restore()
 RESTORING = 'restoring'
+
+# possible task states during shelve()
+SHELVING = 'shelving'
+SHELVING_IMAGE_PENDING_UPLOAD = 'shelving_image_pending_upload'
+SHELVING_IMAGE_UPLOADING = 'shelving_image_uploading'
+
+# possible task states during shelve_offload()
+SHELVING_OFFLOADING = 'shelving_offloading'
+
+# possible task states during unshelve()
+UNSHELVING = 'unshelving'

nova/compute/vm_states.py

@@ -44,3 +44,7 @@ SOFT_DELETED = 'soft-delete'  # VM is marked as deleted but the disk images are
 DELETED = 'deleted'  # VM is permanently deleted.
 
 ERROR = 'error'
+
+SHELVED = 'shelved'  # VM is powered off, resources still on hypervisor
+SHELVED_OFFLOADED = 'shelved_offloaded'  # VM and associated resources are
+# not on hypervisor

nova/conductor/api.py

@@ -366,6 +366,10 @@ class LocalComputeTaskAPI(object):
                 security_groups=security_groups,
                 block_device_mapping=block_device_mapping)
 
+    def unshelve_instance(self, context, instance):
+        utils.spawn_n(self._manager.unshelve_instance, context,
+                instance=instance)
+
 
 class API(LocalAPI):
     """Conductor API that does updates via RPC to the ConductorManager."""
@@ -432,3 +436,7 @@ class ComputeTaskAPI(object):
                 requested_networks=requested_networks,
                 security_groups=security_groups,
                 block_device_mapping=block_device_mapping)
+
+    def unshelve_instance(self, context, instance):
+        self.conductor_compute_rpcapi.unshelve_instance(context,
+                instance=instance)

nova/conductor/manager.py

@@ -20,14 +20,19 @@ from nova.api.ec2 import ec2utils
 from nova import block_device
 from nova.cells import rpcapi as cells_rpcapi
 from nova.compute import api as compute_api
+from nova.compute import rpcapi as compute_rpcapi
+from nova.compute import task_states
 from nova.compute import utils as compute_utils
+from nova.compute import vm_states
 from nova.db import base
 from nova import exception
+from nova.image import glance
 from nova import manager
 from nova import network
 from nova.network.security_group import openstack_driver
 from nova import notifications
 from nova.objects import base as nova_object
+from nova.openstack.common import excutils
 from nova.openstack.common import jsonutils
 from nova.openstack.common import log as logging
 from nova.openstack.common.notifier import api as notifier
@@ -558,10 +563,11 @@ class ComputeTaskManager(base.Base):
     """
 
     RPC_API_NAMESPACE = 'compute_task'
-    RPC_API_VERSION = '1.2'
+    RPC_API_VERSION = '1.3'
 
     def __init__(self):
         super(ComputeTaskManager, self).__init__()
+        self.compute_rpcapi = compute_rpcapi.ComputeAPI()
         self.scheduler_rpcapi = scheduler_rpcapi.SchedulerAPI()
 
     @rpc_common.client_exceptions(exception.NoValidHost,
@@ -593,3 +599,71 @@ class ComputeTaskManager(base.Base):
                 admin_password=admin_password, injected_files=injected_files,
                 requested_networks=requested_networks, is_first_time=True,
                 filter_properties=filter_properties)
+
+    def _instance_update(self, context, instance_uuid, **kwargs):
+        (old_ref, instance_ref) = self.db.instance_update_and_get_original(
+                context, instance_uuid, kwargs)
+        notifications.send_update(context, old_ref, instance_ref, 'conductor')
+        return instance_ref
+
+    def _get_image(self, context, image_id):
+        if not image_id:
+            return None
+
+        (image_service, image_id) = glance.get_remote_image_service(context,
+                image_id)
+        return image_service.show(context, image_id)
+
+    def _delete_image(self, context, image_id):
+        (image_service, image_id) = glance.get_remote_image_service(context,
+                image_id)
+        return image_service.delete(context, image_id)
+
+    def _schedule_instances(self, context, image, filter_properties,
+            *instances):
+        request_spec = scheduler_utils.build_request_spec(context, image,
+                instances)
+        # dict(host='', nodename='', limits='')
+        hosts = self.scheduler_rpcapi.select_destinations(context,
+                request_spec, filter_properties)
+        return hosts
+
+    def unshelve_instance(self, context, instance):
+        sys_meta = instance.system_metadata
+
+        if instance.vm_state == vm_states.SHELVED:
+            instance.task_state = task_states.POWERING_ON
+            instance.save(expected_task_state=task_states.UNSHELVING)
+            self.compute_rpcapi.start_instance(context, instance)
+            snapshot_id = sys_meta.get('shelved_image_id')
+            if snapshot_id:
+                self._delete_image(context, snapshot_id)
+        elif instance.vm_state == vm_states.SHELVED_OFFLOADED:
+            try:
+                with compute_utils.EventReporter(context, self.db,
+                        'get_image_info', instance.uuid):
+                    image = self._get_image(context,
+                            sys_meta['shelved_image_id'])
+            except exception.ImageNotFound:
+                with excutils.save_and_reraise_exception():
+                    LOG.error(_('Unshelve attempted but vm_state not SHELVED '
+                                'or SHELVED_OFFLOADED'), instance=instance)
+                    instance.vm_state = vm_states.ERROR
+                    instance.save()
+
+            hosts = self._schedule_instances(context, image, [], instance)
+            host = hosts.pop(0)['host']
+            self.compute_rpcapi.unshelve_instance(context, instance, host,
+                    image)
+        else:
+            LOG.error(_('Unshelve attempted but vm_state not SHELVED or '
+                        'SHELVED_OFFLOADED'), instance=instance)
+            instance.vm_state = vm_states.ERROR
+            instance.save()
+            return
+
+        for key in ['shelved_at', 'shelved_image_id', 'shelved_host']:
+            if key in sys_meta:
+                del(sys_meta[key])
+        instance.system_metadata = sys_meta
+        instance.save()

nova/conductor/rpcapi.py

@@ -526,6 +526,7 @@ class ComputeTaskAPI(nova.openstack.common.rpc.proxy.RpcProxy):
     1.0 - Initial version (empty).
     1.1 - Added unified migrate_server call.
     1.2 - Added build_instances
+    1.3 - Added unshelve_instance
     """
 
     BASE_RPC_API_VERSION = '1.0'
@@ -534,7 +535,8 @@ class ComputeTaskAPI(nova.openstack.common.rpc.proxy.RpcProxy):
     def __init__(self):
         super(ComputeTaskAPI, self).__init__(
                 topic=CONF.conductor.topic,
-                default_version=self.BASE_RPC_API_VERSION)
+                default_version=self.BASE_RPC_API_VERSION,
+                serializer=objects_base.NovaObjectSerializer())
 
     def migrate_server(self, context, instance, scheduler_hint, live, rebuild,
                   flavor, block_migration, disk_over_commit):
@@ -558,3 +560,7 @@ class ComputeTaskAPI(nova.openstack.common.rpc.proxy.RpcProxy):
                 security_groups=security_groups,
                 block_device_mapping=block_device_mapping)
         self.cast(context, msg, version='1.2')
+
+    def unshelve_instance(self, context, instance):
+        msg = self.make_msg('unshelve_instance', instance=instance)
+        self.cast(context, msg, version='1.3')

nova/scheduler/filter_scheduler.py

@@ -175,7 +175,7 @@ class FilterScheduler(driver.Scheduler):
             raise exception.NoValidHost(reason='')
 
         dests = [dict(host=host.obj.host, nodename=host.obj.nodename,
-                      limists=host.obj.limits) for host in selected_hosts]
+                      limits=host.obj.limits) for host in selected_hosts]
         return dests
 
     def _provision_resource(self, context, weighed_host, request_spec,

nova/tests/compute/test_shelve.py

@@ -0,0 +1,291 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import iso8601
+import mox
+
+from nova.compute import task_states
+from nova.compute import vm_states
+from nova import db
+from nova.objects import instance as instance_obj
+from nova.openstack.common import jsonutils
+from nova.openstack.common import timeutils
+from nova.tests.compute import test_compute
+from nova.tests.image import fake as fake_image
+from nova import utils
+
+
+class ShelveComputeManagerTestCase(test_compute.BaseTestCase):
+    def test_shelve(self):
+        db_instance = jsonutils.to_primitive(self._create_fake_instance())
+        self.compute.run_instance(self.context, instance=db_instance)
+        instance = instance_obj.Instance.get_by_uuid(
+            self.context, db_instance['uuid'],
+            expected_attrs=['metadata', 'system_metadata'])
+        image_id = 'fake_image_id'
+        host = 'fake-mini'
+        cur_time = timeutils.utcnow()
+        timeutils.set_time_override(cur_time)
+        instance.task_state = task_states.SHELVING
+        instance.save()
+        sys_meta = instance.system_metadata
+        sys_meta['shelved_at'] = timeutils.strtime(at=cur_time)
+        sys_meta['shelved_image_id'] = image_id
+        sys_meta['shelved_host'] = host
+        db_instance['system_metadata'] = utils.dict_to_metadata(sys_meta)
+
+        self.mox.StubOutWithMock(self.compute, '_notify_about_instance_usage')
+        self.mox.StubOutWithMock(self.compute.driver, 'snapshot')
+        self.mox.StubOutWithMock(self.compute.driver, 'power_off')
+        self.mox.StubOutWithMock(self.compute, '_get_power_state')
+        self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
+
+        self.compute._notify_about_instance_usage(self.context, instance,
+                'shelve.start')
+        self.compute.driver.power_off(instance)
+        self.compute._get_power_state(self.context,
+                instance).AndReturn(123)
+        self.compute.driver.snapshot(self.context, instance, 'fake_image_id',
+                mox.IgnoreArg())
+
+        db.instance_update_and_get_original(self.context, instance['uuid'],
+                {'power_state': 123,
+                 'vm_state': vm_states.SHELVED,
+                 'task_state': None,
+                 'expected_task_state': [task_states.SHELVING,
+                    task_states.SHELVING_IMAGE_UPLOADING],
+                 'system_metadata': sys_meta}).AndReturn((db_instance,
+                                                          db_instance))
+        self.compute._notify_about_instance_usage(self.context,
+                                                  instance, 'shelve.end')
+        self.mox.ReplayAll()
+
+        self.compute.shelve_instance(self.context, instance,
+                image_id=image_id)
+
+        self.mox.VerifyAll()
+        self.mox.UnsetStubs()
+
+        self.compute.terminate_instance(self.context, instance=instance)
+
+    def test_shelve_volume_backed(self):
+        db_instance = jsonutils.to_primitive(self._create_fake_instance())
+        self.compute.run_instance(self.context, instance=db_instance)
+        instance = instance_obj.Instance.get_by_uuid(
+            self.context, db_instance['uuid'],
+            expected_attrs=['metadata', 'system_metadata'])
+        instance.task_state = task_states.SHELVING
+        instance.save()
+        host = 'fake-mini'
+        cur_time = timeutils.utcnow()
+        timeutils.set_time_override(cur_time)
+        sys_meta = instance.system_metadata
+        sys_meta['shelved_at'] = timeutils.strtime(at=cur_time)
+        sys_meta['shelved_image_id'] = None
+        sys_meta['shelved_host'] = host
+        db_instance['system_metadata'] = utils.dict_to_metadata(sys_meta)
+
+        self.mox.StubOutWithMock(self.compute, '_notify_about_instance_usage')
+        self.mox.StubOutWithMock(self.compute.driver, 'power_off')
+        self.mox.StubOutWithMock(self.compute, '_get_power_state')
+        self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
+
+        self.compute._notify_about_instance_usage(self.context, instance,
+                'shelve_offload.start')
+        self.compute.driver.power_off(instance)
+        self.compute._get_power_state(self.context,
+                instance).AndReturn(123)
+        db.instance_update_and_get_original(self.context, instance['uuid'],
+                {'power_state': 123, 'host': None, 'node': None,
+                 'vm_state': vm_states.SHELVED_OFFLOADED,
+                 'task_state': None,
+                 'expected_task_state': [task_states.SHELVING,
+                    task_states.SHELVING_OFFLOADING]}).AndReturn(
+                            (db_instance, db_instance))
+        self.compute._notify_about_instance_usage(self.context, instance,
+                'shelve_offload.end')
+        self.mox.ReplayAll()
+
+        self.compute.shelve_offload_instance(self.context, instance)
+
+        self.mox.VerifyAll()
+        self.mox.UnsetStubs()
+
+        self.compute.terminate_instance(self.context, instance=instance)
+
+    def test_unshelve(self):
+        db_instance = jsonutils.to_primitive(self._create_fake_instance())
+        self.compute.run_instance(self.context, instance=db_instance)
+        instance = instance_obj.Instance.get_by_uuid(
+            self.context, db_instance['uuid'],
+            expected_attrs=['metadata', 'system_metadata'])
+        instance.task_state = task_states.UNSHELVING
+        instance.save()
+        image = {'id': 'fake_id'}
+        host = 'fake-mini'
+        cur_time = timeutils.utcnow()
+        cur_time_tz = cur_time.replace(tzinfo=iso8601.iso8601.Utc())
+        timeutils.set_time_override(cur_time)
+        sys_meta = instance.system_metadata
+        sys_meta['shelved_at'] = timeutils.strtime(at=cur_time)
+        sys_meta['shelved_image_id'] = image['id']
+        sys_meta['shelved_host'] = host
+
+        self.mox.StubOutWithMock(self.compute, '_notify_about_instance_usage')
+        self.mox.StubOutWithMock(self.compute, '_prep_block_device')
+        self.mox.StubOutWithMock(self.compute.driver, 'spawn')
+        self.mox.StubOutWithMock(self.compute, '_get_power_state')
+        self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
+
+        self.deleted_image_id = None
+
+        def fake_delete(self2, ctxt, image_id):
+            self.deleted_image_id = image_id
+
+        fake_image.stub_out_image_service(self.stubs)
+        self.stubs.Set(fake_image._FakeImageService, 'delete', fake_delete)
+
+        self.compute._notify_about_instance_usage(self.context, instance,
+                'unshelve.start')
+        db.instance_update_and_get_original(self.context, instance['uuid'],
+                {'task_state': task_states.SPAWNING}).AndReturn(
+                        (db_instance, db_instance))
+        self.compute._prep_block_device(self.context, instance,
+                []).AndReturn('fake_bdm')
+        db_instance['key_data'] = None
+        db_instance['auto_disk_config'] = None
+        self.compute.driver.spawn(self.context, instance, image,
+                injected_files=[], admin_password=None,
+                network_info=[],
+                block_device_info='fake_bdm')
+        self.compute._get_power_state(self.context, instance).AndReturn(123)
+        db.instance_update_and_get_original(self.context, instance['uuid'],
+                {'power_state': 123,
+                 'vm_state': vm_states.ACTIVE,
+                 'task_state': None,
+                 'key_data': None,
+                 'auto_disk_config': False,
+                 'expected_task_state': task_states.SPAWNING,
+                 'launched_at': cur_time_tz}).AndReturn((db_instance,
+                                                         db_instance))
+        self.compute._notify_about_instance_usage(self.context, instance,
+                'unshelve.end')
+        self.mox.ReplayAll()
+
+        self.compute.unshelve_instance(self.context, instance,
+                image=image)
+        self.assertEqual(image['id'], self.deleted_image_id)
+
+        self.mox.VerifyAll()
+        self.mox.UnsetStubs()
+
+        self.compute.terminate_instance(self.context, instance=instance)
+
+    def test_unshelve_volume_backed(self):
+        db_instance = jsonutils.to_primitive(self._create_fake_instance())
+        host = 'fake-mini'
+        cur_time = timeutils.utcnow()
+        cur_time_tz = cur_time.replace(tzinfo=iso8601.iso8601.Utc())
+        timeutils.set_time_override(cur_time)
+        self.compute.run_instance(self.context, instance=db_instance)
+        instance = instance_obj.Instance.get_by_uuid(
+            self.context, db_instance['uuid'],
+            expected_attrs=['metadata', 'system_metadata'])
+        instance.task_state = task_states.UNSHELVING
+        instance.save()
+        sys_meta = instance.system_metadata
+        sys_meta['shelved_at'] = timeutils.strtime(at=cur_time)
+        sys_meta['shelved_image_id'] = None
+        sys_meta['shelved_host'] = host
+
+        self.mox.StubOutWithMock(self.compute, '_notify_about_instance_usage')
+        self.mox.StubOutWithMock(self.compute, '_prep_block_device')
+        self.mox.StubOutWithMock(self.compute.driver, 'spawn')
+        self.mox.StubOutWithMock(self.compute, '_get_power_state')
+        self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
+
+        self.compute._notify_about_instance_usage(self.context, instance,
+                'unshelve.start')
+        db.instance_update_and_get_original(self.context, instance['uuid'],
+                {'task_state': task_states.SPAWNING}).AndReturn(
+                        (db_instance, db_instance))
+        self.compute._prep_block_device(self.context, instance,
+                []).AndReturn('fake_bdm')
+        db_instance['key_data'] = None
+        db_instance['auto_disk_config'] = None
+        self.compute.driver.spawn(self.context, instance, None,
+                injected_files=[], admin_password=None,
+                network_info=[],
+                block_device_info='fake_bdm')
+        self.compute._get_power_state(self.context, instance).AndReturn(123)
+        db.instance_update_and_get_original(self.context, instance['uuid'],
+                {'power_state': 123,
+                 'vm_state': vm_states.ACTIVE,
+                 'task_state': None,
+                 'key_data': None,
+                 'auto_disk_config': False,
+                 'expected_task_state': task_states.SPAWNING,
+                 'launched_at': cur_time_tz}).AndReturn((db_instance,
+                                                         db_instance))
+        self.compute._notify_about_instance_usage(self.context, instance,
+                'unshelve.end')
+        self.mox.ReplayAll()
+
+        self.compute.unshelve_instance(self.context, instance, image=None)
+
+        self.mox.VerifyAll()
+        self.mox.UnsetStubs()
+
+        self.compute.terminate_instance(self.context, instance=instance)
+
+
+class ShelveComputeAPITestCase(test_compute.BaseTestCase):
+    def test_shelve(self):
+        # Ensure instance can be shelved.
+        instance = jsonutils.to_primitive(self._create_fake_instance())
+        instance_uuid = instance['uuid']
+        self.compute.run_instance(self.context, instance=instance)
+
+        self.assertEqual(instance['task_state'], None)
+
+        inst_obj = instance_obj.Instance.get_by_uuid(self.context,
+                                                     instance_uuid)
+        self.compute_api.shelve(self.context, inst_obj)
+
+        inst_obj.refresh()
+        self.assertEqual(inst_obj.task_state, task_states.SHELVING)
+
+        db.instance_destroy(self.context, instance['uuid'])
+
+    def test_unshelve(self):
+        # Ensure instance can be unshelved.
+        instance = jsonutils.to_primitive(self._create_fake_instance())
+        instance_uuid = instance['uuid']
+        self.compute.run_instance(self.context, instance=instance)
+
+        self.assertEqual(instance['task_state'], None)
+
+        inst_obj = instance_obj.Instance.get_by_uuid(self.context,
+                                                     instance_uuid)
+        self.compute_api.shelve(self.context, inst_obj)
+
+        inst_obj.refresh()
+        inst_obj.task_state = None
+        inst_obj.vm_state = vm_states.SHELVED
+        inst_obj.save()
+
+        self.compute_api.unshelve(self.context, inst_obj)
+
+        inst_obj.refresh()
+        self.assertEqual(inst_obj.task_state, task_states.UNSHELVING)
+
+        db.instance_destroy(self.context, instance['uuid'])

nova/tests/conductor/test_conductor.py

@@ -18,6 +18,7 @@ import mox
 
 from nova.api.ec2 import ec2utils
 from nova.compute import flavors
+from nova.compute import task_states
 from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova import conductor
@@ -29,6 +30,7 @@ from nova import db
 from nova.db.sqlalchemy import models
 from nova import exception as exc
 from nova import notifications
+from nova.objects import instance as instance_obj
 from nova.openstack.common import jsonutils
 from nova.openstack.common.notifier import api as notifier_api
 from nova.openstack.common.notifier import test_notifier
@@ -36,6 +38,8 @@ from nova.openstack.common.rpc import common as rpc_common
 from nova.openstack.common import timeutils
 from nova import quota
 from nova import test
+from nova.tests.compute import test_compute
+from nova.tests import fake_instance_actions
 
 
 FAKE_IMAGE_REF = 'fake-image-ref'
@@ -1208,6 +1212,7 @@ class _BaseTaskTestCase(object):
         self.user_id = 'fake'
         self.project_id = 'fake'
         self.context = FakeContext(self.user_id, self.project_id)
+        fake_instance_actions.stub_out_action_events(self.stubs)
 
     def test_migrate_server(self):
         self.mox.StubOutWithMock(self.conductor_manager.scheduler_rpcapi,
@@ -1277,8 +1282,88 @@ class _BaseTaskTestCase(object):
                 security_groups='security_groups',
                 block_device_mapping='block_device_mapping')
 
+    def test_unshelve_instance_on_host(self):
+        db_instance = jsonutils.to_primitive(self._create_fake_instance())
+        instance = instance_obj.Instance.get_by_uuid(self.context,
+                db_instance['uuid'], expected_attrs=['system_metadata'])
+        instance.vm_state = vm_states.SHELVED
+        instance.task_state = task_states.UNSHELVING
+        instance.save()
+        system_metadata = instance.system_metadata
 
-class ConductorTaskTestCase(_BaseTaskTestCase, test.TestCase):
+        self.mox.StubOutWithMock(self.conductor_manager.compute_rpcapi,
+                'start_instance')
+        self.mox.StubOutWithMock(self.conductor_manager, '_delete_image')
+        self.mox.StubOutWithMock(self.conductor_manager.compute_rpcapi,
+                'unshelve_instance')
+
+        self.conductor_manager.compute_rpcapi.start_instance(self.context,
+                instance)
+        self.conductor_manager._delete_image(self.context,
+                'fake_image_id')
+        self.mox.ReplayAll()
+
+        system_metadata['shelved_at'] = timeutils.utcnow()
+        system_metadata['shelved_image_id'] = 'fake_image_id'
+        system_metadata['shelved_host'] = 'fake-mini'
+        self.conductor_manager.unshelve_instance(self.context, instance)
+
+    def test_unshelve_instance_schedule_and_rebuild(self):
+        db_instance = jsonutils.to_primitive(self._create_fake_instance())
+        instance = instance_obj.Instance.get_by_uuid(self.context,
+                db_instance['uuid'], expected_attrs=['system_metadata'])
+        instance.vm_state = vm_states.SHELVED_OFFLOADED
+        instance.save()
+        system_metadata = instance.system_metadata
+
+        self.mox.StubOutWithMock(self.conductor_manager, '_get_image')
+        self.mox.StubOutWithMock(self.conductor_manager, '_schedule_instances')
+        self.mox.StubOutWithMock(self.conductor_manager.compute_rpcapi,
+                'unshelve_instance')
+
+        self.conductor_manager._get_image(self.context,
+                'fake_image_id').AndReturn('fake_image')
+        self.conductor_manager._schedule_instances(self.context,
+                'fake_image', [], instance).AndReturn(
+                        [{'host': 'fake_host'}])
+        self.conductor_manager.compute_rpcapi.unshelve_instance(self.context,
+                instance, 'fake_host', 'fake_image')
+        self.mox.ReplayAll()
+
+        system_metadata['shelved_at'] = timeutils.utcnow()
+        system_metadata['shelved_image_id'] = 'fake_image_id'
+        system_metadata['shelved_host'] = 'fake-mini'
+        self.conductor_manager.unshelve_instance(self.context, instance)
+
+    def test_unshelve_instance_schedule_and_rebuild_volume_backed(self):
+        db_instance = jsonutils.to_primitive(self._create_fake_instance())
+        instance = instance_obj.Instance.get_by_uuid(self.context,
+                db_instance['uuid'], expected_attrs=['system_metadata'])
+        instance.vm_state = vm_states.SHELVED_OFFLOADED
+        instance.save()
+        system_metadata = instance.system_metadata
+
+        self.mox.StubOutWithMock(self.conductor_manager, '_get_image')
+        self.mox.StubOutWithMock(self.conductor_manager, '_schedule_instances')
+        self.mox.StubOutWithMock(self.conductor_manager.compute_rpcapi,
+                'unshelve_instance')
+
+        self.conductor_manager._get_image(self.context,
+                'fake_image_id').AndReturn(None)
+        self.conductor_manager._schedule_instances(self.context,
+                None, [], instance).AndReturn(
+                        [{'host': 'fake_host'}])
+        self.conductor_manager.compute_rpcapi.unshelve_instance(self.context,
+                instance, 'fake_host', None)
+        self.mox.ReplayAll()
+
+        system_metadata['shelved_at'] = timeutils.utcnow()
+        system_metadata['shelved_image_id'] = 'fake_image_id'
+        system_metadata['shelved_host'] = 'fake-mini'
+        self.conductor_manager.unshelve_instance(self.context, instance)
+
+
+class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
     """ComputeTaskManager Tests."""
     def setUp(self):
         super(ConductorTaskTestCase, self).setUp()
@@ -1286,7 +1371,8 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test.TestCase):
         self.conductor_manager = self.conductor
 
 
-class ConductorTaskRPCAPITestCase(_BaseTaskTestCase, test.TestCase):
+class ConductorTaskRPCAPITestCase(_BaseTaskTestCase,
+        test_compute.BaseTestCase):
     """Conductor compute_task RPC namespace Tests."""
     def setUp(self):
         super(ConductorTaskRPCAPITestCase, self).setUp()
@@ -1297,7 +1383,7 @@ class ConductorTaskRPCAPITestCase(_BaseTaskTestCase, test.TestCase):
         self.conductor_manager = service_manager.compute_task_mgr
 
 
-class ConductorTaskAPITestCase(_BaseTaskTestCase, test.TestCase):
+class ConductorTaskAPITestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
     """Compute task API Tests."""
     def setUp(self):
         super(ConductorTaskAPITestCase, self).setUp()

nova/tests/fake_policy.py

@@ -84,6 +84,10 @@ policy_data = """
     "compute:snapshot": "",
     "compute:backup": "",
 
+    "compute:shelve": "",
+    "compute:shelve_offload": "",
+    "compute:unshelve": "",
+
     "compute:security_groups:add_to_instance": "",
     "compute:security_groups:remove_from_instance": "",