Refactor server restore function

Change-Id: I4e45796b49784597fbb415ac5221dc55ac3a28ec
Implements: blueprint remove-heat

karbor/services/protection/protection_plugins/server/nova_protection_plugin.py

@@ -10,16 +10,19 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from functools import partial
+
+from novaclient import exceptions
+from oslo_config import cfg
+from oslo_log import log as logging
+
 from karbor.common import constants
 from karbor import exception
 from karbor.services.protection.client_factory import ClientFactory
 from karbor.services.protection import protection_plugin
 from karbor.services.protection.protection_plugins.server \
     import server_plugin_schemas
-from karbor.services.protection.restore_heat import HeatResource
+from karbor.services.protection.protection_plugins import utils
-from oslo_config import cfg
-from oslo_log import log as logging
-from oslo_utils import uuidutils
 
 
 CONF = cfg.CONF
@@ -28,6 +31,13 @@ LOG = logging.getLogger(__name__)
 VOLUME_ATTACHMENT_RESOURCE = 'OS::Cinder::VolumeAttachment'
 FLOATING_IP_ASSOCIATION = 'OS::Nova::FloatingIPAssociation'
 
+nova_backup_opts = [
+    cfg.IntOpt(
+        'poll_interval', default=15,
+        help='Poll interval for Nova backup status'
+    ),
+]
+
 
 class ProtectOperation(protection_plugin.Operation):
     def on_main(self, checkpoint, resource, context, parameters, **kwargs):
@@ -170,36 +180,54 @@ class DeleteOperation(protection_plugin.Operation):
 
 
 class RestoreOperation(protection_plugin.Operation):
+    def __init__(self, poll_interval):
+        super(RestoreOperation, self).__init__()
+        self._interval = poll_interval
+
     def on_complete(self, checkpoint, resource, context, parameters, **kwargs):
         original_server_id = resource.id
-        heat_template = kwargs.get("heat_template")
-
-        restore_name = parameters.get("restore_name", "karbor-restore-server")
-
         LOG.info("Restoring server backup, server_id: %s.", original_server_id)
 
-        bank_section = checkpoint.get_resource_bank_section(original_server_id)
+        update_method = None
         try:
-            resource_definition = bank_section.get_object("metadata")
+            resource_definition = checkpoint.get_resource_bank_section(
+                original_server_id).get_object("metadata")
+
+            nova_client = ClientFactory.create_client("nova", context)
+            heat_template = kwargs.get("heat_template")
 
             # restore server instance
-            self._heat_restore_server_instance(
+            new_server_id = self._restore_server_instance(
-                heat_template, original_server_id,
+                nova_client, heat_template, original_server_id,
-                restore_name, resource_definition)
+                parameters.get("restore_name", "karbor-restore-server"),
+                resource_definition)
+
+            update_method = partial(utils.udpate_resource_restore_result,
+                                    kwargs.get('restore'), resource.type,
+                                    new_server_id)
+            update_method(constants.RESOURCE_STATUS_RESTORING)
+            self._wait_server_to_active(nova_client, new_server_id)
 
             # restore volume attachment
-            self._heat_restore_volume_attachment(
+            self._restore_volume_attachment(
-                heat_template, original_server_id, resource_definition)
+                nova_client, ClientFactory.create_client("cinder", context),
+                heat_template, new_server_id, resource_definition)
 
             # restore floating ip association
-            self._heat_restore_floating_association(
+            self._restore_floating_association(
-                heat_template, original_server_id, resource_definition)
+                nova_client, new_server_id, resource_definition)
-            LOG.debug("Restoring server backup, heat_template: %s.",
+
-                      heat_template)
+            heat_template.put_parameter(original_server_id, new_server_id)
+
+            update_method(constants.RESOURCE_STATUS_AVAILABLE)
+
             LOG.info("Finish restore server, server_id: %s.",
                      original_server_id)
+
         except Exception as e:
-            LOG.exception("restore server backup failed, server_id: %s.",
+            if update_method:
+                update_method(constants.RESOURCE_STATUS_ERROR, str(e))
+            LOG.exception("Restore server backup failed, server_id: %s.",
                           original_server_id)
             raise exception.RestoreBackupFailed(
                 reason=e,
@@ -207,112 +235,177 @@ class RestoreOperation(protection_plugin.Operation):
                 resource_type=constants.SERVER_RESOURCE_TYPE
             )
 
-    def _heat_restore_server_instance(self, heat_template,
+    def _restore_server_instance(self, nova_client, heat_template,
-                                      original_id, restore_name,
+                                 original_id, restore_name,
-                                      resource_definition):
+                                 resource_definition):
         server_metadata = resource_definition["server_metadata"]
         properties = {
-            "availability_zone": server_metadata["availability_zone"],
+            "availability_zone": server_metadata.get("availability_zone"),
-            "flavor": server_metadata["flavor"],
+            "flavor": server_metadata.get("flavor"),
             "name": restore_name,
+            "image": None
         }
+
         # server boot device
         boot_metadata = resource_definition["boot_metadata"]
-        boot_device_type = boot_metadata["boot_device_type"]
+        boot_device_type = boot_metadata.get("boot_device_type")
         if boot_device_type == "image":
-            original_image_id = boot_metadata["boot_image_id"]
+            properties["image"] = heat_template.get_resource_reference(
-            image_id = heat_template.get_resource_reference(
+                boot_metadata.get("boot_image_id"))
-                original_image_id)
+
-            properties["image"] = image_id
         elif boot_device_type == "volume":
-            original_volume_id = boot_metadata["boot_volume_id"]
-            volume_id = heat_template.get_resource_reference(
-                original_volume_id)
             properties["block_device_mapping_v2"] = [{
-                "volume_id": volume_id,
+                'uuid': heat_template.get_resource_reference(
-                "delete_on_termination": False,
+                    boot_metadata.get("boot_volume_id")),
-                "boot_index": 0,
+                'source_type': 'volume',
+                'destination_type': 'volume',
+                'boot_index': 0,
+                'delete_on_termination': False,
             }]
         else:
-            LOG.exception("Restore server backup failed, server_id: %s.",
+            reason = "Can not find the boot device of the server."
-                          original_id)
+            LOG.error("Restore server backup failed, (server_id:"
-            raise exception.RestoreBackupFailed(
+                      "%(server_id)s): %(reason)s.",
-                reason="Can not find the boot device of the server.",
+                      {'server_id': original_id,
-                resource_id=original_id,
+                       'reason': reason})
-                resource_type=constants.SERVER_RESOURCE_TYPE
+            raise Exception(reason)
-            )
 
         # server key_name, security_groups, networks
-        if server_metadata["key_name"] is not None:
+        properties["key_name"] = server_metadata.get("key_name", None)
-            properties["key_name"] = server_metadata["key_name"]
 
-        if server_metadata["security_groups"] is not None:
+        if server_metadata.get("security_groups"):
-            security_groups = []
+            properties["security_groups"] = [
-            for security_group in server_metadata["security_groups"]:
+                security_group["name"]
-                security_groups.append(security_group["name"])
+                for security_group in server_metadata["security_groups"]
-            properties["security_groups"] = security_groups
+            ]
 
-        networks = []
+        if server_metadata.get("networks"):
-        for network in server_metadata["networks"]:
+            properties["nics"] = [
-            networks.append({"network": network})
+                {'net-id': network}
-        properties["networks"] = networks
+                for network in server_metadata["networks"]
+            ]
 
-        heat_resource_id = uuidutils.generate_uuid()
+        properties["userdata"] = None
-        heat_server_resource = HeatResource(heat_resource_id,
-                                            constants.SERVER_RESOURCE_TYPE)
-        for key, value in properties.items():
-            heat_server_resource.set_property(key, value)
 
-        heat_template.put_resource(original_id,
+        try:
-                                   heat_server_resource)
+            server = nova_client.servers.create(**properties)
+        except Exception as ex:
+            LOG.error('Error creating server (server_id:%(server_id)s): '
+                      '%(reason)s',
+                      {'server_id': original_id,
+                       'reason': ex})
+            raise
 
-    def _heat_restore_volume_attachment(self, heat_template,
+        return server.id
-                                        original_server_id,
+
-                                        resource_definition):
+    def _restore_volume_attachment(self, nova_client, cinder_client,
-        attach_metadata = resource_definition["attach_metadata"]
+                                   heat_template, new_server_id,
+                                   resource_definition):
+        attach_metadata = resource_definition.get("attach_metadata", {})
         for original_id, attach_metadata_item in attach_metadata.items():
-            device = attach_metadata_item.get("device", None)
+            if attach_metadata_item.get("bootable", None) == "true":
-            if attach_metadata_item.get("bootable", None) != "true":
+                continue
-                instance_uuid = heat_template.get_resource_reference(
-                    original_server_id)
-                volume_id = heat_template.get_resource_reference(
-                    original_id)
-                properties = {"mountpoint": device,
-                              "instance_uuid": instance_uuid,
-                              "volume_id": volume_id}
-                heat_resource_id = uuidutils.generate_uuid()
-                heat_attachment_resource = HeatResource(
-                    heat_resource_id,
-                    VOLUME_ATTACHMENT_RESOURCE)
-                for key, value in properties.items():
-                    heat_attachment_resource.set_property(key, value)
-                heat_template.put_resource(
-                    "%s_%s" % (original_server_id, original_id),
-                    heat_attachment_resource)
 
-    def _heat_restore_floating_association(self, heat_template,
+            volume_id = heat_template.get_resource_reference(original_id)
-                                           original_server_id,
+            try:
-                                           resource_definition):
+                nova_client.volumes.create_server_volume(
+                    server_id=new_server_id,
+                    volume_id=volume_id,
+                    device=attach_metadata_item.get("device", None))
+
+            except Exception as ex:
+                LOG.error("Failed to attach volume %(vol)s to server %(srv)s, "
+                          "reason: %(err)s",
+                          {'vol': volume_id,
+                           'srv': new_server_id,
+                           'err': ex})
+                raise
+
+            self._wait_volume_to_attached(cinder_client, volume_id)
+
+    def _restore_floating_association(self, nova_client, new_server_id,
+                                      resource_definition):
         server_metadata = resource_definition["server_metadata"]
-        for floating_ip in server_metadata["floating_ips"]:
+        for floating_ip in server_metadata.get("floating_ips", []):
-            instance_uuid = heat_template.get_resource_reference(
+            nova_client.servers.add_floating_ip(
-                original_server_id)
+                nova_client.servers.get(new_server_id), floating_ip)
-            properties = {"instance_uuid": instance_uuid,
-                          "floating_ip": floating_ip}
-            heat_resource_id = uuidutils.generate_uuid()
-            heat_floating_resource = HeatResource(
-                heat_resource_id, FLOATING_IP_ASSOCIATION)
 
-            for key, value in properties.items():
+    def _wait_volume_to_attached(self, cinder_client, volume_id):
-                heat_floating_resource.set_property(key, value)
+        def _get_volume_status():
-            heat_template.put_resource(
+            try:
-                "%s_%s" % (original_server_id, floating_ip),
+                return cinder_client.volumes.get(volume_id).status
-                heat_floating_resource)
+            except Exception as ex:
+                LOG.error('Fetch volume(%(volume_id)s) status failed, '
+                          'reason: %(reason)s',
+                          {'volume_id': volume_id,
+                           'reason': ex})
+                return 'ERROR'
+
+        is_success = utils.status_poll(
+            _get_volume_status,
+            interval=self._interval,
+            success_statuses={'in-use', },
+            failure_statuses={'ERROR', },
+            ignore_statuses={'available', 'attaching'}
+        )
+        if not is_success:
+            raise Exception('Attach the volume to server failed')
+
+    def _wait_server_to_active(self, nova_client, server_id):
+        def _get_server_status():
+            try:
+                server = self._fetch_server(nova_client, server_id)
+                return server.status.split('(')[0] if server else 'BUILD'
+            except Exception as ex:
+                LOG.error('Fetch server(%(server_id)s) failed, '
+                          'reason: %(reason)s',
+                          {'server_id': server_id,
+                           'reason': ex})
+                return 'ERROR'
+
+        is_success = utils.status_poll(
+            _get_server_status,
+            interval=self._interval,
+            success_statuses={'ACTIVE', },
+            failure_statuses={'ERROR', },
+            ignore_statuses={'BUILD', 'HARD_REBOOT', 'PASSWORD', 'REBOOT',
+                             'RESCUE', 'RESIZE', 'REVERT_RESIZE', 'SHUTOFF',
+                             'SUSPENDED', 'VERIFY_RESIZE'},
+        )
+        if not is_success:
+            raise Exception('The server does not start successfully')
+
+    def _fetch_server(self, nova_client, server_id):
+        server = None
+        try:
+            server = nova_client.servers.get(server_id)
+        except exceptions.OverLimit as exc:
+            LOG.warning("Received an OverLimit response when "
+                        "fetching server (%(id)s) : %(exception)s",
+                        {'id': server_id,
+                         'exception': exc})
+        except exceptions.ClientException as exc:
+            if ((getattr(exc, 'http_status', getattr(exc, 'code', None)) in
+                 (500, 503))):
+                LOG.warning("Received the following exception when "
+                            "fetching server (%(id)s) : %(exception)s",
+                            {'id': server_id,
+                             'exception': exc})
+            else:
+                raise
+        return server
 
 
 class NovaProtectionPlugin(protection_plugin.ProtectionPlugin):
     _SUPPORT_RESOURCE_TYPES = [constants.SERVER_RESOURCE_TYPE]
 
+    def __init__(self, config=None):
+        super(NovaProtectionPlugin, self).__init__(config)
+        self._config.register_opts(nova_backup_opts,
+                                   'nova_backup_protection_plugin')
+        self._poll_interval = (
+            self._config.nova_backup_protection_plugin.poll_interval)
+
     @classmethod
     def get_supported_resources_types(cls):
         return cls._SUPPORT_RESOURCE_TYPES
@@ -337,7 +430,7 @@ class NovaProtectionPlugin(protection_plugin.ProtectionPlugin):
         return ProtectOperation()
 
     def get_restore_operation(self, resource):
-        return RestoreOperation()
+        return RestoreOperation(self._poll_interval)
 
     def get_delete_operation(self, resource):
         return DeleteOperation()

karbor/tests/unit/protection/test_nova_protection_plugin.py

@@ -10,9 +10,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import collections
-
 from collections import namedtuple
+import mock
+from oslo_config import cfg
+
 from karbor.common import constants
 from karbor.context import RequestContext
 from karbor.resource import Resource
@@ -24,7 +25,6 @@ from karbor.services.protection.protection_plugins.server \
 from karbor.services.protection.protection_plugins.server. \
     nova_protection_plugin import NovaProtectionPlugin
 from karbor.tests import base
-import mock
 
 
 class Server(object):
@@ -238,7 +238,7 @@ class FakeBankPlugin(BankPlugin):
 
 fake_bank = Bank(FakeBankPlugin())
 
-ResourceNode = collections.namedtuple(
+ResourceNode = namedtuple(
     "ResourceNode",
     ["value",
      "child_nodes"]
@@ -287,7 +287,7 @@ class NovaProtectionPluginTest(base.TestCase):
         self.cntxt = RequestContext(user_id='demo',
                                     project_id='abcd',
                                     auth_token='efgh')
-        self.plugin = NovaProtectionPlugin()
+        self.plugin = NovaProtectionPlugin(cfg.CONF)
         self.glance_client = FakeGlanceClient()
         self.nova_client = FakeNovaClient()
         self.cinder_client = FakeCinderClient()