Update the Cinder Backup Protection Plugin

Adjust the Cinder Backup protection plugin to the protection plugin
API, using hooks.

Change-Id: Ie03b6555f16c990dadf14f00bf219bf68aa990b9

karbor/services/protection/protection_plugins/volume/cinder_protection_plugin.py

@@ -10,15 +10,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from functools import partial
 import six
 
-from cinderclient.exceptions import NotFound
+from cinderclient import exceptions as cinder_exc
 from karbor.common import constants
 from karbor import exception
 from karbor.i18n import _LE, _LI
 from karbor.services.protection.client_factory import ClientFactory
-from karbor.services.protection.protection_plugins.base_protection_plugin \
+from karbor.services.protection import protection_plugin
-    import BaseProtectionPlugin
 from karbor.services.protection.protection_plugins.volume \
     import volume_plugin_cinder_schemas as cinder_schemas
 from karbor.services.protection.restore_heat import HeatResource
@@ -27,30 +27,226 @@ from oslo_log import log as logging
 from oslo_service import loopingcall
 from oslo_utils import uuidutils
 
-
-protection_opts = [
-    cfg.IntOpt('protection_sync_interval',
-               default=60,
-               help='update protection status interval')
-]
-CONF = cfg.CONF
-CONF.register_opts(protection_opts)
-
 LOG = logging.getLogger(__name__)
 
+cinder_backup_opts = [
+    cfg.IntOpt('poll_interval', default=30,
+               help='Poll interval for Cinder backup status'),
+]
 
-class CinderProtectionPlugin(BaseProtectionPlugin):
+CONF = cfg.CONF
+CONF.register_opts(cinder_backup_opts, 'cinder_backup_protection_plugin')
+
+
+def status_poll(get_status_func, interval, success_statuses=set(),
+                failure_statuses=set(), ignore_statuses=set(),
+                ignore_unexpected=False):
+    def _poll():
+        status = get_status_func()
+        if status in success_statuses:
+            raise loopingcall.LoopingCallDone(retvalue=True)
+        if status in failure_statuses:
+            raise loopingcall.LoopingCallDone(retvalue=False)
+        if status in ignore_statuses:
+            return
+        if ignore_unexpected is False:
+            raise loopingcall.LoopingCallDone(retvalue=False)
+
+    loop = loopingcall.FixedIntervalLoopingCall(_poll)
+    return loop.start(interval=interval, initial_delay=interval).wait()
+
+
+def get_backup_status(cinder_client, backup_id):
+    LOG.debug('Polling backup (id: %s)', backup_id)
+    try:
+        backup = cinder_client.backups.get(backup_id)
+        status = backup.status
+    except cinder_exc.NotFound:
+        status = 'not-found'
+    LOG.debug('Polled backup (id: %(backup_id)s) status: %(status)s',
+              {'backup_id': backup_id, 'status': status})
+    return status
+
+
+def get_volume_status(cinder_client, volume_id):
+    LOG.debug('Polling volume (id: %s)', volume_id)
+    try:
+        volume = cinder_client.volumes.get(volume_id)
+        status = volume.status
+    except cinder_exc.NotFound:
+        status = 'not-found'
+    LOG.debug('Polled volume (id: %(volume_id)s) status: %(status)s',
+              {'volume_id': volume_id, 'status': status})
+    return status
+
+
+class ProtectOperation(protection_plugin.Operation):
+    def __init__(self):
+        super(ProtectOperation, self).__init__()
+        self._interval = CONF.cinder_backup_protection_plugin.poll_interval
+
+    def on_main(self, checkpoint, resource, context, parameters, **kwargs):
+        volume_id = resource.id
+        bank_section = checkpoint.get_resource_bank_section(volume_id)
+        cinder_client = ClientFactory.create_client('cinder', context)
+        LOG.info(_LI('creating volume backup, volume_id: %s'), volume_id)
+        bank_section.create_object('status',
+                                   constants.RESOURCE_STATUS_PROTECTING)
+        resource_metadata = {
+            'volume_id': volume_id,
+        }
+        is_success = status_poll(
+            partial(get_volume_status, cinder_client, volume_id),
+            interval=self._interval,
+            success_statuses={'available', 'in-use', 'error_extending',
+                              'error_restoring'},
+            failure_statuses={'error', 'error_deleting', 'deleting',
+                              'not-found'},
+            ignore_statuses={'attaching', 'creating', 'backing-up',
+                             'restoring-backup'},
+        )
+        if not is_success:
+            raise exception.CreateBackupFailed(
+                reason='Volume is in errorneous state',
+                resource_id=volume_id,
+                resource_type=constants.VOLUME_RESOURCE_TYPE,
+            )
+
+        backup_name = parameters.get('backup_name')
+        backup = None
+        try:
+            backup = cinder_client.backups.create(
+                volume_id=volume_id,
+                name=backup_name,
+                force=True,
+            )
+        except Exception as e:
+            LOG.error(_LE('Error creating backup (volume_id: %(volume_id)s): '
+                          '%(reason)s'),
+                      {'volume_id': volume_id, 'reason': e})
+            bank_section.create_object('status',
+                                       constants.RESOURCE_STATUS_ERROR)
+            raise
+
+        backup_id = backup.id
+        resource_metadata['backup_id'] = backup_id
+        bank_section.create_object('metadata', resource_metadata)
+
+        is_success = status_poll(
+            partial(get_backup_status, cinder_client, backup_id),
+            interval=self._interval,
+            success_statuses={'available'},
+            failure_statuses={'error'},
+            ignore_statuses={'creating'},
+        )
+
+        if is_success is True:
+            LOG.info(
+                _LI('protecting volume (id: %(volume_id)s) to backup '
+                    '(id: %(backup_id)s) completed successfully'),
+                {'backup_id': backup_id, 'volume_id': volume_id}
+            )
+            bank_section.create_object('status',
+                                       constants.RESOURCE_STATUS_AVAILABLE)
+        else:
+            reason = None
+            try:
+                backup = cinder_client.backups.get(backup_id)
+            except Exception:
+                reason = 'Unable to find backup'
+            else:
+                reason = backup.fail_reason
+            LOG.error(
+                _LE('protecting volume (id: %(volume_id)s) to backup '
+                    '(id: %(backup_id)s) failed. Reason: "%(reason)s"'),
+                {
+                    'backup_id': backup_id,
+                    'volume_id': volume_id,
+                    'reason': reason,
+                }
+            )
+            bank_section.create_object('status',
+                                       constants.RESOURCE_STATUS_ERROR)
+            raise exception.CreateBackupFailed(
+                reason=reason,
+                resource_id=volume_id,
+                resource_type=constants.VOLUME_RESOURCE_TYPE,
+            )
+
+
+class RestoreOperation(protection_plugin.Operation):
+    def __init__(self):
+        super(RestoreOperation, self).__init__()
+
+    def on_main(self, checkpoint, resource, context, parameters, heat_template,
+                **kwargs):
+        resource_id = resource.id
+        bank_section = checkpoint.get_resource_bank_section(resource_id)
+        resource_metadata = bank_section.get_object('metadata')
+        name = parameters.get('restore_name',
+                              '%s@%s' % (checkpoint.id, resource_id))
+        heat_resource_id = uuidutils.generate_uuid()
+        heat_resource = HeatResource(heat_resource_id,
+                                     constants.VOLUME_RESOURCE_TYPE)
+        heat_resource.set_property('name', name)
+        if 'restore_description' in parameters:
+            heat_resource.set_property('description',
+                                       parameters['restore_description'])
+
+        heat_resource.set_property('backup_id', resource_metadata['backup_id'])
+        heat_template.put_resource(resource_id, heat_resource)
+
+
+class DeleteOperation(protection_plugin.Operation):
+    def __init__(self):
+        super(DeleteOperation, self).__init__()
+        self._interval = CONF.cinder_backup_protection_plugin.poll_interval
+
+    def on_main(self, checkpoint, resource, context, parameters, **kwargs):
+        resource_id = resource.id
+        bank_section = checkpoint.get_resource_bank_section(resource_id)
+        backup_id = None
+        try:
+            bank_section.update_object('status',
+                                       constants.RESOURCE_STATUS_DELETING)
+            resource_metadata = bank_section.get_object('metadata')
+            backup_id = resource_metadata['backup_id']
+            cinder_client = ClientFactory.create_client('cinder', context)
+            try:
+                backup = cinder_client.backups.get(backup_id)
+                cinder_client.backups.delete(backup)
+            except cinder_exc.NotFound:
+                LOG.info(_LI('Backup id: %s not found. Assuming deleted'),
+                         backup_id)
+            is_success = status_poll(
+                partial(get_backup_status, cinder_client, backup_id),
+                interval=self._interval,
+                success_statuses={'deleted', 'not-found'},
+                failure_statuses={'error', 'error_deleting'},
+                ignore_statuses={'deleting'},
+            )
+            if not is_success:
+                raise exception.NotFound()
+            bank_section.delete_object('metadata')
+            bank_section.update_object('status',
+                                       constants.RESOURCE_STATUS_DELETED)
+        except Exception as e:
+            LOG.error(_LE('delete volume backup failed, backup_id: %s'),
+                      backup_id)
+            bank_section.update_object('status',
+                                       constants.RESOURCE_STATUS_ERROR)
+            raise exception.DeleteBackupFailed(
+                reason=six.text_type(e),
+                resource_id=resource_id,
+                resource_type=constants.VOLUME_RESOURCE_TYPE
+            )
+
+
+class CinderBackupProtectionPlugin(protection_plugin.ProtectionPlugin):
     _SUPPORT_RESOURCE_TYPES = [constants.VOLUME_RESOURCE_TYPE]
 
     def __init__(self, config=None):
-        super(CinderProtectionPlugin, self).__init__(config)
+        super(CinderBackupProtectionPlugin, self).__init__(config)
-        self.protection_resource_map = {}
-        self.protection_sync_interval = CONF.protection_sync_interval
-
-        sync_status_loop = loopingcall.FixedIntervalLoopingCall(
-            self.sync_status)
-        sync_status_loop.start(interval=self.protection_sync_interval,
-                               initial_delay=self.protection_sync_interval)
 
     @classmethod
     def get_supported_resources_types(cls):
@@ -73,141 +269,11 @@ class CinderProtectionPlugin(BaseProtectionPlugin):
         # TODO(hurong)
         pass
 
-    def _cinder_client(self, cntxt):
+    def get_protect_operation(self, resource):
-        return ClientFactory.create_client("cinder", cntxt)
+        return ProtectOperation()
 
-    def create_backup(self, cntxt, checkpoint, **kwargs):
+    def get_restore_operation(self, resource):
-        resource_node = kwargs.get("node")
+        return RestoreOperation()
-        backup_name = kwargs.get("backup_name")
-        resource = resource_node.value
-        volume_id = resource.id
 
-        bank_section = checkpoint.get_resource_bank_section(volume_id)
+    def get_delete_operation(self, resource):
-
+        return DeleteOperation()
-        resource_definition = {"volume_id": volume_id}
-        cinder_client = self._cinder_client(cntxt)
-
-        LOG.info(_LI("creating volume backup, volume_id: %s."), volume_id)
-        try:
-            bank_section.create_object("status",
-                                       constants.RESOURCE_STATUS_PROTECTING)
-
-            backup = cinder_client.backups.create(volume_id=volume_id,
-                                                  name=backup_name,
-                                                  force=True)
-            resource_definition["backup_id"] = backup.id
-            bank_section.create_object("metadata", resource_definition)
-            self.protection_resource_map[volume_id] = {
-                "bank_section": bank_section,
-                "backup_id": backup.id,
-                "cinder_client": cinder_client,
-                "operation": "create"
-            }
-        except Exception as e:
-            LOG.error(_LE("create volume backup failed, volume_id: %s."),
-                      volume_id)
-            bank_section.update_object("status",
-                                       constants.RESOURCE_STATUS_ERROR)
-            raise exception.CreateBackupFailed(
-                reason=six.text_type(e),
-                resource_id=volume_id,
-                resource_type=constants.VOLUME_RESOURCE_TYPE
-            )
-
-    def delete_backup(self, cntxt, checkpoint, **kwargs):
-        resource_node = kwargs.get("node")
-        resource_id = resource_node.value.id
-
-        bank_section = checkpoint.get_resource_bank_section(resource_id)
-        cinder_client = self._cinder_client(cntxt)
-        LOG.info(_LI("deleting volume backup, volume_id: %s."), resource_id)
-        try:
-            bank_section.update_object("status",
-                                       constants.RESOURCE_STATUS_DELETING)
-            resource_definition = bank_section.get_object("metadata")
-            backup_id = resource_definition["backup_id"]
-            cinder_client.backups.delete(backup_id)
-            bank_section.delete_object("metadata")
-            self.protection_resource_map[resource_id] = {
-                "bank_section": bank_section,
-                "backup_id": backup_id,
-                "cinder_client": cinder_client,
-                "operation": "delete"
-            }
-        except Exception as e:
-            LOG.error(_LE("delete volume backup failed, volume_id: %s."),
-                      resource_id)
-            bank_section.update_object("status",
-                                       constants.CHECKPOINT_STATUS_ERROR)
-
-            raise exception.DeleteBackupFailed(
-                reason=six.text_type(e),
-                resource_id=resource_id,
-                resource_type=constants.VOLUME_RESOURCE_TYPE
-            )
-
-    def sync_status(self):
-        for resource_id, resource_info in self.protection_resource_map.items():
-            backup_id = resource_info["backup_id"]
-            bank_section = resource_info["bank_section"]
-            cinder_client = resource_info["cinder_client"]
-            operation = resource_info["operation"]
-            try:
-                backup = cinder_client.backups.get(backup_id)
-                if backup.status == "available":
-                    bank_section.update_object(
-                        "status", constants.RESOURCE_STATUS_AVAILABLE)
-                    self.protection_resource_map.pop(resource_id)
-                elif backup.status in ["error", "error-deleting"]:
-                    bank_section.update_object(
-                        "status", constants.RESOURCE_STATUS_ERROR)
-                    self.protection_resource_map.pop(resource_id)
-                else:
-                    continue
-            except Exception as exc:
-                if operation == "delete" and type(exc) == NotFound:
-                    bank_section.update_object(
-                        "status",
-                        constants.RESOURCE_STATUS_DELETED)
-                    LOG.info(_LI("deleting volume backup finished, "
-                                 "backup id: %s"), backup_id)
-                else:
-                    LOG.error(_LE("deleting volume backup error.exc: %s"),
-                              six.text_type(exc))
-                self.protection_resource_map.pop(resource_id)
-
-    def restore_backup(self, cntxt, checkpoint, **kwargs):
-        resource_node = kwargs.get("node")
-        resource_id = resource_node.value.id
-        heat_template = kwargs.get("heat_template")
-
-        name = kwargs.get("restore_name",
-                          "%s@%s" % (checkpoint.id, resource_id))
-        description = kwargs.get("restore_description")
-
-        heat_resource_id = uuidutils.generate_uuid()
-        heat_resource = HeatResource(heat_resource_id,
-                                     constants.VOLUME_RESOURCE_TYPE)
-
-        bank_section = checkpoint.get_resource_bank_section(resource_id)
-        try:
-            resource_definition = bank_section.get_object("metadata")
-            backup_id = resource_definition["backup_id"]
-            properties = {"backup_id": backup_id,
-                          "name": name}
-
-            if description is not None:
-                properties["description"] = description
-
-            for key, value in properties.items():
-                heat_resource.set_property(key, value)
-
-            heat_template.put_resource(resource_id, heat_resource)
-        except Exception as e:
-            LOG.error(_LE("restore volume backup failed, volume_id: %s."),
-                      resource_id)
-            raise exception.RestoreBackupFailed(
-                reason=six.text_type(e),
-                resource_id=resource_id,
-                resource_type=constants.VOLUME_RESOURCE_TYPE
-            )

karbor/tests/fullstack/test_checkpoints.py

@@ -19,10 +19,9 @@ class CheckpointsTest(karbor_base.KarborBaseTest):
     """Test Checkpoints operation """
     def setUp(self):
         super(CheckpointsTest, self).setUp()
-        self.provider_id = self.provider_id_noop
+        self.provider_id = self.provider_id_os
 
     def test_checkpoint_create(self):
-        self.skipTest('Requires cinder protection plugin adjustment')
         volume = self.store(objects.Volume())
         volume.create(1)
         plan = self.store(objects.Plan())
@@ -95,7 +94,7 @@ class CheckpointsTest(karbor_base.KarborBaseTest):
 
     def test_checkpoint_for_server_attached_volume(self):
         """Test checkpoint for server which has attached some volumes"""
-        self.skipTest('Requires cinder protection plugin adjustment')
+        self.skipTest('Requires server protection plugin adjustment')
         volume = self.store(objects.Volume())
         volume.create(1)
         server = self.store(objects.Server())

karbor/tests/unit/conf_fixture.py

@@ -31,3 +31,8 @@ def set_defaults(conf):
         os.path.join(os.path.dirname(__file__), '..', '..', '..')))
     conf.set_default('provider_config_dir',
                      os.path.join(os.path.dirname(__file__), 'fake_providers'))
+    conf.set_default(
+        name='poll_interval',
+        group='cinder_backup_protection_plugin',
+        default=0.3,
+    )

karbor/tests/unit/protection/fakes.py

@@ -81,8 +81,9 @@ class FakeBankPlugin(BankPlugin):
         fake_bank_opts = [
             cfg.StrOpt('fake_host'),
         ]
-        config.register_opts(fake_bank_opts, 'fake_bank')
+        if config:
-        self.fake_host = config['fake_bank']['fake_host']
+            config.register_opts(fake_bank_opts, 'fake_bank')
+            self.fake_host = config['fake_bank']['fake_host']
 
     def create_object(self, key, value):
         self._objects[key] = value
@@ -109,6 +110,9 @@ class FakeBankPlugin(BankPlugin):
     def delete_object(self, key):
         self._objects.pop(key)
 
+    def get_owner_id(self):
+        return
+
 
 def fake_restore():
     restore = {

karbor/tests/unit/protection/test_cinder_protection_plugin.py

@@ -10,47 +10,26 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from cinderclient import exceptions as cinder_exc
 import collections
 import datetime
 from karbor.common import constants
 from karbor.context import RequestContext
+from karbor import exception
 from karbor.resource import Resource
-from karbor.services.protection.bank_plugin import Bank
+from karbor.services.protection import bank_plugin
-from karbor.services.protection.bank_plugin import BankPlugin
-from karbor.services.protection.bank_plugin import BankSection
 from karbor.services.protection.client_factory import ClientFactory
 from karbor.services.protection.protection_plugins.volume. \
-    cinder_protection_plugin import CinderProtectionPlugin
+    cinder_protection_plugin import CinderBackupProtectionPlugin
 from karbor.services.protection.protection_plugins.volume \
     import volume_plugin_cinder_schemas as cinder_schemas
 from karbor.services.protection.restore_heat import HeatTemplate
 from karbor.tests import base
+from karbor.tests.unit.protection import fakes
 import mock
 from oslo_config import cfg
 
 
-class FakeBankPlugin(BankPlugin):
-    def create_object(self, key, value):
-        return
-
-    def update_object(self, key, value):
-        return
-
-    def get_object(self, key):
-        return
-
-    def list_objects(self, prefix=None, limit=None, marker=None):
-        return
-
-    def delete_object(self, key):
-        return
-
-    def get_owner_id(self):
-        return
-
-fake_bank = Bank(FakeBankPlugin())
-fake_bank_section = BankSection(bank=fake_bank, prefix="fake")
-
 ResourceNode = collections.namedtuple(
     "ResourceNode",
     ["value",
@@ -65,19 +44,54 @@ Image = collections.namedtuple(
 )
 
 
+def call_hooks(operation, checkpoint, resource, context, parameters, **kwargs):
+    def noop(*args, **kwargs):
+        pass
+
+    hooks = (
+        'on_prepare_begin',
+        'on_prepare_finish',
+        'on_main',
+        'on_complete',
+    )
+    for hook_name in hooks:
+        hook = getattr(operation, hook_name, noop)
+        hook(checkpoint, resource, context, parameters, **kwargs)
+
+
 class FakeCheckpoint(object):
-    def __init__(self):
+    def __init__(self, section):
-        self.bank_section = fake_bank_section
+        self.bank_section = section
         self.id = "fake_id"
 
-    def get_resource_bank_section(self, resource_id):
+    def get_resource_bank_section(self, resource_id=None):
         return self.bank_section
 
 
+class BackupResponse(object):
+    def __init__(self, bkup_id, final_status, working_status, time_to_work):
+        self._final_status = final_status
+        self._working_status = working_status
+        self._time_to_work = time_to_work
+        self._id = bkup_id
+
+    def __call__(self, *args, **kwargs):
+        res = mock.Mock()
+        res.id = self._id
+        if self._time_to_work > 0:
+            self._time_to_work -= 1
+            res.status = self._working_status
+        else:
+            res.status = self._final_status
+        if res.status == 'not-found':
+            raise cinder_exc.NotFound(403)
+        return res
+
+
 class CinderProtectionPluginTest(base.TestCase):
     def setUp(self):
         super(CinderProtectionPluginTest, self).setUp()
-        self.plugin = CinderProtectionPlugin()
+        self.plugin = CinderBackupProtectionPlugin()
         cfg.CONF.set_default('cinder_endpoint',
                              'http://127.0.0.1:8776/v2',
                              'cinder_client')
@@ -86,7 +100,14 @@ class CinderProtectionPluginTest(base.TestCase):
                                     project_id='abcd',
                                     auth_token='efgh')
         self.cinder_client = ClientFactory.create_client("cinder", self.cntxt)
-        self.checkpoint = FakeCheckpoint()
+
+    def _get_checkpoint(self):
+        fake_bank = bank_plugin.Bank(fakes.FakeBankPlugin())
+        fake_bank_section = bank_plugin.BankSection(
+            bank=fake_bank,
+            prefix="fake"
+        )
+        return FakeCheckpoint(fake_bank_section)
 
     def test_get_options_schema(self):
         options_schema = self.plugin.get_options_schema(
@@ -104,68 +125,160 @@ class CinderProtectionPluginTest(base.TestCase):
         self.assertEqual(options_schema,
                          cinder_schemas.SAVED_INFO_SCHEMA)
 
-    def test_create_backup(self):
+    @mock.patch('karbor.services.protection.clients.cinder.create')
-        resource = Resource(id="123",
+    def test_protect_succeed(self, mock_cinder_create):
-                            type=constants.VOLUME_RESOURCE_TYPE,
+        resource = Resource(
-                            name="test")
+            id="123",
-        resource_node = ResourceNode(value=resource,
+            type=constants.VOLUME_RESOURCE_TYPE,
-                                     child_nodes=[])
+            name="test",
+        )
+        checkpoint = self._get_checkpoint()
+        section = checkpoint.get_resource_bank_section()
+        operation = self.plugin.get_protect_operation(resource)
+        section.create_object = mock.MagicMock()
+        mock_cinder_create.return_value = self.cinder_client
+        with mock.patch.multiple(
+            self.cinder_client,
+            volumes=mock.DEFAULT,
+            backups=mock.DEFAULT
+        ) as mocks:
+            mocks['volumes'].get.return_value = mock.Mock()
+            mocks['volumes'].get.return_value.status = 'available'
+            mocks['backups'].create = BackupResponse(
+                '456', 'creating', '---', 0)
+            mocks['backups'].get = BackupResponse(
+                '456', 'available', 'creating', 2)
+            call_hooks(operation, checkpoint, resource, self.cntxt, {})
 
-        fake_bank_section.create_object = mock.MagicMock()
+    @mock.patch('karbor.services.protection.clients.cinder.create')
+    def test_protect_fail_backup(self, mock_cinder_create):
+        resource = Resource(
+            id="123",
+            type=constants.VOLUME_RESOURCE_TYPE,
+            name="test",
+        )
+        checkpoint = self._get_checkpoint()
+        operation = self.plugin.get_protect_operation(resource)
+        mock_cinder_create.return_value = self.cinder_client
+        with mock.patch.multiple(
+            self.cinder_client,
+            volumes=mock.DEFAULT,
+            backups=mock.DEFAULT
+        ) as mocks:
+            mocks['volumes'].get.return_value = mock.Mock()
+            mocks['volumes'].get.return_value.status = 'available'
+            mocks['backups'].backups.create = BackupResponse(
+                '456', 'creating', '---', 0)
+            mocks['backups'].backups.get = BackupResponse(
+                '456', 'error', 'creating', 2)
+            self.assertRaises(
+                exception.CreateBackupFailed,
+                call_hooks,
+                operation,
+                checkpoint,
+                resource,
+                self.cntxt,
+                {}
+            )
 
-        self.plugin._cinder_client = mock.MagicMock()
+    @mock.patch('karbor.services.protection.clients.cinder.create')
-        self.plugin._cinder_client.return_value = self.cinder_client
+    def test_protect_fail_volume(self, mock_cinder_create):
+        resource = Resource(
+            id="123",
+            type=constants.VOLUME_RESOURCE_TYPE,
+            name="test",
+        )
+        checkpoint = self._get_checkpoint()
+        operation = self.plugin.get_protect_operation(resource)
+        mock_cinder_create.return_value = self.cinder_client
+        with mock.patch.multiple(
+            self.cinder_client,
+            volumes=mock.DEFAULT,
+            backups=mock.DEFAULT
+        ) as mocks:
+            mocks['volumes'].get.return_value = mock.Mock()
+            mocks['volumes'].get.return_value.status = 'error'
+            mocks['backups'].backups.create = BackupResponse(
+                '456', 'creating', '---', 0)
+            mocks['backups'].backups.get = BackupResponse(
+                '456', 'error', 'creating', 2)
+            self.assertRaises(
+                exception.CreateBackupFailed,
+                call_hooks,
+                operation,
+                checkpoint,
+                resource,
+                self.cntxt,
+                {}
+            )
 
-        self.cinder_client.backups.create = mock.MagicMock()
+    @mock.patch('karbor.services.protection.clients.cinder.create')
-        self.cinder_client.backups.return_value = "456"
+    def test_delete_succeed(self, mock_cinder_create):
+        resource = Resource(
+            id="123",
+            type=constants.VOLUME_RESOURCE_TYPE,
+            name="test",
+        )
+        checkpoint = self._get_checkpoint()
+        section = checkpoint.get_resource_bank_section()
+        section.create_object('metadata', {
+            'backup_id': '456',
+        })
+        operation = self.plugin.get_delete_operation(resource)
+        mock_cinder_create.return_value = self.cinder_client
+        with mock.patch.object(self.cinder_client, 'backups') as backups:
+            backups.delete = BackupResponse('456', 'deleting', '---', 0)
+            backups.get = BackupResponse('456', 'not-found', 'deleting', 2)
+            call_hooks(operation, checkpoint, resource, self.cntxt, {})
 
-        fake_bank_section.create_object = mock.MagicMock()
+    @mock.patch('karbor.services.protection.clients.cinder.create')
-        # fake_bank_section.update_object = mock.MagicMock()
+    def test_delete_fail(self, mock_cinder_create):
+        resource = Resource(
+            id="123",
+            type=constants.VOLUME_RESOURCE_TYPE,
+            name="test",
+        )
+        checkpoint = self._get_checkpoint()
+        section = checkpoint.get_resource_bank_section()
+        section.create_object('metadata', {
+            'backup_id': '456',
+        })
+        operation = self.plugin.get_delete_operation(resource)
+        mock_cinder_create.return_value = self.cinder_client
+        with mock.patch.object(self.cinder_client, 'backups') as backups:
+            backups.delete = BackupResponse('456', 'deleting', '---', 0)
+            backups.get = BackupResponse('456', 'error', 'deleting', 2)
+            self.assertRaises(
+                exception.DeleteBackupFailed,
+                call_hooks,
+                operation,
+                checkpoint,
+                resource,
+                self.cntxt,
+                {}
+            )
 
-        self.plugin.create_backup(self.cntxt, self.checkpoint,
+    def test_restore(self):
-                                  node=resource_node)
+        heat_template = HeatTemplate()
+        resource = Resource(
+            id="123",
+            type=constants.VOLUME_RESOURCE_TYPE,
+            name="fake",
+        )
+        checkpoint = self._get_checkpoint()
+        section = checkpoint.get_resource_bank_section()
+        section.create_object('metadata', {
+            'backup_id': '456',
+        })
 
-    def test_delete_backup(self):
+        parameters = {
-        resource = Resource(id="123",
+            "restore_name": "karbor restore volume",
-                            type=constants.SERVER_RESOURCE_TYPE,
+            "restore_description": "karbor restore",
-                            name="test")
-        resource_node = ResourceNode(value=resource,
-                                     child_nodes=[])
-
-        fake_bank_section.update_object = mock.MagicMock()
-
-        fake_bank_section.get_object = mock.MagicMock()
-        fake_bank_section.get_object.return_value = {
-            "backup_id": "456"
         }
 
-        self.plugin._cinder_client = mock.MagicMock()
+        operation = self.plugin.get_restore_operation(resource)
-        self.plugin._cinder_client.return_value = self.cinder_client
+        call_hooks(operation, checkpoint, resource, self.cntxt,
-        self.cinder_client.backups.delete = mock.MagicMock()
+                   parameters, heat_template=heat_template)
-
-        fake_bank_section.delete_object = mock.MagicMock()
-
-        self.plugin.delete_backup(self.cntxt, self.checkpoint,
-                                  node=resource_node)
-
-    def test_restore_backup(self):
-        heat_template = HeatTemplate()
-        resource = Resource(id="123",
-                            type=constants.VOLUME_RESOURCE_TYPE,
-                            name="fake")
-        resource_node = ResourceNode(value=resource,
-                                     child_nodes=[])
-        resource_definition = {"backup_id": "456"}
-        kwargs = {"node": resource_node,
-                  "heat_template": heat_template,
-                  "restore_name": "karbor restore volume",
-                  "restore_description": "karbor restore"}
-
-        fake_bank_section.get_object = mock.MagicMock()
-        fake_bank_section.get_object.return_value = resource_definition
-
-        self.plugin.restore_backup(self.cntxt, self.checkpoint,
-                                   **kwargs)
         self.assertEqual(1, len(heat_template._resources))
 
         heat_resource_id = heat_template._original_id_resource_map["123"]
@@ -189,6 +302,3 @@ class CinderProtectionPluginTest(base.TestCase):
         types = self.plugin.get_supported_resources_types()
         self.assertEqual(types,
                          [constants.VOLUME_RESOURCE_TYPE])
-
-    def tearDown(self):
-        super(CinderProtectionPluginTest, self).tearDown()

setup.cfg

@@ -41,7 +41,7 @@ karbor.database.migration_backend =
     sqlalchemy = oslo_db.sqlalchemy.migration
 karbor.protections =
     karbor-swift-bank-plugin = karbor.services.protection.bank_plugins.swift_bank_plugin:SwiftBankPlugin
-    karbor-volume-protection-plugin = karbor.services.protection.protection_plugins.volume.cinder_protection_plugin:CinderProtectionPlugin
+    karbor-volume-protection-plugin = karbor.services.protection.protection_plugins.volume.cinder_protection_plugin:CinderBackupProtectionPlugin
     karbor-image-protection-plugin = karbor.services.protection.protection_plugins.image.image_protection_plugin:GlanceProtectionPlugin
     karbor-server-protection-plugin = karbor.services.protection.protection_plugins.server.nova_protection_plugin:NovaProtectionPlugin
     karbor-noop-protection-plugin = karbor.services.protection.protection_plugins.noop_plugin:NoopProtectionPlugin