From 3b7f49986257c8e747a02b090a09c9deb8d577a5 Mon Sep 17 00:00:00 2001 From: Rajat Dhasmana Date: Fri, 16 Apr 2021 07:14:34 -0400 Subject: [PATCH] Add user messages for backup operations This patch adds user messages for the following backup operations: 1) Create backup 2) Restore backup 3) Delete Backup Change-Id: Idc00b125b33bf9abd2e2057d9cee25a337e6d418 --- cinder/backup/manager.py | 137 +++- cinder/context.py | 3 + cinder/message/api.py | 34 + cinder/message/message_field.py | 36 + cinder/scheduler/manager.py | 6 + .../tests/unit/backup/test_backup_messages.py | 624 ++++++++++++++++++ cinder/tests/unit/message/test_api.py | 29 + ...backup-user-messages-5ee0c7ead3def8f9.yaml | 8 + 8 files changed, 860 insertions(+), 17 deletions(-) create mode 100644 cinder/tests/unit/backup/test_backup_messages.py create mode 100644 releasenotes/notes/backup-user-messages-5ee0c7ead3def8f9.yaml diff --git a/cinder/backup/manager.py b/cinder/backup/manager.py index 4457ce32bfd..74d572c9339 100644 --- a/cinder/backup/manager.py +++ b/cinder/backup/manager.py @@ -51,6 +51,8 @@ from cinder import exception from cinder.i18n import _ from cinder.keymgr import migration as key_migration from cinder import manager +from cinder.message import api as message_api +from cinder.message import message_field from cinder import objects from cinder.objects import fields from cinder import quota @@ -135,6 +137,7 @@ class BackupManager(manager.SchedulerDependentManager): self.driver_name, new_name) self.driver_name = new_name self.service = importutils.import_class(self.driver_name) + self.message_api = message_api.API() def init_host(self, **kwargs): """Run initialization needed for a standalone service.""" @@ -340,6 +343,9 @@ class BackupManager(manager.SchedulerDependentManager): context, snapshot_id) if snapshot_id else None previous_status = volume.get('previous_status', None) updates = {} + context.message_resource_id = backup.id + context.message_resource_type = message_field.Resource.VOLUME_BACKUP + context.message_action = message_field.Action.BACKUP_CREATE if snapshot_id: log_message = ('Create backup started, backup: %(backup_id)s ' 'volume: %(volume_id)s snapshot: %(snapshot_id)s.' @@ -386,12 +392,18 @@ class BackupManager(manager.SchedulerDependentManager): 'actual_status': actual_status, } volume_utils.update_backup_error(backup, err) + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.BACKUP_INVALID_STATE) raise exception.InvalidBackup(reason=err) try: if not self.is_working(): err = _('Create backup aborted due to backup service is down.') volume_utils.update_backup_error(backup, err) + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.BACKUP_SERVICE_DOWN) raise exception.InvalidBackup(reason=err) backup.service = self.driver_name @@ -444,6 +456,7 @@ class BackupManager(manager.SchedulerDependentManager): self._notify_about_backup_usage(context, backup, "create.end") def _run_backup(self, context, backup, volume): + message_created = False # Save a copy of the encryption key ID in case the volume is deleted. if (volume.encryption_key_id is not None and backup.encryption_key_id is None): @@ -461,13 +474,33 @@ class BackupManager(manager.SchedulerDependentManager): # context switching and may end up blocking the greenthread, so we go # with native threads proxy-wrapping the device file object. try: - backup_device = self.volume_rpcapi.get_backup_device(context, - backup, - volume) - attach_info = self._attach_device(context, - backup_device.device_obj, - properties, - backup_device.is_snapshot) + try: + backup_device = self.volume_rpcapi.get_backup_device(context, + backup, + volume) + except Exception: + with excutils.save_and_reraise_exception(): + # We set message_create to True before creating the + # message because if the message create call fails + # and is catched by the base/outer exception handler + # then we will end up storing a wrong message + message_created = True + self.message_api.create_from_request_context( + context, + detail= + message_field.Detail.BACKUP_CREATE_DEVICE_ERROR) + try: + attach_info = self._attach_device(context, + backup_device.device_obj, + properties, + backup_device.is_snapshot) + except Exception: + with excutils.save_and_reraise_exception(): + if not message_created: + message_created = True + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.ATTACH_ERROR) try: device_path = attach_info['device']['path'] if (isinstance(device_path, str) and @@ -485,17 +518,41 @@ class BackupManager(manager.SchedulerDependentManager): else: updates = backup_service.backup(backup, tpool.Proxy(device_path)) - + except Exception: + with excutils.save_and_reraise_exception(): + if not message_created: + message_created = True + self.message_api.create_from_request_context( + context, + detail= + message_field.Detail.BACKUP_CREATE_DRIVER_ERROR) finally: - self._detach_device(context, attach_info, - backup_device.device_obj, properties, - backup_device.is_snapshot, force=True, - ignore_errors=True) + try: + self._detach_device(context, attach_info, + backup_device.device_obj, properties, + backup_device.is_snapshot, force=True, + ignore_errors=True) + except Exception: + with excutils.save_and_reraise_exception(): + if not message_created: + message_created = True + self.message_api.create_from_request_context( + context, + detail= + message_field.Detail.DETACH_ERROR) finally: with backup.as_read_deleted(): backup.refresh() - self._cleanup_temp_volumes_snapshots_when_backup_created( - context, backup) + try: + self._cleanup_temp_volumes_snapshots_when_backup_created( + context, backup) + except Exception: + with excutils.save_and_reraise_exception(): + if not message_created: + self.message_api.create_from_request_context( + context, + detail= + message_field.Detail.BACKUP_CREATE_CLEANUP_ERROR) return updates def _is_our_backup(self, backup): @@ -523,6 +580,9 @@ class BackupManager(manager.SchedulerDependentManager): @utils.limit_operations def restore_backup(self, context, backup, volume_id): """Restore volume backups from configured backup service.""" + context.message_resource_id = backup.id + context.message_resource_type = message_field.Resource.VOLUME_BACKUP + context.message_action = message_field.Action.BACKUP_RESTORE LOG.info('Restore backup started, backup: %(backup_id)s ' 'volume: %(volume_id)s.', {'backup_id': backup.id, 'volume_id': volume_id}) @@ -546,6 +606,12 @@ class BackupManager(manager.SchedulerDependentManager): (fields.VolumeStatus.ERROR if volume_previous_status == fields.VolumeStatus.CREATING else fields.VolumeStatus.ERROR_RESTORING)}) + self.message_api.create( + context, + action=message_field.Action.BACKUP_RESTORE, + resource_type=message_field.Resource.VOLUME_BACKUP, + resource_uuid=volume.id, + detail=message_field.Detail.VOLUME_INVALID_STATE) raise exception.InvalidVolume(reason=err) expected_status = fields.BackupStatus.RESTORING @@ -558,6 +624,9 @@ class BackupManager(manager.SchedulerDependentManager): volume_utils.update_backup_error(backup, err) self.db.volume_update(context, volume_id, {'status': fields.VolumeStatus.ERROR}) + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.BACKUP_INVALID_STATE) raise exception.InvalidBackup(reason=err) if volume['size'] > backup['size']: @@ -628,6 +697,7 @@ class BackupManager(manager.SchedulerDependentManager): self._notify_about_backup_usage(context, backup, "restore.end") def _run_restore(self, context, backup, volume): + message_created = False orig_key_id = volume.encryption_key_id backup_service = self.service(context) @@ -635,7 +705,13 @@ class BackupManager(manager.SchedulerDependentManager): secure_enabled = ( self.volume_rpcapi.secure_file_operations_enabled(context, volume)) - attach_info = self._attach_device(context, volume, properties) + try: + attach_info = self._attach_device(context, volume, properties) + except Exception: + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.ATTACH_ERROR) + raise # NOTE(geguileo): Not all I/O disk operations properly do greenthread # context switching and may end up blocking the greenthread, so we go @@ -664,10 +740,25 @@ class BackupManager(manager.SchedulerDependentManager): LOG.exception('Restoring backup %(backup_id)s to volume ' '%(volume_id)s failed.', {'backup_id': backup.id, 'volume_id': volume.id}) + # We set message_create to True before creating the + # message because if the message create call fails + # and is catched by the base/outer exception handler + # then we will end up storing a wrong message + message_created = True + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.BACKUP_RESTORE_ERROR) raise finally: - self._detach_device(context, attach_info, volume, properties, - force=True) + try: + self._detach_device(context, attach_info, volume, properties, + force=True) + except Exception: + if not message_created: + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.DETACH_ERROR) + raise # Regardless of whether the restore was successful, do some # housekeeping to ensure the restored volume's encryption key ID is @@ -717,6 +808,9 @@ class BackupManager(manager.SchedulerDependentManager): self._notify_about_backup_usage(context, backup, "delete.start") + context.message_resource_id = backup.id + context.message_resource_type = message_field.Resource.VOLUME_BACKUP + context.message_action = message_field.Action.BACKUP_DELETE expected_status = fields.BackupStatus.DELETING actual_status = backup.status if actual_status != expected_status: @@ -725,12 +819,18 @@ class BackupManager(manager.SchedulerDependentManager): % {'expected_status': expected_status, 'actual_status': actual_status} volume_utils.update_backup_error(backup, err) + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.BACKUP_INVALID_STATE) raise exception.InvalidBackup(reason=err) if backup.service and not self.is_working(): err = _('Delete backup is aborted due to backup service is down.') status = fields.BackupStatus.ERROR_DELETING volume_utils.update_backup_error(backup, err, status) + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.BACKUP_SERVICE_DOWN) raise exception.InvalidBackup(reason=err) if not self._is_our_backup(backup): @@ -750,6 +850,9 @@ class BackupManager(manager.SchedulerDependentManager): except Exception as err: with excutils.save_and_reraise_exception(): volume_utils.update_backup_error(backup, str(err)) + self.message_api.create_from_request_context( + context, + detail=message_field.Detail.BACKUP_DELETE_DRIVER_ERROR) # Get reservations try: diff --git a/cinder/context.py b/cinder/context.py index 9118312ef78..123e6e5ebdd 100644 --- a/cinder/context.py +++ b/cinder/context.py @@ -108,6 +108,9 @@ class RequestContext(context.RequestContext): timestamp = timeutils.parse_isotime(timestamp) self.timestamp = timestamp self.quota_class = quota_class + self.message_resource_id = None + self.message_resource_type = None + self.message_action = None if service_catalog: # Only include required parts of service_catalog diff --git a/cinder/message/api.py b/cinder/message/api.py index 291ff19e7b5..141ca200faa 100644 --- a/cinder/message/api.py +++ b/cinder/message/api.py @@ -111,6 +111,40 @@ class API(base.Base): LOG.exception("Failed to create message record " "for request_id %s", context.request_id) + def create_from_request_context(self, context, exception=None, + detail=None, level="ERROR"): + """Create a message record with the specified information. + + :param context: + current context object which we must have populated with the + message_action, message_resource_type and message_resource_id + fields + :param exception: + if an exception has occurred, you can pass it in and it will be + translated into an appropriate message detail ID (possibly + message_field.Detail.UNKNOWN_ERROR). The message + in the exception itself is ignored in order not to expose + sensitive information to end users. Default is None + :param detail: + a message_field.Detail field describing the event the message + is about. Default is None, in which case + message_field.Detail.UNKNOWN_ERROR will be used for the message + unless an exception in the message_field.EXCEPTION_DETAIL_MAPPINGS + is passed; in that case the message_field.Detail field that's + mapped to the exception is used. + :param level: + a string describing the severity of the message. Suggested + values are 'INFO', 'ERROR', 'WARNING'. Default is 'ERROR'. + """ + + self.create(context=context, + action=context.message_action, + resource_type=context.message_resource_type, + resource_uuid=context.message_resource_id, + exception=exception, + detail=detail, + level=level) + def get(self, context, id): """Return message with the specified id.""" return self.db.message_get(context, id) diff --git a/cinder/message/message_field.py b/cinder/message/message_field.py index aa0aceab709..d12b63c6ef7 100644 --- a/cinder/message/message_field.py +++ b/cinder/message/message_field.py @@ -28,6 +28,7 @@ class Resource(object): VOLUME = 'VOLUME' VOLUME_SNAPSHOT = 'VOLUME_SNAPSHOT' + VOLUME_BACKUP = 'VOLUME_BACKUP' class Action(object): @@ -45,6 +46,9 @@ class Action(object): SNAPSHOT_DELETE = ('010', _('delete snapshot')) SNAPSHOT_UPDATE = ('011', _('update snapshot')) SNAPSHOT_METADATA_UPDATE = ('012', _('update snapshot metadata')) + BACKUP_CREATE = ('013', _('create backup')) + BACKUP_DELETE = ('014', _('delete backup')) + BACKUP_RESTORE = ('015', _('restore backup')) ALL = (SCHEDULE_ALLOCATE_VOLUME, ATTACH_VOLUME, @@ -58,6 +62,9 @@ class Action(object): SNAPSHOT_DELETE, SNAPSHOT_UPDATE, SNAPSHOT_METADATA_UPDATE, + BACKUP_CREATE, + BACKUP_DELETE, + BACKUP_RESTORE, ) @@ -100,6 +107,24 @@ class Detail(object): _("Volume snapshot update metadata failed.")) SNAPSHOT_IS_BUSY = ('015', _("Snapshot is busy.")) SNAPSHOT_DELETE_ERROR = ('016', _("Snapshot failed to delete.")) + BACKUP_INVALID_STATE = ('017', _("Backup status is invalid.")) + BACKUP_SERVICE_DOWN = ('018', _("Backup service is down.")) + BACKUP_CREATE_DEVICE_ERROR = ( + '019', _("Failed to get backup device from the volume service.")) + BACKUP_CREATE_DRIVER_ERROR = ( + '020', ("Backup driver failed to create backup.")) + ATTACH_ERROR = ('021', _("Failed to attach volume.")) + DETACH_ERROR = ('022', _("Failed to detach volume.")) + BACKUP_CREATE_CLEANUP_ERROR = ( + '023', _("Cleanup of temporary volume/snapshot failed.")) + BACKUP_SCHEDULE_ERROR = ( + '024', + ("Backup failed to schedule. Service not found for creating backup.")) + BACKUP_DELETE_DRIVER_ERROR = ( + '025', _("Backup driver failed to delete backup.")) + BACKUP_RESTORE_ERROR = ( + '026', _("Backup driver failed to restore backup.")) + VOLUME_INVALID_STATE = ('027', _("Volume status is invalid.")) ALL = (UNKNOWN_ERROR, DRIVER_NOT_INITIALIZED, @@ -117,6 +142,17 @@ class Detail(object): SNAPSHOT_UPDATE_METADATA_FAILED, SNAPSHOT_IS_BUSY, SNAPSHOT_DELETE_ERROR, + BACKUP_INVALID_STATE, + BACKUP_SERVICE_DOWN, + BACKUP_CREATE_DEVICE_ERROR, + BACKUP_CREATE_DRIVER_ERROR, + ATTACH_ERROR, + DETACH_ERROR, + BACKUP_CREATE_CLEANUP_ERROR, + BACKUP_SCHEDULE_ERROR, + BACKUP_DELETE_DRIVER_ERROR, + BACKUP_RESTORE_ERROR, + VOLUME_INVALID_STATE, ) # Exception and detail mappings diff --git a/cinder/scheduler/manager.py b/cinder/scheduler/manager.py index d16385685f1..3a41d05ce92 100644 --- a/cinder/scheduler/manager.py +++ b/cinder/scheduler/manager.py @@ -643,3 +643,9 @@ class SchedulerManager(manager.CleanableManager, manager.Manager): msg = "Service not found for creating backup." LOG.error(msg) vol_utils.update_backup_error(backup, msg) + self.message_api.create( + context, + action=message_field.Action.BACKUP_CREATE, + resource_type=message_field.Resource.VOLUME_BACKUP, + resource_uuid=backup.id, + detail=message_field.Detail.BACKUP_SCHEDULE_ERROR) diff --git a/cinder/tests/unit/backup/test_backup_messages.py b/cinder/tests/unit/backup/test_backup_messages.py new file mode 100644 index 00000000000..7f2c1ad9aeb --- /dev/null +++ b/cinder/tests/unit/backup/test_backup_messages.py @@ -0,0 +1,624 @@ +# Copyright 2021, Red Hat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Tests for User Facing Messages in Backup Operations.""" + +from unittest import mock + +from cinder.backup import manager as backup_manager +from cinder import exception +from cinder.message import message_field +from cinder.scheduler import manager as sch_manager +from cinder.tests.unit import fake_constants as fake +from cinder.tests.unit import test + + +class BackupUserMessagesTest(test.TestCase): + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager._run_backup') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + def test_backup_create_invalid_status( + self, mock_notify, mock_working, mock_run, + mock_msg_create, mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='available', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = {'status': 'backing-up'}.__getitem__ + mock_get_vol.return_value = mock_vol + + self.assertRaises( + exception.InvalidBackup, manager.create_backup, fake_context, + fake_backup) + self.assertEqual(message_field.Action.BACKUP_CREATE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_INVALID_STATE) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager._run_backup') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + def test_backup_create_service_down( + self, mock_notify, mock_working, mock_run, mock_msg_create, + mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = {'status': 'backing-up'}.__getitem__ + mock_get_vol.return_value = mock_vol + mock_working.return_value = False + + mock_run.side_effect = exception.InvalidBackup(reason='test reason') + self.assertRaises( + exception.InvalidBackup, manager.create_backup, fake_context, + fake_backup) + self.assertEqual(message_field.Action.BACKUP_CREATE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_SERVICE_DOWN) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.get_backup_device') + @mock.patch('cinder.backup.manager.BackupManager.' + '_cleanup_temp_volumes_snapshots_when_backup_created') + def test_backup_create_device_error( + self, mock_cleanup, mock_get_bak_dev, mock_get_conn, mock_notify, + mock_working, mock_msg_create, mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = {'status': 'backing-up'}.__getitem__ + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_get_bak_dev.side_effect = exception.InvalidVolume( + reason="test reason") + + self.assertRaises(exception.InvalidVolume, manager.create_backup, + fake_context, fake_backup) + self.assertEqual(message_field.Action.BACKUP_CREATE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_CREATE_DEVICE_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.get_backup_device') + @mock.patch('cinder.backup.manager.BackupManager.' + '_cleanup_temp_volumes_snapshots_when_backup_created') + @mock.patch('cinder.backup.manager.BackupManager._attach_device') + def test_backup_create_attach_error( + self, mock_attach, mock_cleanup, mock_get_bak_dev, mock_get_conn, + mock_notify, mock_working, mock_msg_create, mock_get_vol, + mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = {'status': 'backing-up'}.__getitem__ + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_attach.side_effect = exception.InvalidVolume(reason="test reason") + + self.assertRaises(exception.InvalidVolume, manager.create_backup, + fake_context, fake_backup) + self.assertEqual(message_field.Action.BACKUP_CREATE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.ATTACH_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.get_backup_device') + @mock.patch('cinder.backup.manager.BackupManager.' + '_cleanup_temp_volumes_snapshots_when_backup_created') + @mock.patch('cinder.backup.manager.BackupManager._attach_device') + @mock.patch( + 'cinder.tests.unit.backup.fake_service.FakeBackupService.backup') + @mock.patch('cinder.backup.manager.open') + @mock.patch('cinder.backup.manager.BackupManager._detach_device') + def test_backup_create_driver_error( + self, mock_detach, mock_open, mock_backup, mock_attach, + mock_cleanup, mock_get_bak_dev, mock_get_conn, mock_notify, + mock_working, mock_msg_create, mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = {'status': 'backing-up'}.__getitem__ + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_attach.return_value = {'device': {'path': '/dev/sdb'}} + mock_backup.side_effect = exception.InvalidBackup(reason="test reason") + + self.assertRaises(exception.InvalidBackup, manager.create_backup, + fake_context, fake_backup) + self.assertEqual(message_field.Action.BACKUP_CREATE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_CREATE_DRIVER_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.get_backup_device') + @mock.patch('cinder.backup.manager.BackupManager.' + '_cleanup_temp_volumes_snapshots_when_backup_created') + @mock.patch('cinder.backup.manager.BackupManager._attach_device') + @mock.patch( + 'cinder.tests.unit.backup.fake_service.FakeBackupService.backup') + @mock.patch('cinder.backup.manager.open') + @mock.patch('cinder.backup.manager.BackupManager._detach_device') + def test_backup_create_detach_error( + self, mock_detach, mock_open, mock_backup, mock_attach, + mock_cleanup, mock_get_bak_dev, mock_get_conn, mock_notify, + mock_working, mock_msg_create, mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = {'status': 'backing-up'}.__getitem__ + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_attach.return_value = {'device': {'path': '/dev/sdb'}} + mock_detach.side_effect = exception.InvalidVolume(reason="test reason") + + self.assertRaises(exception.InvalidVolume, manager.create_backup, + fake_context, fake_backup) + self.assertEqual(message_field.Action.BACKUP_CREATE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.DETACH_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch('cinder.volume.rpcapi.VolumeAPI.get_backup_device') + @mock.patch('cinder.backup.manager.BackupManager.' + '_cleanup_temp_volumes_snapshots_when_backup_created') + @mock.patch('cinder.backup.manager.BackupManager._attach_device') + @mock.patch( + 'cinder.tests.unit.backup.fake_service.FakeBackupService.backup') + @mock.patch('cinder.backup.manager.open') + @mock.patch('cinder.backup.manager.BackupManager._detach_device') + def test_backup_create_cleanup_error( + self, mock_detach, mock_open, mock_backup, mock_attach, + mock_cleanup, mock_get_bak_dev, mock_get_conn, mock_notify, + mock_working, mock_msg_create, mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = {'status': 'backing-up'}.__getitem__ + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_attach.return_value = {'device': {'path': '/dev/sdb'}} + mock_cleanup.side_effect = exception.InvalidVolume( + reason="test reason") + + self.assertRaises(exception.InvalidVolume, manager.create_backup, + fake_context, fake_backup) + self.assertEqual(message_field.Action.BACKUP_CREATE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_CREATE_CLEANUP_ERROR) + + @mock.patch('cinder.scheduler.host_manager.HostManager.' + '_get_available_backup_service_host') + @mock.patch('cinder.volume.volume_utils.update_backup_error') + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.db.volume_get') + @mock.patch('cinder.message.api.API.create') + def test_backup_create_scheduling_error( + self, mock_msg_create, mock_get_vol, mock_vol_update, + mock_update_error, mock_get_backup_host): + manager = sch_manager.SchedulerManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock(id=fake.BACKUP_ID, + volume_id=fake.VOLUME_ID) + mock_get_vol.return_value = mock.MagicMock() + exception.ServiceNotFound(service_id='cinder-backup') + mock_get_backup_host.side_effect = exception.ServiceNotFound( + service_id='cinder-backup') + + manager.create_backup(fake_context, fake_backup) + mock_msg_create.assert_called_once_with( + fake_context, + action=message_field.Action.BACKUP_CREATE, + resource_type=message_field.Resource.VOLUME_BACKUP, + resource_uuid=fake_backup.id, + detail=message_field.Detail.BACKUP_SCHEDULE_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch( + 'cinder.backup.manager.BackupManager._notify_about_backup_usage') + def test_backup_delete_invalid_state( + self, mock_notify, mock_msg_create, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='available', volume_id=fake.VOLUME_ID, + snapshot_id=None) + + self.assertRaises( + exception.InvalidBackup, manager.delete_backup, fake_context, + fake_backup) + self.assertEqual(message_field.Action.BACKUP_DELETE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_INVALID_STATE) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch( + 'cinder.backup.manager.BackupManager._notify_about_backup_usage') + def test_backup_delete_service_down( + self, mock_notify, mock_working, mock_msg_create, + mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='deleting', volume_id=fake.VOLUME_ID, + snapshot_id=None) + mock_working.return_value = False + + self.assertRaises( + exception.InvalidBackup, manager.delete_backup, fake_context, + fake_backup) + self.assertEqual(message_field.Action.BACKUP_DELETE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_SERVICE_DOWN) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager._is_our_backup') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch( + 'cinder.backup.manager.BackupManager._notify_about_backup_usage') + def test_backup_delete_driver_error( + self, mock_notify, mock_working, mock_our_back, + mock_msg_create, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='deleting', volume_id=fake.VOLUME_ID, + snapshot_id=None) + fake_backup.__getitem__.side_effect = ( + {'display_name': 'fail_on_delete'}.__getitem__) + mock_working.return_value = True + mock_our_back.return_value = True + + self.assertRaises( + IOError, manager.delete_backup, fake_context, + fake_backup) + self.assertEqual(message_field.Action.BACKUP_DELETE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_DELETE_DRIVER_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + def test_backup_restore_volume_invalid_state( + self, mock_notify, mock_msg_create, mock_get_vol, + mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + fake_backup.__getitem__.side_effect = ( + {'status': 'restoring', 'size': 1}.__getitem__) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = ( + {'id': fake.VOLUME_ID, 'status': 'available', + 'size': 1}.__getitem__) + mock_get_vol.return_value = mock_vol + + self.assertRaises( + exception.InvalidVolume, manager.restore_backup, + fake_context, fake_backup, fake.VOLUME_ID) + mock_msg_create.assert_called_once_with( + fake_context, + action=message_field.Action.BACKUP_RESTORE, + resource_type=message_field.Resource.VOLUME_BACKUP, + resource_uuid=mock_vol.id, + detail=message_field.Detail.VOLUME_INVALID_STATE) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + def test_backup_restore_backup_invalid_state( + self, mock_notify, mock_msg_create, mock_get_vol, + mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + fake_backup.__getitem__.side_effect = ( + {'status': 'available', 'size': 1}.__getitem__) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = ( + {'status': 'restoring-backup', 'size': 1}.__getitem__) + mock_get_vol.return_value = mock_vol + + self.assertRaises( + exception.InvalidBackup, manager.restore_backup, + fake_context, fake_backup, fake.VOLUME_ID) + self.assertEqual(message_field.Action.BACKUP_RESTORE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_INVALID_STATE) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager._is_our_backup') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch( + 'cinder.volume.rpcapi.VolumeAPI.secure_file_operations_enabled') + @mock.patch('cinder.backup.manager.BackupManager._attach_device') + @mock.patch('cinder.backup.manager.BackupManager._detach_device') + def test_backup_restore_attach_error( + self, mock_detach, mock_attach, mock_sec_opts, mock_get_conn, + mock_notify, mock_working, mock_our_back, mock_msg_create, + mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + fake_backup.__getitem__.side_effect = ( + {'status': 'restoring', 'size': 1}.__getitem__) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = ( + {'status': 'restoring-backup', 'size': 1}.__getitem__) + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_our_back.return_value = True + mock_attach.side_effect = exception.InvalidBackup( + reason="test reason") + + self.assertRaises( + exception.InvalidBackup, manager.restore_backup, + fake_context, fake_backup, fake.VOLUME_ID) + self.assertEqual(message_field.Action.BACKUP_RESTORE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.ATTACH_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager._is_our_backup') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch( + 'cinder.volume.rpcapi.VolumeAPI.secure_file_operations_enabled') + @mock.patch('cinder.backup.manager.BackupManager._attach_device') + @mock.patch('cinder.backup.manager.open') + @mock.patch( + 'cinder.tests.unit.backup.fake_service.FakeBackupService.restore') + @mock.patch('cinder.backup.manager.BackupManager._detach_device') + def test_backup_restore_driver_error( + self, mock_detach, mock_restore, mock_open, mock_attach, + mock_sec_opts, mock_get_conn, mock_notify, mock_working, + mock_our_back, mock_msg_create, mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + fake_backup.__getitem__.side_effect = ( + {'status': 'restoring', 'size': 1}.__getitem__) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = ( + {'status': 'restoring-backup', 'size': 1}.__getitem__) + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_our_back.return_value = True + mock_attach.return_value = {'device': {'path': '/dev/sdb'}} + mock_restore.side_effect = exception.InvalidBackup( + reason="test reason") + + self.assertRaises( + exception.InvalidBackup, manager.restore_backup, + fake_context, fake_backup, fake.VOLUME_ID) + self.assertEqual(message_field.Action.BACKUP_RESTORE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.BACKUP_RESTORE_ERROR) + + @mock.patch('cinder.db.volume_update') + @mock.patch('cinder.objects.volume.Volume.get_by_id') + @mock.patch('cinder.message.api.API.create_from_request_context') + @mock.patch('cinder.backup.manager.BackupManager._is_our_backup') + @mock.patch('cinder.backup.manager.BackupManager.is_working') + @mock.patch('cinder.backup.manager.BackupManager.' + '_notify_about_backup_usage') + @mock.patch( + 'cinder.backup.manager.volume_utils.brick_get_connector_properties') + @mock.patch( + 'cinder.volume.rpcapi.VolumeAPI.secure_file_operations_enabled') + @mock.patch('cinder.backup.manager.BackupManager._attach_device') + @mock.patch('cinder.backup.manager.open') + @mock.patch( + 'cinder.tests.unit.backup.fake_service.FakeBackupService.restore') + @mock.patch('cinder.backup.manager.BackupManager._detach_device') + def test_backup_restore_detach_error( + self, mock_detach, mock_restore, mock_open, mock_attach, + mock_sec_opts, mock_get_conn, mock_notify, mock_working, + mock_our_back, mock_msg_create, mock_get_vol, mock_vol_update): + manager = backup_manager.BackupManager() + fake_context = mock.MagicMock() + fake_backup = mock.MagicMock( + id=fake.BACKUP_ID, status='creating', volume_id=fake.VOLUME_ID, + snapshot_id=None) + fake_backup.__getitem__.side_effect = ( + {'status': 'restoring', 'size': 1}.__getitem__) + mock_vol = mock.MagicMock() + mock_vol.__getitem__.side_effect = ( + {'status': 'restoring-backup', 'size': 1}.__getitem__) + mock_get_vol.return_value = mock_vol + mock_working.return_value = True + mock_our_back.return_value = True + mock_attach.return_value = {'device': {'path': '/dev/sdb'}} + mock_detach.side_effect = exception.InvalidBackup( + reason="test reason") + + self.assertRaises( + exception.InvalidBackup, manager.restore_backup, + fake_context, fake_backup, fake.VOLUME_ID) + self.assertEqual(message_field.Action.BACKUP_RESTORE, + fake_context.message_action) + self.assertEqual(message_field.Resource.VOLUME_BACKUP, + fake_context.message_resource_type) + self.assertEqual(fake_backup.id, + fake_context.message_resource_id) + mock_msg_create.assert_called_with( + fake_context, + detail=message_field.Detail.DETACH_ERROR) diff --git a/cinder/tests/unit/message/test_api.py b/cinder/tests/unit/message/test_api.py index fd7f9dfba68..ed43713fb24 100644 --- a/cinder/tests/unit/message/test_api.py +++ b/cinder/tests/unit/message/test_api.py @@ -279,6 +279,35 @@ class MessageApiTest(test.TestCase): self.message_api.db.message_create.assert_called_once_with( self.ctxt, mock.ANY) + @mock.patch('oslo_utils.timeutils.utcnow') + def test_create_from_request_context(self, mock_utcnow): + CONF.set_override('message_ttl', 300) + mock_utcnow.return_value = datetime.datetime.utcnow() + expected_expires_at = timeutils.utcnow() + datetime.timedelta( + seconds=300) + + self.ctxt.message_resource_id = 'fake-uuid' + self.ctxt.message_resource_type = 'fake_resource_type' + self.ctxt.message_action = message_field.Action.BACKUP_CREATE + expected_message_record = { + 'project_id': 'fakeproject', + 'request_id': 'fakerequestid', + 'resource_type': 'fake_resource_type', + 'resource_uuid': 'fake-uuid', + 'action_id': message_field.Action.BACKUP_CREATE[0], + 'detail_id': message_field.Detail.BACKUP_INVALID_STATE[0], + 'message_level': 'ERROR', + 'expires_at': expected_expires_at, + 'event_id': "VOLUME_fake_resource_type_013_017", + } + self.message_api.create_from_request_context( + self.ctxt, + detail=message_field.Detail.BACKUP_INVALID_STATE) + + self.message_api.db.message_create.assert_called_once_with( + self.ctxt, expected_message_record) + mock_utcnow.assert_called_with() + def test_get(self): self.message_api.get(self.ctxt, 'fake_id') diff --git a/releasenotes/notes/backup-user-messages-5ee0c7ead3def8f9.yaml b/releasenotes/notes/backup-user-messages-5ee0c7ead3def8f9.yaml new file mode 100644 index 00000000000..08094f3e7dc --- /dev/null +++ b/releasenotes/notes/backup-user-messages-5ee0c7ead3def8f9.yaml @@ -0,0 +1,8 @@ +--- +other: + - | + Added user messages for backup operations that a user + can query through the `Messages API + `_. + These allow users to retrieve error messages for asynchronous + failures in backup operations like create, delete, and restore. \ No newline at end of file