From 6fdf11ea7f5c77e83dd746fa33b7a354417aec08 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Tue, 16 Feb 2021 12:23:19 +1300 Subject: [PATCH] Support to restore backup from remote location In multi-region deployment with geo-replicated Swift, the user can restore a backup in one region by manually specifying the original backup data location created in another region. Change-Id: Iefef3bf969163af707935445bc23299400dc88c3 --- api-ref/source/backups.inc | 14 ++ api-ref/source/parameters.yaml | 13 +- doc/source/user/backup-db.rst | 18 +++ .../notes/wallaby-restore-backup.yaml | 5 + trove/backup/models.py | 128 ++++++++++++------ trove/backup/service.py | 23 ++-- trove/backup/state.py | 3 +- trove/common/apischema.py | 17 ++- trove/common/constants.py | 16 +++ trove/common/swift.py | 42 ++++++ trove/instance/models.py | 8 +- trove/taskmanager/models.py | 7 +- .../unittests/backup/test_backup_models.py | 4 +- trove/tests/unittests/backup/test_service.py | 84 ++++++++++++ .../unittests/taskmanager/test_models.py | 10 +- 15 files changed, 328 insertions(+), 64 deletions(-) create mode 100644 releasenotes/notes/wallaby-restore-backup.yaml create mode 100644 trove/common/constants.py create mode 100644 trove/common/swift.py create mode 100644 trove/tests/unittests/backup/test_service.py diff --git a/api-ref/source/backups.inc b/api-ref/source/backups.inc index a9a086c22a..2e3d56f0c6 100644 --- a/api-ref/source/backups.inc +++ b/api-ref/source/backups.inc @@ -76,6 +76,19 @@ In the Trove deployment with service tenant enabled, The backup data is stored as objects in OpenStack Swift service in the user's container. If not specified, the container name is defined by the cloud admin. +The user can create a backup strategy within the project scope or specific to +a particular instance. + +In multi-region deployment with geo-replicated Swift, the user can also restore +a backup in a region by manually specifying the backup data location created in +another region, then create instances from the backup. Instance ID is not +required in this case. + +.. warning:: + + The restored backup is dependent on the original backup data, if the + original backup is deleted, the restored backup is invalid. + Normal response codes: 202 Request @@ -90,6 +103,7 @@ Request - incremental: backup_incremental - description: backup_description - swift_container: swift_container + - restore_from: backup_restore_from Request Example --------------- diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index 6918f2e4a1..a2c6c91035 100755 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -148,7 +148,7 @@ backup_instanceId: description: | The ID of the instance to create backup for. in: body - required: true + required: false type: string backup_list: description: | @@ -180,6 +180,17 @@ backup_parentId1: in: body required: true type: string +backup_restore_from: + description: | + The information needed to restore a backup, including: + + - ``remote_location``: The original backup data location. + - ``local_datastore_version_id``: The local datastore version corresponding + to the original backup. + - ``size``: The original backup size. + in: body + required: false + type: object backup_size: description: | Size of the backup, the unit is GB. diff --git a/doc/source/user/backup-db.rst b/doc/source/user/backup-db.rst index 9a1a10e5b5..9aa5e13b53 100644 --- a/doc/source/user/backup-db.rst +++ b/doc/source/user/backup-db.rst @@ -276,3 +276,21 @@ Create an incremental backup based on a parent backup: | status | NEW | | updated | 2014-03-19T14:09:13 | +-------------+--------------------------------------+ + +Restore backup from other regions +--------------------------------- + +Restoring backup from other regions were introduced in Wallaby, + +In multi-region deployment with geo-replicated Swift, the user is able to +create a backup in one region using the backup data created in the others, +which is useful in Disaster Recovery scenario. Instance ID is not required in +this case when restoring backup, but the original backup data location (a swift +object URL), the local datastore version and the backup data size are required. + +.. warning:: + + The restored backup is dependent on the original backup data, if the + original backup is deleted, the restored backup is invalid. + +TODO: Add CLI example once supported in python-troveclient. diff --git a/releasenotes/notes/wallaby-restore-backup.yaml b/releasenotes/notes/wallaby-restore-backup.yaml new file mode 100644 index 0000000000..f7e369e720 --- /dev/null +++ b/releasenotes/notes/wallaby-restore-backup.yaml @@ -0,0 +1,5 @@ +--- +features: + - In multi-region deployment with geo-replicated Swift, the user can restore + a backup in one region by manually specifying the original backup data + location created in another region. diff --git a/trove/backup/models.py b/trove/backup/models.py index 10c58ad2fb..5903b1b66c 100644 --- a/trove/backup/models.py +++ b/trove/backup/models.py @@ -22,7 +22,9 @@ from swiftclient.client import ClientException from trove.backup.state import BackupState from trove.common import cfg from trove.common import clients +from trove.common import constants from trove.common import exception +from trove.common import swift from trove.common import utils from trove.common.i18n import _ from trove.datastore import models as datastore_models @@ -49,7 +51,8 @@ class Backup(object): @classmethod def create(cls, context, instance, name, description=None, - parent_id=None, incremental=False, swift_container=None): + parent_id=None, incremental=False, swift_container=None, + restore_from=None): """ create db record for Backup :param cls: @@ -59,31 +62,61 @@ class Backup(object): :param description: :param parent_id: :param incremental: flag to indicate incremental backup - based on previous backup + based on previous backup :param swift_container: Swift container name. + :param restore_from: A dict that contains backup information of another + region. :return: """ + backup_state = BackupState.NEW + checksum = None + instance_id = None + parent = None + last_backup_id = None + location = None + backup_type = constants.BACKUP_TYPE_FULL + size = None - def _create_resources(): - # parse the ID from the Ref + if restore_from: + # Check location and datastore version. + LOG.info(f"Restoring backup, restore_from: {restore_from}") + backup_state = BackupState.RESTORED + + ds_version_id = restore_from.get('local_datastore_version_id') + ds_version = datastore_models.DatastoreVersion.load_by_uuid( + ds_version_id) + + location = restore_from.get('remote_location') + swift_client = clients.create_swift_client(context) + try: + obj_meta = swift.get_metadata(swift_client, location, + extra_attrs=['etag']) + except Exception: + msg = f'Failed to restore backup from {location}' + LOG.exception(msg) + raise exception.BackupCreationError(msg) + + checksum = obj_meta['etag'] + if 'parent_location' in obj_meta: + backup_type = constants.BACKUP_TYPE_INC + + size = restore_from['size'] + else: instance_id = utils.get_id_from_href(instance) - - # verify that the instance exists and can perform actions - from trove.instance.models import Instance - instance_model = Instance.load(context, instance_id) + # Import here to avoid circular imports. + from trove.instance import models as inst_model + instance_model = inst_model.Instance.load(context, instance_id) instance_model.validate_can_perform_action() - cls.validate_can_perform_action( - instance_model, 'backup_create') - - cls.verify_swift_auth_token(context) - if instance_model.cluster_id is not None: raise exception.ClusterInstanceOperationNotSupported() + cls.validate_can_perform_action(instance_model, 'backup_create') + + cls.verify_swift_auth_token(context) + ds = instance_model.datastore ds_version = instance_model.datastore_version - parent = None - last_backup_id = None + if parent_id: # Look up the parent info or fail early if not found or if # the user does not have access to the parent. @@ -100,36 +133,53 @@ class Backup(object): 'checksum': _parent.checksum } last_backup_id = _parent.id + + if parent: + backup_type = constants.BACKUP_TYPE_INC + + def _create_resources(): try: - db_info = DBBackup.create(name=name, - description=description, - tenant_id=context.project_id, - state=BackupState.NEW, - instance_id=instance_id, - parent_id=parent_id or - last_backup_id, - datastore_version_id=ds_version.id, - deleted=False) + db_info = DBBackup.create( + name=name, + description=description, + tenant_id=context.project_id, + state=backup_state, + instance_id=instance_id, + parent_id=parent_id or last_backup_id, + datastore_version_id=ds_version.id, + deleted=False, + location=location, + checksum=checksum, + backup_type=backup_type, + size=size + ) except exception.InvalidModelError as ex: LOG.exception("Unable to create backup record for " "instance: %s", instance_id) raise exception.BackupCreationError(str(ex)) - backup_info = {'id': db_info.id, - 'name': name, - 'description': description, - 'instance_id': instance_id, - 'backup_type': db_info.backup_type, - 'checksum': db_info.checksum, - 'parent': parent, - 'datastore': ds.name, - 'datastore_version': ds_version.name, - 'swift_container': swift_container - } - api.API(context).create_backup(backup_info, instance_id) + if not restore_from: + backup_info = { + 'id': db_info.id, + 'name': name, + 'description': description, + 'instance_id': instance_id, + 'backup_type': db_info.backup_type, + 'checksum': db_info.checksum, + 'parent': parent, + 'datastore': ds.name, + 'datastore_version': ds_version.name, + 'swift_container': swift_container + } + api.API(context).create_backup(backup_info, instance_id) + else: + context.notification.payload.update( + {'backup_id': db_info.id} + ) + return db_info - return run_with_quotas(context.project_id, - {'backups': 1}, + + return run_with_quotas(context.project_id, {'backups': 1}, _create_resources) @classmethod @@ -372,7 +422,7 @@ class DBBackup(DatabaseModelBase): @property def is_done_successfuly(self): - return self.state == BackupState.COMPLETED + return self.state in [BackupState.COMPLETED, BackupState.RESTORED] @property def filename(self): diff --git a/trove/backup/service.py b/trove/backup/service.py index ed783199ab..8b3d9d0f9d 100644 --- a/trove/backup/service.py +++ b/trove/backup/service.py @@ -80,27 +80,32 @@ class BackupController(wsgi.Controller): context = req.environ[wsgi.CONTEXT_KEY] policy.authorize_on_tenant(context, 'backup:create') data = body['backup'] - instance = data['instance'] + instance = data.get('instance') name = data['name'] desc = data.get('description') parent = data.get('parent_id') incremental = data.get('incremental') swift_container = data.get('swift_container') + restore_from = data.get('restore_from') - context.notification = notification.DBaaSBackupCreate(context, - request=req) + context.notification = notification.DBaaSBackupCreate( + context, request=req) - if not swift_container: - instance_id = utils.get_id_from_href(instance) - backup_strategy = BackupStrategy.get(context, instance_id) - if backup_strategy: - swift_container = backup_strategy.swift_container + if not restore_from: + if not instance: + raise exception.BackupCreationError('instance is missing.') + if not swift_container: + instance_id = utils.get_id_from_href(instance) + backup_strategy = BackupStrategy.get(context, instance_id) + if backup_strategy: + swift_container = backup_strategy.swift_container with StartNotification(context, name=name, instance_id=instance, description=desc, parent_id=parent): backup = Backup.create(context, instance, name, desc, parent_id=parent, incremental=incremental, - swift_container=swift_container) + swift_container=swift_container, + restore_from=restore_from) return wsgi.Result(views.BackupView(backup).data(), 202) diff --git a/trove/backup/state.py b/trove/backup/state.py index b0c4676622..ce6e1588db 100644 --- a/trove/backup/state.py +++ b/trove/backup/state.py @@ -21,6 +21,7 @@ class BackupState(object): SAVING = "SAVING" COMPLETED = "COMPLETED" FAILED = "FAILED" + RESTORED = "RESTORED" DELETE_FAILED = "DELETE_FAILED" RUNNING_STATES = [NEW, BUILDING, SAVING] - END_STATES = [COMPLETED, FAILED, DELETE_FAILED] + END_STATES = [COMPLETED, FAILED, DELETE_FAILED, RESTORED] diff --git a/trove/common/apischema.py b/trove/common/apischema.py index 4c799992cb..bc815fadab 100644 --- a/trove/common/apischema.py +++ b/trove/common/apischema.py @@ -652,14 +652,27 @@ backup = { "properties": { "backup": { "type": "object", - "required": ["instance", "name"], + "required": ["name"], "properties": { "description": non_empty_string, "instance": uuid, "name": non_empty_string, "parent_id": uuid, "incremental": boolean_string, - "swift_container": non_empty_string + "swift_container": non_empty_string, + "restore_from": { + "type": "object", + "required": [ + "remote_location", + "local_datastore_version_id", + "size" + ], + "properties": { + "remote_location": non_empty_string, + "local_datastore_version_id": uuid, + "size": {"type": "number"} + } + } } } } diff --git a/trove/common/constants.py b/trove/common/constants.py new file mode 100644 index 0000000000..0f53477b24 --- /dev/null +++ b/trove/common/constants.py @@ -0,0 +1,16 @@ +# Copyright 2021 Catalyst Cloud Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BACKUP_TYPE_FULL = 'full' +BACKUP_TYPE_INC = 'incremental' diff --git a/trove/common/swift.py b/trove/common/swift.py new file mode 100644 index 0000000000..43910af1d7 --- /dev/null +++ b/trove/common/swift.py @@ -0,0 +1,42 @@ +# Copyright 2021 Catalyst Cloud Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def parse_location(location): + storage_url = "/".join(location.split('/')[:-2]) + container_name = location.split('/')[-2] + object_name = location.split('/')[-1] + return storage_url, container_name, object_name + + +def _get_attr(original): + """Get a friendly name from an object header key.""" + key = original.replace('-', '_') + key = key.replace('x_object_meta_', '') + return key + + +def get_metadata(client, location, extra_attrs=[]): + _, container_name, object_name = parse_location(location) + headers = client.head_object(container_name, object_name) + + meta = {} + for key, value in headers.items(): + if key.startswith('x-object-meta'): + meta[_get_attr(key)] = value + + for key in extra_attrs: + meta[key] = headers.get(key) + + return meta diff --git a/trove/instance/models.py b/trove/instance/models.py index b04e3ae68a..e00bff0c1c 100644 --- a/trove/instance/models.py +++ b/trove/instance/models.py @@ -351,10 +351,10 @@ class SimpleInstance(object): - Then server status - Otherwise, unknown """ - LOG.info(f"Getting instance status for {self.id}, " - f"task status: {self.db_info.task_status}, " - f"datastore status: {self.datastore_status.status}, " - f"server status: {self.db_info.server_status}") + LOG.debug(f"Getting instance status for {self.id}, " + f"task status: {self.db_info.task_status}, " + f"datastore status: {self.datastore_status.status}, " + f"server status: {self.db_info.server_status}") task_status = self.db_info.task_status server_status = self.db_info.server_status diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index e5e0026f63..4acb92034c 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -1470,8 +1470,7 @@ class BackupTasks(object): def _delete(backup): backup.deleted = True backup.deleted_at = timeutils.utcnow() - # Set datastore_version_id to None so that datastore_version could - # be deleted. + # Set datastore_version_id to None to remove dependency. backup.datastore_version_id = None backup.save() @@ -1479,7 +1478,9 @@ class BackupTasks(object): backup = bkup_models.Backup.get_by_id(context, backup_id) try: filename = backup.filename - if filename: + # Do not remove the object if the backup was restored from remote + # location. + if filename and backup.state != bkup_models.BackupState.RESTORED: BackupTasks.delete_files_from_swift(context, backup.container_name, filename) diff --git a/trove/tests/unittests/backup/test_backup_models.py b/trove/tests/unittests/backup/test_backup_models.py index 2f97b4519d..ffa205dd87 100644 --- a/trove/tests/unittests/backup/test_backup_models.py +++ b/trove/tests/unittests/backup/test_backup_models.py @@ -176,7 +176,7 @@ class BackupCreateTest(trove_testtools.TestCase): BACKUP_NAME, BACKUP_DESC) def test_create_backup_swift_token_invalid(self): - instance = MagicMock() + instance = MagicMock(cluster_id=None) with patch.object(instance_models.BuiltInstance, 'load', return_value=instance): instance.validate_can_perform_action = MagicMock( @@ -191,7 +191,7 @@ class BackupCreateTest(trove_testtools.TestCase): BACKUP_NAME, BACKUP_DESC) def test_create_backup_datastore_operation_not_supported(self): - instance = MagicMock() + instance = MagicMock(cluster_id=None) with patch.object(instance_models.BuiltInstance, 'load', return_value=instance): with patch.object( diff --git a/trove/tests/unittests/backup/test_service.py b/trove/tests/unittests/backup/test_service.py new file mode 100644 index 0000000000..9be60d5fa4 --- /dev/null +++ b/trove/tests/unittests/backup/test_service.py @@ -0,0 +1,84 @@ +# Copyright 2021 Catalyst Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest import mock + +from trove.backup import service +from trove.backup.state import BackupState +from trove.common import context +from trove.common import wsgi +from trove.datastore import models as ds_models +from trove.tests.unittests import trove_testtools +from trove.tests.unittests.util import util + + +class TestBackupController(trove_testtools.TestCase): + @classmethod + def setUpClass(cls): + util.init_db() + + cls.ds_name = cls.random_name('datastore', + prefix='TestBackupController') + ds_models.update_datastore(name=cls.ds_name, default_version=None) + cls.ds = ds_models.Datastore.load(cls.ds_name) + + ds_models.update_datastore_version( + cls.ds_name, 'fake-ds-version', 'mysql', '', ['trove', 'mysql'], + '', 1) + cls.ds_version = ds_models.DatastoreVersion.load( + cls.ds, 'fake-ds-version') + + cls.controller = service.BackupController() + + super(TestBackupController, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + util.cleanup_db() + super(TestBackupController, cls).tearDownClass() + + def setUp(self): + trove_testtools.patch_notifier(self) + self.context = context.TroveContext(project_id=self.random_uuid()) + + super(TestBackupController, self).setUp() + + @mock.patch('trove.common.clients.create_swift_client') + def test_create_restore_from(self, mock_swift_client): + swift_client = mock.MagicMock() + swift_client.head_object.return_value = {'etag': 'fake-etag'} + mock_swift_client.return_value = swift_client + + req = mock.MagicMock(environ={wsgi.CONTEXT_KEY: self.context}) + + name = self.random_name( + name='backup', prefix='TestBackupController') + body = { + 'backup': { + "name": name, + "restore_from": { + "remote_location": "http://192.168.206.8:8080/v1/" + "AUTH_055b2fb9a2264ae5a5f6b3cc066c4a1d/" + "fake-container/fake-object", + "local_datastore_version_id": self.ds_version.id, + "size": 0.2 + } + } + } + ret = self.controller.create(req, body, self.context.project_id) + self.assertEqual(202, ret.status) + + ret_backup = ret.data(None)['backup'] + + self.assertEqual(BackupState.RESTORED, ret_backup.get('status')) + self.assertEqual(name, ret_backup.get('name')) diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py index 353f82f4b2..b81194a36c 100644 --- a/trove/tests/unittests/taskmanager/test_models.py +++ b/trove/tests/unittests/taskmanager/test_models.py @@ -1037,10 +1037,8 @@ class BackupTasksTest(trove_testtools.TestCase): self.assertTrue(self.backup.deleted) - @patch('trove.taskmanager.models.LOG') @patch('trove.common.clients.create_swift_client') - def test_delete_backup_fail_delete_manifest(self, mock_swift_client, - mock_logging): + def test_delete_backup_fail_delete_manifest(self, mock_swift_client): client_mock = MagicMock() client_mock.head_object.return_value = {} client_mock.delete_object.side_effect = ClientException("foo") @@ -1070,6 +1068,12 @@ class BackupTasksTest(trove_testtools.TestCase): client_mock.delete_object.assert_called_once_with('container', '12e48.xbstream.gz') + def test_delete_backup_restored(self): + self.backup.state = state.BackupState.RESTORED + taskmanager_models.BackupTasks.delete_backup(mock.ANY, self.backup.id) + + self.assertTrue(self.backup.deleted) + def test_parse_manifest(self): manifest = 'container/prefix' cont, prefix = taskmanager_models.BackupTasks._parse_manifest(manifest)