Support to restore backup from remote location

In multi-region deployment with geo-replicated Swift, the user can
restore a backup in one region by manually specifying the original
backup data location created in another region.

Change-Id: Iefef3bf969163af707935445bc23299400dc88c3

api-ref/source/backups.inc

@@ -76,6 +76,19 @@ In the Trove deployment with service tenant enabled, The backup data is
 stored as objects in OpenStack Swift service in the user's container. If not
 specified, the container name is defined by the cloud admin.
 
+The user can create a backup strategy within the project scope or specific to
+a particular instance.
+
+In multi-region deployment with geo-replicated Swift, the user can also restore
+a backup in a region by manually specifying the backup data location created in
+another region, then create instances from the backup. Instance ID is not
+required in this case.
+
+.. warning::
+
+   The restored backup is dependent on the original backup data, if the
+   original backup is deleted, the restored backup is invalid.
+
 Normal response codes: 202
 
 Request
@@ -90,6 +103,7 @@ Request
    - incremental: backup_incremental
    - description: backup_description
    - swift_container: swift_container
+   - restore_from: backup_restore_from
 
 Request Example
 ---------------

api-ref/source/parameters.yaml

@@ -148,7 +148,7 @@ backup_instanceId:
   description: |
     The ID of the instance to create backup for.
   in: body
-  required: true
+  required: false
   type: string
 backup_list:
   description: |
@@ -180,6 +180,17 @@ backup_parentId1:
   in: body
   required: true
   type: string
+backup_restore_from:
+  description: |
+    The information needed to restore a backup, including:
+
+    - ``remote_location``: The original backup data location.
+    - ``local_datastore_version_id``: The local datastore version corresponding
+      to the original backup.
+    - ``size``: The original backup size.
+  in: body
+  required: false
+  type: object
 backup_size:
   description: |
     Size of the backup, the unit is GB.

doc/source/user/backup-db.rst

@@ -276,3 +276,21 @@ Create an incremental backup based on a parent backup:
     |    status   |                 NEW                  |
     |   updated   |         2014-03-19T14:09:13          |
     +-------------+--------------------------------------+
+
+Restore backup from other regions
+---------------------------------
+
+Restoring backup from other regions were introduced in Wallaby,
+
+In multi-region deployment with geo-replicated Swift, the user is able to
+create a backup in one region using the backup data created in the others,
+which is useful in Disaster Recovery scenario. Instance ID is not required in
+this case when restoring backup, but the original backup data location (a swift
+object URL), the local datastore version and the backup data size are required.
+
+.. warning::
+
+   The restored backup is dependent on the original backup data, if the
+   original backup is deleted, the restored backup is invalid.
+
+TODO: Add CLI example once supported in python-troveclient.

releasenotes/notes/wallaby-restore-backup.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - In multi-region deployment with geo-replicated Swift, the user can restore
+    a backup in one region by manually specifying the original backup data
+    location created in another region.

trove/backup/models.py

@@ -22,7 +22,9 @@ from swiftclient.client import ClientException
 from trove.backup.state import BackupState
 from trove.common import cfg
 from trove.common import clients
+from trove.common import constants
 from trove.common import exception
+from trove.common import swift
 from trove.common import utils
 from trove.common.i18n import _
 from trove.datastore import models as datastore_models
@@ -49,7 +51,8 @@ class Backup(object):
 
     @classmethod
     def create(cls, context, instance, name, description=None,
-               parent_id=None, incremental=False, swift_container=None):
+               parent_id=None, incremental=False, swift_container=None,
+               restore_from=None):
         """
         create db record for Backup
         :param cls:
@@ -61,29 +64,59 @@ class Backup(object):
         :param incremental: flag to indicate incremental backup
                             based on previous backup
         :param swift_container: Swift container name.
+        :param restore_from: A dict that contains backup information of another
+                             region.
         :return:
         """
+        backup_state = BackupState.NEW
+        checksum = None
+        instance_id = None
+        parent = None
+        last_backup_id = None
+        location = None
+        backup_type = constants.BACKUP_TYPE_FULL
+        size = None
 
-        def _create_resources():
+        if restore_from:
-            # parse the ID from the Ref
+            # Check location and datastore version.
+            LOG.info(f"Restoring backup, restore_from: {restore_from}")
+            backup_state = BackupState.RESTORED
+
+            ds_version_id = restore_from.get('local_datastore_version_id')
+            ds_version = datastore_models.DatastoreVersion.load_by_uuid(
+                ds_version_id)
+
+            location = restore_from.get('remote_location')
+            swift_client = clients.create_swift_client(context)
+            try:
+                obj_meta = swift.get_metadata(swift_client, location,
+                                              extra_attrs=['etag'])
+            except Exception:
+                msg = f'Failed to restore backup from {location}'
+                LOG.exception(msg)
+                raise exception.BackupCreationError(msg)
+
+            checksum = obj_meta['etag']
+            if 'parent_location' in obj_meta:
+                backup_type = constants.BACKUP_TYPE_INC
+
+            size = restore_from['size']
+        else:
             instance_id = utils.get_id_from_href(instance)
-
+            # Import here to avoid circular imports.
-            # verify that the instance exists and can perform actions
+            from trove.instance import models as inst_model
-            from trove.instance.models import Instance
+            instance_model = inst_model.Instance.load(context, instance_id)
-            instance_model = Instance.load(context, instance_id)
             instance_model.validate_can_perform_action()
-            cls.validate_can_perform_action(
-                instance_model, 'backup_create')
-
-            cls.verify_swift_auth_token(context)
-
             if instance_model.cluster_id is not None:
                 raise exception.ClusterInstanceOperationNotSupported()
 
+            cls.validate_can_perform_action(instance_model, 'backup_create')
+
+            cls.verify_swift_auth_token(context)
+
             ds = instance_model.datastore
             ds_version = instance_model.datastore_version
-            parent = None
+
-            last_backup_id = None
             if parent_id:
                 # Look up the parent info or fail early if not found or if
                 # the user does not have access to the parent.
@@ -100,22 +133,34 @@ class Backup(object):
                         'checksum': _parent.checksum
                     }
                     last_backup_id = _parent.id
+
+            if parent:
+                backup_type = constants.BACKUP_TYPE_INC
+
+        def _create_resources():
             try:
-                db_info = DBBackup.create(name=name,
+                db_info = DBBackup.create(
+                    name=name,
                     description=description,
                     tenant_id=context.project_id,
-                                          state=BackupState.NEW,
+                    state=backup_state,
                     instance_id=instance_id,
-                                          parent_id=parent_id or
+                    parent_id=parent_id or last_backup_id,
-                                          last_backup_id,
                     datastore_version_id=ds_version.id,
-                                          deleted=False)
+                    deleted=False,
+                    location=location,
+                    checksum=checksum,
+                    backup_type=backup_type,
+                    size=size
+                )
             except exception.InvalidModelError as ex:
                 LOG.exception("Unable to create backup record for "
                               "instance: %s", instance_id)
                 raise exception.BackupCreationError(str(ex))
 
-            backup_info = {'id': db_info.id,
+            if not restore_from:
+                backup_info = {
+                    'id': db_info.id,
                     'name': name,
                     'description': description,
                     'instance_id': instance_id,
@@ -127,9 +172,14 @@ class Backup(object):
                     'swift_container': swift_container
                 }
                 api.API(context).create_backup(backup_info, instance_id)
+            else:
+                context.notification.payload.update(
+                    {'backup_id': db_info.id}
+                )
+
             return db_info
-        return run_with_quotas(context.project_id,
+
-                               {'backups': 1},
+        return run_with_quotas(context.project_id, {'backups': 1},
                                _create_resources)
 
     @classmethod
@@ -372,7 +422,7 @@ class DBBackup(DatabaseModelBase):
 
     @property
     def is_done_successfuly(self):
-        return self.state == BackupState.COMPLETED
+        return self.state in [BackupState.COMPLETED, BackupState.RESTORED]
 
     @property
     def filename(self):

trove/backup/service.py

@@ -80,16 +80,20 @@ class BackupController(wsgi.Controller):
         context = req.environ[wsgi.CONTEXT_KEY]
         policy.authorize_on_tenant(context, 'backup:create')
         data = body['backup']
-        instance = data['instance']
+        instance = data.get('instance')
         name = data['name']
         desc = data.get('description')
         parent = data.get('parent_id')
         incremental = data.get('incremental')
         swift_container = data.get('swift_container')
+        restore_from = data.get('restore_from')
 
-        context.notification = notification.DBaaSBackupCreate(context,
+        context.notification = notification.DBaaSBackupCreate(
-                                                              request=req)
+            context, request=req)
 
+        if not restore_from:
+            if not instance:
+                raise exception.BackupCreationError('instance is missing.')
             if not swift_container:
                 instance_id = utils.get_id_from_href(instance)
                 backup_strategy = BackupStrategy.get(context, instance_id)
@@ -100,7 +104,8 @@ class BackupController(wsgi.Controller):
                                description=desc, parent_id=parent):
             backup = Backup.create(context, instance, name, desc,
                                    parent_id=parent, incremental=incremental,
-                                   swift_container=swift_container)
+                                   swift_container=swift_container,
+                                   restore_from=restore_from)
 
         return wsgi.Result(views.BackupView(backup).data(), 202)
 

trove/backup/state.py

@@ -21,6 +21,7 @@ class BackupState(object):
     SAVING = "SAVING"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
+    RESTORED = "RESTORED"
     DELETE_FAILED = "DELETE_FAILED"
     RUNNING_STATES = [NEW, BUILDING, SAVING]
-    END_STATES = [COMPLETED, FAILED, DELETE_FAILED]
+    END_STATES = [COMPLETED, FAILED, DELETE_FAILED, RESTORED]

trove/common/apischema.py

@@ -652,14 +652,27 @@ backup = {
         "properties": {
             "backup": {
                 "type": "object",
-                "required": ["instance", "name"],
+                "required": ["name"],
                 "properties": {
                     "description": non_empty_string,
                     "instance": uuid,
                     "name": non_empty_string,
                     "parent_id": uuid,
                     "incremental": boolean_string,
-                    "swift_container": non_empty_string
+                    "swift_container": non_empty_string,
+                    "restore_from": {
+                        "type": "object",
+                        "required": [
+                            "remote_location",
+                            "local_datastore_version_id",
+                            "size"
+                        ],
+                        "properties": {
+                            "remote_location": non_empty_string,
+                            "local_datastore_version_id": uuid,
+                            "size": {"type": "number"}
+                        }
+                    }
                 }
             }
         }

trove/common/constants.py

@@ -0,0 +1,16 @@
+# Copyright 2021 Catalyst Cloud Ltd.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+BACKUP_TYPE_FULL = 'full'
+BACKUP_TYPE_INC = 'incremental'

trove/common/swift.py

@@ -0,0 +1,42 @@
+# Copyright 2021 Catalyst Cloud Ltd.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+def parse_location(location):
+    storage_url = "/".join(location.split('/')[:-2])
+    container_name = location.split('/')[-2]
+    object_name = location.split('/')[-1]
+    return storage_url, container_name, object_name
+
+
+def _get_attr(original):
+    """Get a friendly name from an object header key."""
+    key = original.replace('-', '_')
+    key = key.replace('x_object_meta_', '')
+    return key
+
+
+def get_metadata(client, location, extra_attrs=[]):
+    _, container_name, object_name = parse_location(location)
+    headers = client.head_object(container_name, object_name)
+
+    meta = {}
+    for key, value in headers.items():
+        if key.startswith('x-object-meta'):
+            meta[_get_attr(key)] = value
+
+    for key in extra_attrs:
+        meta[key] = headers.get(key)
+
+    return meta

trove/instance/models.py

@@ -351,7 +351,7 @@ class SimpleInstance(object):
         - Then server status
         - Otherwise, unknown
         """
-        LOG.info(f"Getting instance status for {self.id}, "
+        LOG.debug(f"Getting instance status for {self.id}, "
                   f"task status: {self.db_info.task_status}, "
                   f"datastore status: {self.datastore_status.status}, "
                   f"server status: {self.db_info.server_status}")

trove/taskmanager/models.py

@@ -1470,8 +1470,7 @@ class BackupTasks(object):
         def _delete(backup):
             backup.deleted = True
             backup.deleted_at = timeutils.utcnow()
-            # Set datastore_version_id to None so that datastore_version could
+            # Set datastore_version_id to None to remove dependency.
-            # be deleted.
             backup.datastore_version_id = None
             backup.save()
 
@@ -1479,7 +1478,9 @@ class BackupTasks(object):
         backup = bkup_models.Backup.get_by_id(context, backup_id)
         try:
             filename = backup.filename
-            if filename:
+            # Do not remove the object if the backup was restored from remote
+            # location.
+            if filename and backup.state != bkup_models.BackupState.RESTORED:
                 BackupTasks.delete_files_from_swift(context,
                                                     backup.container_name,
                                                     filename)

trove/tests/unittests/backup/test_backup_models.py

@@ -176,7 +176,7 @@ class BackupCreateTest(trove_testtools.TestCase):
                               BACKUP_NAME, BACKUP_DESC)
 
     def test_create_backup_swift_token_invalid(self):
-        instance = MagicMock()
+        instance = MagicMock(cluster_id=None)
         with patch.object(instance_models.BuiltInstance, 'load',
                           return_value=instance):
             instance.validate_can_perform_action = MagicMock(
@@ -191,7 +191,7 @@ class BackupCreateTest(trove_testtools.TestCase):
                                       BACKUP_NAME, BACKUP_DESC)
 
     def test_create_backup_datastore_operation_not_supported(self):
-        instance = MagicMock()
+        instance = MagicMock(cluster_id=None)
         with patch.object(instance_models.BuiltInstance, 'load',
                           return_value=instance):
             with patch.object(

trove/tests/unittests/backup/test_service.py

@@ -0,0 +1,84 @@
+# Copyright 2021 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+from unittest import mock
+
+from trove.backup import service
+from trove.backup.state import BackupState
+from trove.common import context
+from trove.common import wsgi
+from trove.datastore import models as ds_models
+from trove.tests.unittests import trove_testtools
+from trove.tests.unittests.util import util
+
+
+class TestBackupController(trove_testtools.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        util.init_db()
+
+        cls.ds_name = cls.random_name('datastore',
+                                      prefix='TestBackupController')
+        ds_models.update_datastore(name=cls.ds_name, default_version=None)
+        cls.ds = ds_models.Datastore.load(cls.ds_name)
+
+        ds_models.update_datastore_version(
+            cls.ds_name, 'fake-ds-version', 'mysql', '', ['trove', 'mysql'],
+            '', 1)
+        cls.ds_version = ds_models.DatastoreVersion.load(
+            cls.ds, 'fake-ds-version')
+
+        cls.controller = service.BackupController()
+
+        super(TestBackupController, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        util.cleanup_db()
+        super(TestBackupController, cls).tearDownClass()
+
+    def setUp(self):
+        trove_testtools.patch_notifier(self)
+        self.context = context.TroveContext(project_id=self.random_uuid())
+
+        super(TestBackupController, self).setUp()
+
+    @mock.patch('trove.common.clients.create_swift_client')
+    def test_create_restore_from(self, mock_swift_client):
+        swift_client = mock.MagicMock()
+        swift_client.head_object.return_value = {'etag': 'fake-etag'}
+        mock_swift_client.return_value = swift_client
+
+        req = mock.MagicMock(environ={wsgi.CONTEXT_KEY: self.context})
+
+        name = self.random_name(
+            name='backup', prefix='TestBackupController')
+        body = {
+            'backup': {
+                "name": name,
+                "restore_from": {
+                    "remote_location": "http://192.168.206.8:8080/v1/"
+                                       "AUTH_055b2fb9a2264ae5a5f6b3cc066c4a1d/"
+                                       "fake-container/fake-object",
+                    "local_datastore_version_id": self.ds_version.id,
+                    "size": 0.2
+                }
+            }
+        }
+        ret = self.controller.create(req, body, self.context.project_id)
+        self.assertEqual(202, ret.status)
+
+        ret_backup = ret.data(None)['backup']
+
+        self.assertEqual(BackupState.RESTORED, ret_backup.get('status'))
+        self.assertEqual(name, ret_backup.get('name'))

trove/tests/unittests/taskmanager/test_models.py

@@ -1037,10 +1037,8 @@ class BackupTasksTest(trove_testtools.TestCase):
 
         self.assertTrue(self.backup.deleted)
 
-    @patch('trove.taskmanager.models.LOG')
     @patch('trove.common.clients.create_swift_client')
-    def test_delete_backup_fail_delete_manifest(self, mock_swift_client,
+    def test_delete_backup_fail_delete_manifest(self, mock_swift_client):
-                                                mock_logging):
         client_mock = MagicMock()
         client_mock.head_object.return_value = {}
         client_mock.delete_object.side_effect = ClientException("foo")
@@ -1070,6 +1068,12 @@ class BackupTasksTest(trove_testtools.TestCase):
         client_mock.delete_object.assert_called_once_with('container',
                                                           '12e48.xbstream.gz')
 
+    def test_delete_backup_restored(self):
+        self.backup.state = state.BackupState.RESTORED
+        taskmanager_models.BackupTasks.delete_backup(mock.ANY, self.backup.id)
+
+        self.assertTrue(self.backup.deleted)
+
     def test_parse_manifest(self):
         manifest = 'container/prefix'
         cont, prefix = taskmanager_models.BackupTasks._parse_manifest(manifest)