openstack/trove
Code Issues Proposed changes

Support to restore backup from remote location

Browse Source 
In multi-region deployment with geo-replicated Swift, the user can
restore a backup in one region by manually specifying the original
backup data location created in another region.

Change-Id: Iefef3bf969163af707935445bc23299400dc88c3
This commit is contained in:
Lingxian Kong 2021-02-16 12:23:19 +13:00
parent 9c2e0bf3a0
commit 6fdf11ea7f
15 changed files with 328 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api-ref/source/backups.inc
View File

@ -76,6 +76,19 @@ In the Trove deployment with service tenant enabled, The backup data is
stored as objects in OpenStack Swift service in the user's container. If not
specified, the container name is defined by the cloud admin.
The user can create a backup strategy within the project scope or specific to
a particular instance.
In multi-region deployment with geo-replicated Swift, the user can also restore
a backup in a region by manually specifying the backup data location created in
another region, then create instances from the backup. Instance ID is not
required in this case.
.. warning::
The restored backup is dependent on the original backup data, if the
original backup is deleted, the restored backup is invalid.
Normal response codes: 202
Request
@ -90,6 +103,7 @@ Request
- incremental: backup_incremental
- description: backup_description
- swift_container: swift_container
- restore_from: backup_restore_from
Request Example
---------------

13
api-ref/source/parameters.yaml
View File

@ -148,7 +148,7 @@ backup_instanceId:
description: |
The ID of the instance to create backup for.
in: body
required: true
required: false
type: string
backup_list:
description: |
@ -180,6 +180,17 @@ backup_parentId1:
in: body
required: true
type: string
backup_restore_from:
description: |
The information needed to restore a backup, including:
- ``remote_location``: The original backup data location.
- ``local_datastore_version_id``: The local datastore version corresponding
to the original backup.
- ``size``: The original backup size.
in: body
required: false
type: object
backup_size:
description: |
Size of the backup, the unit is GB.

18
doc/source/user/backup-db.rst
View File

@ -276,3 +276,21 @@ Create an incremental backup based on a parent backup:
| status | NEW |
| updated | 2014-03-19T14:09:13 |
+-------------+--------------------------------------+
Restore backup from other regions
---------------------------------
Restoring backup from other regions were introduced in Wallaby,
In multi-region deployment with geo-replicated Swift, the user is able to
create a backup in one region using the backup data created in the others,
which is useful in Disaster Recovery scenario. Instance ID is not required in
this case when restoring backup, but the original backup data location (a swift
object URL), the local datastore version and the backup data size are required.
.. warning::
The restored backup is dependent on the original backup data, if the
original backup is deleted, the restored backup is invalid.
TODO: Add CLI example once supported in python-troveclient.

5
releasenotes/notes/wallaby-restore-backup.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- In multi-region deployment with geo-replicated Swift, the user can restore
a backup in one region by manually specifying the original backup data
location created in another region.

128
trove/backup/models.py
View File

@ -22,7 +22,9 @@ from swiftclient.client import ClientException
from trove.backup.state import BackupState
from trove.common import cfg
from trove.common import clients
from trove.common import constants
from trove.common import exception
from trove.common import swift
from trove.common import utils
from trove.common.i18n import _
from trove.datastore import models as datastore_models
@ -49,7 +51,8 @@ class Backup(object):
@classmethod
def create(cls, context, instance, name, description=None,
parent_id=None, incremental=False, swift_container=None):
parent_id=None, incremental=False, swift_container=None,
restore_from=None):
"""
create db record for Backup
:param cls:
@ -59,31 +62,61 @@ class Backup(object):
:param description:
:param parent_id:
:param incremental: flag to indicate incremental backup
based on previous backup
based on previous backup
:param swift_container: Swift container name.
:param restore_from: A dict that contains backup information of another
region.
:return:
"""
backup_state = BackupState.NEW
checksum = None
instance_id = None
parent = None
last_backup_id = None
location = None
backup_type = constants.BACKUP_TYPE_FULL
size = None
def _create_resources():
# parse the ID from the Ref
if restore_from:
# Check location and datastore version.
LOG.info(f"Restoring backup, restore_from: {restore_from}")
backup_state = BackupState.RESTORED
ds_version_id = restore_from.get('local_datastore_version_id')
ds_version = datastore_models.DatastoreVersion.load_by_uuid(
ds_version_id)
location = restore_from.get('remote_location')
swift_client = clients.create_swift_client(context)
try:
obj_meta = swift.get_metadata(swift_client, location,
extra_attrs=['etag'])
except Exception:
msg = f'Failed to restore backup from {location}'
LOG.exception(msg)
raise exception.BackupCreationError(msg)
checksum = obj_meta['etag']
if 'parent_location' in obj_meta:
backup_type = constants.BACKUP_TYPE_INC
size = restore_from['size']
else:
instance_id = utils.get_id_from_href(instance)
# verify that the instance exists and can perform actions
from trove.instance.models import Instance
instance_model = Instance.load(context, instance_id)
# Import here to avoid circular imports.
from trove.instance import models as inst_model
instance_model = inst_model.Instance.load(context, instance_id)
instance_model.validate_can_perform_action()
cls.validate_can_perform_action(
instance_model, 'backup_create')
cls.verify_swift_auth_token(context)
if instance_model.cluster_id is not None:
raise exception.ClusterInstanceOperationNotSupported()
cls.validate_can_perform_action(instance_model, 'backup_create')
cls.verify_swift_auth_token(context)
ds = instance_model.datastore
ds_version = instance_model.datastore_version
parent = None
last_backup_id = None
if parent_id:
# Look up the parent info or fail early if not found or if
# the user does not have access to the parent.
@ -100,36 +133,53 @@ class Backup(object):
'checksum': _parent.checksum
}
last_backup_id = _parent.id
if parent:
backup_type = constants.BACKUP_TYPE_INC
def _create_resources():
try:
db_info = DBBackup.create(name=name,
description=description,
tenant_id=context.project_id,
state=BackupState.NEW,
instance_id=instance_id,
parent_id=parent_id or
last_backup_id,
datastore_version_id=ds_version.id,
deleted=False)
db_info = DBBackup.create(
name=name,
description=description,
tenant_id=context.project_id,
state=backup_state,
instance_id=instance_id,
parent_id=parent_id or last_backup_id,
datastore_version_id=ds_version.id,
deleted=False,
location=location,
checksum=checksum,
backup_type=backup_type,
size=size
)
except exception.InvalidModelError as ex:
LOG.exception("Unable to create backup record for "
"instance: %s", instance_id)
raise exception.BackupCreationError(str(ex))
backup_info = {'id': db_info.id,
'name': name,
'description': description,
'instance_id': instance_id,
'backup_type': db_info.backup_type,
'checksum': db_info.checksum,
'parent': parent,
'datastore': ds.name,
'datastore_version': ds_version.name,
'swift_container': swift_container
}
api.API(context).create_backup(backup_info, instance_id)
if not restore_from:
backup_info = {
'id': db_info.id,
'name': name,
'description': description,
'instance_id': instance_id,
'backup_type': db_info.backup_type,
'checksum': db_info.checksum,
'parent': parent,
'datastore': ds.name,
'datastore_version': ds_version.name,
'swift_container': swift_container
}
api.API(context).create_backup(backup_info, instance_id)
else:
context.notification.payload.update(
{'backup_id': db_info.id}
)
return db_info
return run_with_quotas(context.project_id,
{'backups': 1},
return run_with_quotas(context.project_id, {'backups': 1},
_create_resources)
@classmethod
@ -372,7 +422,7 @@ class DBBackup(DatabaseModelBase):
@property
def is_done_successfuly(self):
return self.state == BackupState.COMPLETED
return self.state in [BackupState.COMPLETED, BackupState.RESTORED]
@property
def filename(self):

23
trove/backup/service.py
View File

@ -80,27 +80,32 @@ class BackupController(wsgi.Controller):
context = req.environ[wsgi.CONTEXT_KEY]
policy.authorize_on_tenant(context, 'backup:create')
data = body['backup']
instance = data['instance']
instance = data.get('instance')
name = data['name']
desc = data.get('description')
parent = data.get('parent_id')
incremental = data.get('incremental')
swift_container = data.get('swift_container')
restore_from = data.get('restore_from')
context.notification = notification.DBaaSBackupCreate(context,
request=req)
context.notification = notification.DBaaSBackupCreate(
context, request=req)
if not swift_container:
instance_id = utils.get_id_from_href(instance)
backup_strategy = BackupStrategy.get(context, instance_id)
if backup_strategy:
swift_container = backup_strategy.swift_container
if not restore_from:
if not instance:
raise exception.BackupCreationError('instance is missing.')
if not swift_container:
instance_id = utils.get_id_from_href(instance)
backup_strategy = BackupStrategy.get(context, instance_id)
if backup_strategy:
swift_container = backup_strategy.swift_container
with StartNotification(context, name=name, instance_id=instance,
description=desc, parent_id=parent):
backup = Backup.create(context, instance, name, desc,
parent_id=parent, incremental=incremental,
swift_container=swift_container)
swift_container=swift_container,
restore_from=restore_from)
return wsgi.Result(views.BackupView(backup).data(), 202)

3
trove/backup/state.py
View File

@ -21,6 +21,7 @@ class BackupState(object):
SAVING = "SAVING"
COMPLETED = "COMPLETED"
FAILED = "FAILED"
RESTORED = "RESTORED"
DELETE_FAILED = "DELETE_FAILED"
RUNNING_STATES = [NEW, BUILDING, SAVING]
END_STATES = [COMPLETED, FAILED, DELETE_FAILED]
END_STATES = [COMPLETED, FAILED, DELETE_FAILED, RESTORED]

17
trove/common/apischema.py
View File

@ -652,14 +652,27 @@ backup = {
"properties": {
"backup": {
"type": "object",
"required": ["instance", "name"],
"required": ["name"],
"properties": {
"description": non_empty_string,
"instance": uuid,
"name": non_empty_string,
"parent_id": uuid,
"incremental": boolean_string,
"swift_container": non_empty_string
"swift_container": non_empty_string,
"restore_from": {
"type": "object",
"required": [
"remote_location",
"local_datastore_version_id",
"size"
],
"properties": {
"remote_location": non_empty_string,
"local_datastore_version_id": uuid,
"size": {"type": "number"}
}
}
}
}
}

16
trove/common/constants.py Normal file
View File

@ -0,0 +1,16 @@
# Copyright 2021 Catalyst Cloud Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
BACKUP_TYPE_FULL = 'full'
BACKUP_TYPE_INC = 'incremental'

42
trove/common/swift.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2021 Catalyst Cloud Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def parse_location(location):
storage_url = "/".join(location.split('/')[:-2])
container_name = location.split('/')[-2]
object_name = location.split('/')[-1]
return storage_url, container_name, object_name
def _get_attr(original):
"""Get a friendly name from an object header key."""
key = original.replace('-', '_')
key = key.replace('x_object_meta_', '')
return key
def get_metadata(client, location, extra_attrs=[]):
_, container_name, object_name = parse_location(location)
headers = client.head_object(container_name, object_name)
meta = {}
for key, value in headers.items():
if key.startswith('x-object-meta'):
meta[_get_attr(key)] = value
for key in extra_attrs:
meta[key] = headers.get(key)
return meta

8
trove/instance/models.py
View File

@ -351,10 +351,10 @@ class SimpleInstance(object):
- Then server status
- Otherwise, unknown
"""
LOG.info(f"Getting instance status for {self.id}, "
f"task status: {self.db_info.task_status}, "
f"datastore status: {self.datastore_status.status}, "
f"server status: {self.db_info.server_status}")
LOG.debug(f"Getting instance status for {self.id}, "
f"task status: {self.db_info.task_status}, "
f"datastore status: {self.datastore_status.status}, "
f"server status: {self.db_info.server_status}")
task_status = self.db_info.task_status
server_status = self.db_info.server_status

7
trove/taskmanager/models.py
View File

@ -1470,8 +1470,7 @@ class BackupTasks(object):
def _delete(backup):
backup.deleted = True
backup.deleted_at = timeutils.utcnow()
# Set datastore_version_id to None so that datastore_version could
# be deleted.
# Set datastore_version_id to None to remove dependency.
backup.datastore_version_id = None
backup.save()
@ -1479,7 +1478,9 @@ class BackupTasks(object):
backup = bkup_models.Backup.get_by_id(context, backup_id)
try:
filename = backup.filename
if filename:
# Do not remove the object if the backup was restored from remote
# location.
if filename and backup.state != bkup_models.BackupState.RESTORED:
BackupTasks.delete_files_from_swift(context,
backup.container_name,
filename)

4
trove/tests/unittests/backup/test_backup_models.py
View File

@ -176,7 +176,7 @@ class BackupCreateTest(trove_testtools.TestCase):
BACKUP_NAME, BACKUP_DESC)
def test_create_backup_swift_token_invalid(self):
instance = MagicMock()
instance = MagicMock(cluster_id=None)
with patch.object(instance_models.BuiltInstance, 'load',
return_value=instance):
instance.validate_can_perform_action = MagicMock(
@ -191,7 +191,7 @@ class BackupCreateTest(trove_testtools.TestCase):
BACKUP_NAME, BACKUP_DESC)
def test_create_backup_datastore_operation_not_supported(self):
instance = MagicMock()
instance = MagicMock(cluster_id=None)
with patch.object(instance_models.BuiltInstance, 'load',
return_value=instance):
with patch.object(

84
trove/tests/unittests/backup/test_service.py Normal file
View File

@ -0,0 +1,84 @@
# Copyright 2021 Catalyst Cloud
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest import mock
from trove.backup import service
from trove.backup.state import BackupState
from trove.common import context
from trove.common import wsgi
from trove.datastore import models as ds_models
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.util import util
class TestBackupController(trove_testtools.TestCase):
@classmethod
def setUpClass(cls):
util.init_db()
cls.ds_name = cls.random_name('datastore',
prefix='TestBackupController')
ds_models.update_datastore(name=cls.ds_name, default_version=None)
cls.ds = ds_models.Datastore.load(cls.ds_name)
ds_models.update_datastore_version(
cls.ds_name, 'fake-ds-version', 'mysql', '', ['trove', 'mysql'],
'', 1)
cls.ds_version = ds_models.DatastoreVersion.load(
cls.ds, 'fake-ds-version')
cls.controller = service.BackupController()
super(TestBackupController, cls).setUpClass()
@classmethod
def tearDownClass(cls):
util.cleanup_db()
super(TestBackupController, cls).tearDownClass()
def setUp(self):
trove_testtools.patch_notifier(self)
self.context = context.TroveContext(project_id=self.random_uuid())
super(TestBackupController, self).setUp()
@mock.patch('trove.common.clients.create_swift_client')
def test_create_restore_from(self, mock_swift_client):
swift_client = mock.MagicMock()
swift_client.head_object.return_value = {'etag': 'fake-etag'}
mock_swift_client.return_value = swift_client
req = mock.MagicMock(environ={wsgi.CONTEXT_KEY: self.context})
name = self.random_name(
name='backup', prefix='TestBackupController')
body = {
'backup': {
"name": name,
"restore_from": {
"remote_location": "http://192.168.206.8:8080/v1/"
"AUTH_055b2fb9a2264ae5a5f6b3cc066c4a1d/"
"fake-container/fake-object",
"local_datastore_version_id": self.ds_version.id,
"size": 0.2
}
}
}
ret = self.controller.create(req, body, self.context.project_id)
self.assertEqual(202, ret.status)
ret_backup = ret.data(None)['backup']
self.assertEqual(BackupState.RESTORED, ret_backup.get('status'))
self.assertEqual(name, ret_backup.get('name'))

10
trove/tests/unittests/taskmanager/test_models.py
View File

@ -1037,10 +1037,8 @@ class BackupTasksTest(trove_testtools.TestCase):
self.assertTrue(self.backup.deleted)
@patch('trove.taskmanager.models.LOG')
@patch('trove.common.clients.create_swift_client')
def test_delete_backup_fail_delete_manifest(self, mock_swift_client,
mock_logging):
def test_delete_backup_fail_delete_manifest(self, mock_swift_client):
client_mock = MagicMock()
client_mock.head_object.return_value = {}
client_mock.delete_object.side_effect = ClientException("foo")
@ -1070,6 +1068,12 @@ class BackupTasksTest(trove_testtools.TestCase):
client_mock.delete_object.assert_called_once_with('container',
'12e48.xbstream.gz')
def test_delete_backup_restored(self):
self.backup.state = state.BackupState.RESTORED
taskmanager_models.BackupTasks.delete_backup(mock.ANY, self.backup.id)
self.assertTrue(self.backup.deleted)
def test_parse_manifest(self):
manifest = 'container/prefix'
cont, prefix = taskmanager_models.BackupTasks._parse_manifest(manifest)