
Abstract out a "chunked" backup driver class between the top abstract backup driver class and the Swift backup driver class which implements common functionality for backup drivers that store volume data in multiple "chunks" in a backup repository when the size of the backed up cinder volume exceeds the size of a backup repository "chunk." The Swift driver itself is reworked to extend the chunked backup driver and implement abstract methods defined in the chunked backup driver. It is expected that posix filesystem and NFS backup drivers will also extend the chunked backup driver. Co-authors: Kevin Fox <kevin@efox.cc> Tom Barron <tpb@dyncloud.net> Implements: blueprint chunked-backup-base-class Change-Id: Ifbd1462ceb9b98bd8a6d72e5725399474eedf60e
285 lines
12 KiB
Python
285 lines
12 KiB
Python
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
|
|
# Copyright (c) 2014 TrilioData, Inc
|
|
# Copyright (c) 2015 EMC Corporation
|
|
# Copyright (C) 2015 Kevin Fox <kevin@efox.cc>
|
|
# Copyright (C) 2015 Tom Barron <tpb@dyncloud.net>
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Implementation of a backup service that uses Swift as the backend
|
|
|
|
**Related Flags**
|
|
|
|
:backup_swift_url: The URL of the Swift endpoint (default: None, use catalog).
|
|
:swift_catalog_info: Info to match when looking for swift in the service '
|
|
catalog.
|
|
:backup_swift_object_size: The size in bytes of the Swift objects used
|
|
for volume backups (default: 52428800).
|
|
:backup_swift_retry_attempts: The number of retries to make for Swift
|
|
operations (default: 10).
|
|
:backup_swift_retry_backoff: The backoff time in seconds between retrying
|
|
failed Swift operations (default: 10).
|
|
:backup_compression_algorithm: Compression algorithm to use for volume
|
|
backups. Supported options are:
|
|
None (to disable), zlib and bz2 (default: zlib)
|
|
"""
|
|
|
|
import hashlib
|
|
import socket
|
|
|
|
from oslo_config import cfg
|
|
from oslo_utils import timeutils
|
|
import six
|
|
from swiftclient import client as swift
|
|
|
|
from cinder.backup import chunkeddriver
|
|
from cinder import exception
|
|
from cinder.i18n import _
|
|
from cinder.i18n import _LE
|
|
from cinder.openstack.common import log as logging
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
swiftbackup_service_opts = [
|
|
cfg.StrOpt('backup_swift_url',
|
|
default=None,
|
|
help='The URL of the Swift endpoint'),
|
|
cfg.StrOpt('swift_catalog_info',
|
|
default='object-store:swift:publicURL',
|
|
help='Info to match when looking for swift in the service '
|
|
'catalog. Format is: separated values of the form: '
|
|
'<service_type>:<service_name>:<endpoint_type> - '
|
|
'Only used if backup_swift_url is unset'),
|
|
cfg.StrOpt('backup_swift_auth',
|
|
default='per_user',
|
|
help='Swift authentication mechanism'),
|
|
cfg.StrOpt('backup_swift_auth_version',
|
|
default='1',
|
|
help='Swift authentication version. Specify "1" for auth 1.0'
|
|
', or "2" for auth 2.0'),
|
|
cfg.StrOpt('backup_swift_tenant',
|
|
default=None,
|
|
help='Swift tenant/account name. Required when connecting'
|
|
' to an auth 2.0 system'),
|
|
cfg.StrOpt('backup_swift_user',
|
|
default=None,
|
|
help='Swift user name'),
|
|
cfg.StrOpt('backup_swift_key',
|
|
default=None,
|
|
help='Swift key for authentication'),
|
|
cfg.StrOpt('backup_swift_container',
|
|
default='volumebackups',
|
|
help='The default Swift container to use'),
|
|
cfg.IntOpt('backup_swift_object_size',
|
|
default=52428800,
|
|
help='The size in bytes of Swift backup objects'),
|
|
cfg.IntOpt('backup_swift_block_size',
|
|
default=32768,
|
|
help='The size in bytes that changes are tracked '
|
|
'for incremental backups. backup_swift_object_size '
|
|
'has to be multiple of backup_swift_block_size.'),
|
|
cfg.IntOpt('backup_swift_retry_attempts',
|
|
default=3,
|
|
help='The number of retries to make for Swift operations'),
|
|
cfg.IntOpt('backup_swift_retry_backoff',
|
|
default=2,
|
|
help='The backoff time in seconds between Swift retries'),
|
|
cfg.BoolOpt('backup_swift_enable_progress_timer',
|
|
default=True,
|
|
help='Enable or Disable the timer to send the periodic '
|
|
'progress notifications to Ceilometer when backing '
|
|
'up the volume to the Swift backend storage. The '
|
|
'default value is True to enable the timer.'),
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
CONF.register_opts(swiftbackup_service_opts)
|
|
|
|
|
|
class SwiftBackupDriver(chunkeddriver.ChunkedBackupDriver):
|
|
"""Provides backup, restore and delete of backup objects within Swift."""
|
|
|
|
def __init__(self, context, db_driver=None):
|
|
chunk_size_bytes = CONF.backup_swift_object_size
|
|
sha_block_size_bytes = CONF.backup_swift_block_size
|
|
backup_default_container = CONF.backup_swift_container
|
|
enable_progress_timer = CONF.backup_swift_enable_progress_timer
|
|
super(SwiftBackupDriver, self).__init__(context, chunk_size_bytes,
|
|
sha_block_size_bytes,
|
|
backup_default_container,
|
|
enable_progress_timer,
|
|
db_driver)
|
|
if CONF.backup_swift_url is None:
|
|
self.swift_url = None
|
|
info = CONF.swift_catalog_info
|
|
try:
|
|
service_type, service_name, endpoint_type = info.split(':')
|
|
except ValueError:
|
|
raise exception.BackupDriverException(_(
|
|
"Failed to parse the configuration option "
|
|
"'swift_catalog_info', must be in the form "
|
|
"<service_type>:<service_name>:<endpoint_type>"))
|
|
for entry in context.service_catalog:
|
|
if entry.get('type') == service_type:
|
|
self.swift_url = entry.get(
|
|
'endpoints')[0].get(endpoint_type)
|
|
else:
|
|
self.swift_url = '%s%s' % (CONF.backup_swift_url,
|
|
context.project_id)
|
|
if self.swift_url is None:
|
|
raise exception.BackupDriverException(_(
|
|
"Could not determine which Swift endpoint to use. This can "
|
|
" either be set in the service catalog or with the "
|
|
" cinder.conf config option 'backup_swift_url'."))
|
|
LOG.debug("Using swift URL %s", self.swift_url)
|
|
self.swift_attempts = CONF.backup_swift_retry_attempts
|
|
self.swift_backoff = CONF.backup_swift_retry_backoff
|
|
LOG.debug('Connect to %s in "%s" mode', (CONF.backup_swift_url,
|
|
CONF.backup_swift_auth))
|
|
if CONF.backup_swift_auth == 'single_user':
|
|
if CONF.backup_swift_user is None:
|
|
LOG.error(_LE("single_user auth mode enabled, "
|
|
"but %(param)s not set"),
|
|
{'param': 'backup_swift_user'})
|
|
raise exception.ParameterNotFound(param='backup_swift_user')
|
|
self.conn = swift.Connection(
|
|
authurl=CONF.backup_swift_url,
|
|
auth_version=CONF.backup_swift_auth_version,
|
|
tenant_name=CONF.backup_swift_tenant,
|
|
user=CONF.backup_swift_user,
|
|
key=CONF.backup_swift_key,
|
|
retries=self.swift_attempts,
|
|
starting_backoff=self.swift_backoff)
|
|
else:
|
|
self.conn = swift.Connection(retries=self.swift_attempts,
|
|
preauthurl=self.swift_url,
|
|
preauthtoken=self.context.auth_token,
|
|
starting_backoff=self.swift_backoff)
|
|
|
|
class SwiftObjectWriter:
|
|
def __init__(self, container, object_name, conn):
|
|
self.container = container
|
|
self.object_name = object_name
|
|
self.conn = conn
|
|
self.data = ''
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
self.close()
|
|
|
|
def write(self, data):
|
|
self.data += data
|
|
|
|
def close(self):
|
|
reader = six.StringIO(self.data)
|
|
try:
|
|
etag = self.conn.put_object(self.container, self.object_name,
|
|
reader,
|
|
content_length=reader.len)
|
|
except socket.error as err:
|
|
raise exception.SwiftConnectionFailed(reason=err)
|
|
LOG.debug('swift MD5 for %(object_name)s: %(etag)s',
|
|
{'object_name': self.object_name, 'etag': etag, })
|
|
md5 = hashlib.md5(self.data).hexdigest()
|
|
LOG.debug('backup MD5 for %(object_name)s: %(md5)s',
|
|
{'object_name': self.object_name, 'md5': md5})
|
|
if etag != md5:
|
|
err = _('error writing object to swift, MD5 of object in '
|
|
'swift %(etag)s is not the same as MD5 of object sent '
|
|
'to swift %(md5)s'), {'etag': etag, 'md5': md5}
|
|
raise exception.InvalidBackup(reason=err)
|
|
return md5
|
|
|
|
class SwiftObjectReader:
|
|
def __init__(self, container, object_name, conn):
|
|
self.container = container
|
|
self.object_name = object_name
|
|
self.conn = conn
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
pass
|
|
|
|
def read(self):
|
|
try:
|
|
(_resp, body) = self.conn.get_object(self.container,
|
|
self.object_name)
|
|
except socket.error as err:
|
|
raise exception.SwiftConnectionFailed(reason=err)
|
|
return body
|
|
|
|
def put_container(self, container):
|
|
"""Create the container if needed. No failure if it pre-exists."""
|
|
try:
|
|
self.conn.put_container(container)
|
|
except socket.error as err:
|
|
raise exception.SwiftConnectionFailed(reason=err)
|
|
return
|
|
|
|
def get_container_entries(self, container, prefix):
|
|
"""Get container entry names"""
|
|
try:
|
|
swift_objects = self.conn.get_container(container,
|
|
prefix=prefix,
|
|
full_listing=True)[1]
|
|
except socket.error as err:
|
|
raise exception.SwiftConnectionFailed(reason=err)
|
|
swift_object_names = [swift_obj['name'] for swift_obj in swift_objects]
|
|
return swift_object_names
|
|
|
|
def get_object_writer(self, container, object_name, extra_metadata=None):
|
|
"""Returns a writer object that stores a chunk of volume data in a
|
|
Swift object store.
|
|
"""
|
|
return self.SwiftObjectWriter(container, object_name, self.conn)
|
|
|
|
def get_object_reader(self, container, object_name, extra_metadata=None):
|
|
"""Returns a reader object that retrieves a chunk of backed-up volume data
|
|
from a Swift object store.
|
|
"""
|
|
return self.SwiftObjectReader(container, object_name, self.conn)
|
|
|
|
def delete_object(self, container, object_name):
|
|
"""Deletes a backup object from a Swift object store."""
|
|
try:
|
|
self.conn.delete_object(container, object_name)
|
|
except socket.error as err:
|
|
raise exception.SwiftConnectionFailed(reason=err)
|
|
|
|
def _generate_object_name_prefix(self, backup):
|
|
"""Generates a Swift backup object name prefix."""
|
|
az = 'az_%s' % self.az
|
|
backup_name = '%s_backup_%s' % (az, backup['id'])
|
|
volume = 'volume_%s' % (backup['volume_id'])
|
|
timestamp = timeutils.strtime(fmt="%Y%m%d%H%M%S")
|
|
prefix = volume + '/' + timestamp + '/' + backup_name
|
|
LOG.debug('generate_object_name_prefix: %s', prefix)
|
|
return prefix
|
|
|
|
def update_container_name(self, backup, container):
|
|
"""Use the container name as provided - don't update."""
|
|
return container
|
|
|
|
def get_extra_metadata(self, backup, volume):
|
|
"""Swift driver does not use any extra metadata."""
|
|
return None
|
|
|
|
|
|
def get_backup_driver(context):
|
|
return SwiftBackupDriver(context)
|