cinder/cinder/backup/drivers/swift.py

# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
# Copyright (c) 2014 TrilioData, Inc
# Copyright (c) 2015 EMC Corporation
# Copyright (C) 2015 Kevin Fox <kevin@efox.cc>
# Copyright (C) 2015 Tom Barron <tpb@dyncloud.net>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Implementation of a backup service that uses Swift as the backend

**Related Flags**

:backup_swift_url: The URL of the Swift endpoint (default: None, use catalog).
:swift_catalog_info: Info to match when looking for swift in the service '
                     catalog.
:backup_swift_object_size: The size in bytes of the Swift objects used
                                    for volume backups (default: 52428800).
:backup_swift_retry_attempts: The number of retries to make for Swift
                                    operations (default: 10).
:backup_swift_retry_backoff: The backoff time in seconds between retrying
                                    failed Swift operations (default: 10).
:backup_compression_algorithm: Compression algorithm to use for volume
                               backups. Supported options are:
                               None (to disable), zlib and bz2 (default: zlib)
"""

import hashlib
import socket

from oslo_config import cfg
from oslo_utils import timeutils
import six
from swiftclient import client as swift

from cinder.backup import chunkeddriver
from cinder import exception
from cinder.i18n import _
from cinder.i18n import _LE
from cinder.openstack.common import log as logging

LOG = logging.getLogger(__name__)

swiftbackup_service_opts = [
    cfg.StrOpt('backup_swift_url',
               default=None,
               help='The URL of the Swift endpoint'),
    cfg.StrOpt('swift_catalog_info',
               default='object-store:swift:publicURL',
               help='Info to match when looking for swift in the service '
               'catalog. Format is: separated values of the form: '
               '<service_type>:<service_name>:<endpoint_type> - '
               'Only used if backup_swift_url is unset'),
    cfg.StrOpt('backup_swift_auth',
               default='per_user',
               help='Swift authentication mechanism'),
    cfg.StrOpt('backup_swift_auth_version',
               default='1',
               help='Swift authentication version. Specify "1" for auth 1.0'
                    ', or "2" for auth 2.0'),
    cfg.StrOpt('backup_swift_tenant',
               default=None,
               help='Swift tenant/account name. Required when connecting'
                    ' to an auth 2.0 system'),
    cfg.StrOpt('backup_swift_user',
               default=None,
               help='Swift user name'),
    cfg.StrOpt('backup_swift_key',
               default=None,
               help='Swift key for authentication'),
    cfg.StrOpt('backup_swift_container',
               default='volumebackups',
               help='The default Swift container to use'),
    cfg.IntOpt('backup_swift_object_size',
               default=52428800,
               help='The size in bytes of Swift backup objects'),
    cfg.IntOpt('backup_swift_block_size',
               default=32768,
               help='The size in bytes that changes are tracked '
                    'for incremental backups. backup_swift_object_size '
                    'has to be multiple of backup_swift_block_size.'),
    cfg.IntOpt('backup_swift_retry_attempts',
               default=3,
               help='The number of retries to make for Swift operations'),
    cfg.IntOpt('backup_swift_retry_backoff',
               default=2,
               help='The backoff time in seconds between Swift retries'),
    cfg.BoolOpt('backup_swift_enable_progress_timer',
                default=True,
                help='Enable or Disable the timer to send the periodic '
                     'progress notifications to Ceilometer when backing '
                     'up the volume to the Swift backend storage. The '
                     'default value is True to enable the timer.'),
]

CONF = cfg.CONF
CONF.register_opts(swiftbackup_service_opts)


class SwiftBackupDriver(chunkeddriver.ChunkedBackupDriver):
    """Provides backup, restore and delete of backup objects within Swift."""

    def __init__(self, context, db_driver=None):
        chunk_size_bytes = CONF.backup_swift_object_size
        sha_block_size_bytes = CONF.backup_swift_block_size
        backup_default_container = CONF.backup_swift_container
        enable_progress_timer = CONF.backup_swift_enable_progress_timer
        super(SwiftBackupDriver, self).__init__(context, chunk_size_bytes,
                                                sha_block_size_bytes,
                                                backup_default_container,
                                                enable_progress_timer,
                                                db_driver)
        if CONF.backup_swift_url is None:
            self.swift_url = None
            info = CONF.swift_catalog_info
            try:
                service_type, service_name, endpoint_type = info.split(':')
            except ValueError:
                raise exception.BackupDriverException(_(
                    "Failed to parse the configuration option "
                    "'swift_catalog_info', must be in the form "
                    "<service_type>:<service_name>:<endpoint_type>"))
            for entry in context.service_catalog:
                if entry.get('type') == service_type:
                    self.swift_url = entry.get(
                        'endpoints')[0].get(endpoint_type)
        else:
            self.swift_url = '%s%s' % (CONF.backup_swift_url,
                                       context.project_id)
        if self.swift_url is None:
            raise exception.BackupDriverException(_(
                "Could not determine which Swift endpoint to use. This can "
                " either be set in the service catalog or with the "
                " cinder.conf config option 'backup_swift_url'."))
        LOG.debug("Using swift URL %s", self.swift_url)
        self.swift_attempts = CONF.backup_swift_retry_attempts
        self.swift_backoff = CONF.backup_swift_retry_backoff
        LOG.debug('Connect to %s in "%s" mode', (CONF.backup_swift_url,
                                                 CONF.backup_swift_auth))
        if CONF.backup_swift_auth == 'single_user':
            if CONF.backup_swift_user is None:
                LOG.error(_LE("single_user auth mode enabled, "
                              "but %(param)s not set"),
                          {'param': 'backup_swift_user'})
                raise exception.ParameterNotFound(param='backup_swift_user')
            self.conn = swift.Connection(
                authurl=CONF.backup_swift_url,
                auth_version=CONF.backup_swift_auth_version,
                tenant_name=CONF.backup_swift_tenant,
                user=CONF.backup_swift_user,
                key=CONF.backup_swift_key,
                retries=self.swift_attempts,
                starting_backoff=self.swift_backoff)
        else:
            self.conn = swift.Connection(retries=self.swift_attempts,
                                         preauthurl=self.swift_url,
                                         preauthtoken=self.context.auth_token,
                                         starting_backoff=self.swift_backoff)

    class SwiftObjectWriter:
        def __init__(self, container, object_name, conn):
            self.container = container
            self.object_name = object_name
            self.conn = conn
            self.data = ''

        def __enter__(self):
            return self

        def __exit__(self, exc_type, exc_value, traceback):
            self.close()

        def write(self, data):
            self.data += data

        def close(self):
            reader = six.StringIO(self.data)
            try:
                etag = self.conn.put_object(self.container, self.object_name,
                                            reader,
                                            content_length=reader.len)
            except socket.error as err:
                raise exception.SwiftConnectionFailed(reason=err)
            LOG.debug('swift MD5 for %(object_name)s: %(etag)s',
                      {'object_name': self.object_name, 'etag': etag, })
            md5 = hashlib.md5(self.data).hexdigest()
            LOG.debug('backup MD5 for %(object_name)s: %(md5)s',
                      {'object_name': self.object_name, 'md5': md5})
            if etag != md5:
                err = _('error writing object to swift, MD5 of object in '
                        'swift %(etag)s is not the same as MD5 of object sent '
                        'to swift %(md5)s'), {'etag': etag, 'md5': md5}
                raise exception.InvalidBackup(reason=err)
            return md5

    class SwiftObjectReader:
        def __init__(self, container, object_name, conn):
            self.container = container
            self.object_name = object_name
            self.conn = conn

        def __enter__(self):
            return self

        def __exit__(self, exc_type, exc_value, traceback):
            pass

        def read(self):
            try:
                (_resp, body) = self.conn.get_object(self.container,
                                                     self.object_name)
            except socket.error as err:
                raise exception.SwiftConnectionFailed(reason=err)
            return body

    def put_container(self, container):
        """Create the container if needed. No failure if it pre-exists."""
        try:
            self.conn.put_container(container)
        except socket.error as err:
            raise exception.SwiftConnectionFailed(reason=err)
        return

    def get_container_entries(self, container, prefix):
        """Get container entry names"""
        try:
            swift_objects = self.conn.get_container(container,
                                                    prefix=prefix,
                                                    full_listing=True)[1]
        except socket.error as err:
            raise exception.SwiftConnectionFailed(reason=err)
        swift_object_names = [swift_obj['name'] for swift_obj in swift_objects]
        return swift_object_names

    def get_object_writer(self, container, object_name, extra_metadata=None):
        """Returns a writer object that stores a chunk of volume data in a
           Swift object store.
        """
        return self.SwiftObjectWriter(container, object_name, self.conn)

    def get_object_reader(self, container, object_name, extra_metadata=None):
        """Returns a reader object that retrieves a chunk of backed-up volume data
           from a Swift object store.
        """
        return self.SwiftObjectReader(container, object_name, self.conn)

    def delete_object(self, container, object_name):
        """Deletes a backup object from a Swift object store."""
        try:
            self.conn.delete_object(container, object_name)
        except socket.error as err:
            raise exception.SwiftConnectionFailed(reason=err)

    def _generate_object_name_prefix(self, backup):
        """Generates a Swift backup object name prefix."""
        az = 'az_%s' % self.az
        backup_name = '%s_backup_%s' % (az, backup['id'])
        volume = 'volume_%s' % (backup['volume_id'])
        timestamp = timeutils.strtime(fmt="%Y%m%d%H%M%S")
        prefix = volume + '/' + timestamp + '/' + backup_name
        LOG.debug('generate_object_name_prefix: %s', prefix)
        return prefix

    def update_container_name(self, backup, container):
        """Use the container name as provided - don't update."""
        return container

    def get_extra_metadata(self, backup, volume):
        """Swift driver does not use any extra metadata."""
        return None


def get_backup_driver(context):
    return SwiftBackupDriver(context)