tripleo-common/tripleo_common/utils/swift.py

# Copyright 2016 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import dateutil.parser
import logging
import os
import tempfile

import six
from swiftclient.service import SwiftError
from swiftclient.service import SwiftUploadObject

from tripleo_common.utils import tarball

LOG = logging.getLogger(__name__)


def empty_container(swiftclient, name):
    container_names = [container["name"] for container
                       in swiftclient.get_account()[1]]

    if name in container_names:
        headers, objects = swiftclient.get_container(name)
        # FIXME(rbrady): remove delete_object loop when
        # LP#1615830 is fixed.  See LP#1615825 for more info.
        # delete files from plan
        for o in objects:
            swiftclient.delete_object(name, o['name'])
    else:
        error_text = "The {name} container does not exist.".format(name=name)
        raise ValueError(error_text)


def delete_container(swiftclient, name):
    try:
        empty_container(swiftclient, name)
        swiftclient.delete_container(name)
    except ValueError as e:
        # ValueError is raised when we can't find the container, which means
        # that it's already deleted.
        LOG.info(six.text_type(e))


def download_container(swiftclient, container, dest,
                       overwrite_only_newer=False):
    """Download the contents of a Swift container to a directory"""

    objects = swiftclient.get_container(container)[1]

    for obj in objects:
        is_newer = False
        filename = obj['name']
        contents = swiftclient.get_object(container, filename)[1]
        try:
            contents = contents.encode('utf-8')
        except (UnicodeDecodeError, AttributeError):
            pass
        path = os.path.join(dest, filename)
        dirname = os.path.dirname(path)
        already_exists = os.path.exists(path)

        if already_exists:
            last_modified = obj.get('last_modified', None)

            if last_modified is not None:
                last_mod_swift = int(dateutil.parser.parse(
                    obj['last_modified']).strftime('%s'))

                last_mod_disk = int(os.path.getmtime(path))

                if last_mod_swift > last_mod_disk:
                    is_newer = True

        # write file if `overwrite_only_newer` is not set,
        # or if file does not exist at destination,
        # or if we found a newer file at source
        if (not overwrite_only_newer or
                not already_exists or
                (overwrite_only_newer and is_newer)):
            if not os.path.exists(dirname):
                os.makedirs(dirname)

            # open in binary as the swift client returns error
            # under python3 if opened as text
            with open(path, 'wb') as f:
                f.write(contents)


def create_container(swiftclient, container):
    # If the container already exists, it will return 202 and everything will
    # work
    swiftclient.put_container(container)


def create_and_upload_tarball(swiftservice,
                              tmp_dir,
                              container,
                              tarball_name,
                              tarball_options='-czf',
                              delete_after=3600,
                              segment_size=1048576000,
                              use_slo=True,
                              segment_container=None,
                              leave_segments=False,
                              changed=None,
                              skip_identical=False,
                              fail_fast=True,
                              dir_marker=False):
    """Create a tarball containing the tmp_dir and upload it to Swift.

       This method allows to upload files bigger than 5GB.
       It will create 2 swift containers to store the segments and
       one container to reference the manifest with the segment pointers
    """

    try:
        with tempfile.NamedTemporaryFile() as tmp_tarball:
            tarball.create_tarball(tmp_dir,
                                   tmp_tarball.name,
                                   tarball_options)
            objs = [SwiftUploadObject(tmp_tarball,
                                      object_name=tarball_name)]
            options = {'meta': [],
                       'header': ['X-Delete-After: ' + str(delete_after)],
                       'segment_size': segment_size,
                       'use_slo': use_slo,
                       'segment_container': segment_container,
                       'leave_segments': leave_segments,
                       'changed': changed,
                       'skip_identical': skip_identical,
                       'fail_fast': fail_fast,
                       'dir_marker': dir_marker
                       }

            for r in swiftservice.upload(container,
                                         objs,
                                         options=options):
                if r['success']:
                    if 'object' in r:
                        LOG.info(r['object'])
                    elif 'for_object' in r:
                        LOG.info(
                            '%s segment %s' % (r['for_object'],
                                               r['segment_index'])
                            )
                else:
                    error = r['error']
                    if r['action'] == "create_container":
                        LOG.warning(
                            'Warning: failed to create container '
                            "'%s'%s", container, error
                        )
                    elif r['action'] == "upload_object":
                        LOG.error(
                            "Failed to upload object %s to container %s: %s" %
                            (container, r['object'], error)
                        )
                    else:
                        LOG.error("%s" % error)
    except SwiftError as e:
        LOG.error(e.value)


def get_object_string(swift, container, object_name):
    """Get the object contents as a string """
    data = swift.get_object(container, object_name)[1]
    try:
        return data.decode('utf-8')
    except AttributeError:
        return data


def put_object_string(swift, container, object_name, contents):
    """Put the object contents as a string """
    try:
        contents = contents.decode('utf-8')
    except AttributeError:
        pass
    return swift.put_object(container, object_name, contents)