214 lines
7.7 KiB
Python
214 lines
7.7 KiB
Python
# Copyright 2016 Red Hat, Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import dateutil.parser
|
|
import logging
|
|
import os
|
|
import tempfile
|
|
import uuid
|
|
|
|
import six
|
|
from six.moves import urllib
|
|
from swiftclient import exceptions as swiftexceptions
|
|
from swiftclient.service import SwiftError
|
|
from swiftclient.service import SwiftUploadObject
|
|
from swiftclient.utils import generate_temp_url
|
|
|
|
from tripleo_common.utils import tarball
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def empty_container(swiftclient, name):
|
|
container_names = [container["name"] for container
|
|
in swiftclient.get_account()[1]]
|
|
|
|
if name in container_names:
|
|
headers, objects = swiftclient.get_container(name)
|
|
# FIXME(rbrady): remove delete_object loop when
|
|
# LP#1615830 is fixed. See LP#1615825 for more info.
|
|
# delete files from plan
|
|
for o in objects:
|
|
swiftclient.delete_object(name, o['name'])
|
|
else:
|
|
error_text = "The {name} container does not exist.".format(name=name)
|
|
raise ValueError(error_text)
|
|
|
|
|
|
def delete_container(swiftclient, name):
|
|
try:
|
|
empty_container(swiftclient, name)
|
|
swiftclient.delete_container(name)
|
|
except ValueError as e:
|
|
# ValueError is raised when we can't find the container, which means
|
|
# that it's already deleted.
|
|
LOG.info(six.text_type(e))
|
|
|
|
|
|
def download_container(swiftclient, container, dest,
|
|
overwrite_only_newer=False):
|
|
"""Download the contents of a Swift container to a directory"""
|
|
|
|
objects = swiftclient.get_container(container)[1]
|
|
|
|
for obj in objects:
|
|
is_newer = False
|
|
filename = obj['name']
|
|
contents = swiftclient.get_object(container, filename)[1]
|
|
try:
|
|
contents = contents.encode('utf-8')
|
|
except (UnicodeDecodeError, AttributeError):
|
|
pass
|
|
path = os.path.join(dest, filename)
|
|
dirname = os.path.dirname(path)
|
|
already_exists = os.path.exists(path)
|
|
|
|
if already_exists:
|
|
last_modified = obj.get('last_modified', None)
|
|
|
|
if last_modified is not None:
|
|
last_mod_swift = int(dateutil.parser.parse(
|
|
obj['last_modified']).strftime('%s'))
|
|
|
|
last_mod_disk = int(os.path.getmtime(path))
|
|
|
|
if last_mod_swift > last_mod_disk:
|
|
is_newer = True
|
|
|
|
# write file if `overwrite_only_newer` is not set,
|
|
# or if file does not exist at destination,
|
|
# or if we found a newer file at source
|
|
if (not overwrite_only_newer or
|
|
not already_exists or
|
|
(overwrite_only_newer and is_newer)):
|
|
if not os.path.exists(dirname):
|
|
os.makedirs(dirname)
|
|
|
|
# open in binary as the swift client returns error
|
|
# under python3 if opened as text
|
|
with open(path, 'wb') as f:
|
|
f.write(contents)
|
|
|
|
|
|
def create_container(swiftclient, container):
|
|
# If the container already exists, it will return 202 and everything will
|
|
# work
|
|
swiftclient.put_container(container)
|
|
|
|
|
|
def create_and_upload_tarball(swiftservice,
|
|
tmp_dir,
|
|
container,
|
|
tarball_name,
|
|
tarball_options='-czf',
|
|
delete_after=3600,
|
|
segment_size=1048576000,
|
|
use_slo=True,
|
|
segment_container=None,
|
|
leave_segments=False,
|
|
changed=None,
|
|
skip_identical=False,
|
|
fail_fast=True,
|
|
dir_marker=False):
|
|
"""Create a tarball containing the tmp_dir and upload it to Swift.
|
|
|
|
This method allows to upload files bigger than 5GB.
|
|
It will create 2 swift containers to store the segments and
|
|
one container to reference the manifest with the segment pointers
|
|
"""
|
|
|
|
try:
|
|
with tempfile.NamedTemporaryFile() as tmp_tarball:
|
|
tarball.create_tarball(tmp_dir,
|
|
tmp_tarball.name,
|
|
tarball_options)
|
|
objs = [SwiftUploadObject(tmp_tarball,
|
|
object_name=tarball_name)]
|
|
options = {'meta': [],
|
|
'header': ['X-Delete-After: ' + str(delete_after)],
|
|
'segment_size': segment_size,
|
|
'use_slo': use_slo,
|
|
'segment_container': segment_container,
|
|
'leave_segments': leave_segments,
|
|
'changed': changed,
|
|
'skip_identical': skip_identical,
|
|
'fail_fast': fail_fast,
|
|
'dir_marker': dir_marker
|
|
}
|
|
|
|
for r in swiftservice.upload(container,
|
|
objs,
|
|
options=options):
|
|
if r['success']:
|
|
if 'object' in r:
|
|
LOG.info(r['object'])
|
|
elif 'for_object' in r:
|
|
LOG.info(
|
|
'%s segment %s' % (r['for_object'],
|
|
r['segment_index'])
|
|
)
|
|
else:
|
|
error = r['error']
|
|
if r['action'] == "create_container":
|
|
LOG.warning(
|
|
'Warning: failed to create container '
|
|
"'%s'%s", container, error
|
|
)
|
|
elif r['action'] == "upload_object":
|
|
LOG.error(
|
|
"Failed to upload object %s to container %s: %s" %
|
|
(container, r['object'], error)
|
|
)
|
|
else:
|
|
LOG.error("%s" % error)
|
|
except SwiftError as e:
|
|
LOG.error(e.value)
|
|
|
|
|
|
def get_object_string(swift, container, object_name):
|
|
"""Get the object contents as a string """
|
|
data = swift.get_object(container, object_name)[1]
|
|
try:
|
|
return data.decode('utf-8')
|
|
except AttributeError:
|
|
return data
|
|
|
|
|
|
def put_object_string(swift, container, object_name, contents):
|
|
"""Put the object contents as a string """
|
|
try:
|
|
contents = contents.decode('utf-8')
|
|
except AttributeError:
|
|
pass
|
|
return swift.put_object(container, object_name, contents)
|
|
|
|
|
|
def get_temp_url(swift, container, object_name, method='GET', valid='86400'):
|
|
try:
|
|
cont_stat = swift.head_container(container)
|
|
except swiftexceptions.ClientException:
|
|
cont_stat = {}
|
|
|
|
key = cont_stat.get('x-container-meta-temp-url-key')
|
|
if not key:
|
|
key = str(uuid.uuid4())
|
|
cont_stat = swift.put_container(
|
|
container, {'X-Container-Meta-Temp-Url-Key': key})
|
|
parsed = urllib.parse.urlparse(swift.url)
|
|
path = "%s/%s/%s" % (parsed.path, container, object_name)
|
|
temp_path = generate_temp_url(path, valid, key, method)
|
|
return "%s://%s%s" % (parsed.scheme, parsed.netloc, temp_path)
|