4c955751d3
If the supplied path is not of the form /v1/a/c/o then swift tempurl subcommand will now return an error message. Also removes redundant check for seconds parameter being an int from shell.py because the same check is made when calling utils.generate_temp_url. Drive-by fix for missing param definition for generate_temp_url. Change-Id: I41f4389948b01fadaa5fc4939ea12e0ed2167345 Related-Change: I0fb2ce125fe12d660e4deb778265016bdd5ff31b
303 lines
9.5 KiB
Python
303 lines
9.5 KiB
Python
# Copyright (c) 2010-2012 OpenStack, LLC.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Miscellaneous utility functions for use with Swift."""
|
|
import gzip
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import logging
|
|
import six
|
|
import time
|
|
import traceback
|
|
|
|
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
|
|
EMPTY_ETAG = 'd41d8cd98f00b204e9800998ecf8427e'
|
|
|
|
|
|
def config_true_value(value):
|
|
"""
|
|
Returns True if the value is either True or a string in TRUE_VALUES.
|
|
Returns False otherwise.
|
|
This function comes from swift.common.utils.config_true_value()
|
|
"""
|
|
return value is True or \
|
|
(isinstance(value, six.string_types) and value.lower() in TRUE_VALUES)
|
|
|
|
|
|
def prt_bytes(num_bytes, human_flag):
|
|
"""
|
|
convert a number > 1024 to printable format, either in 4 char -h format as
|
|
with ls -lh or return as 12 char right justified string
|
|
"""
|
|
|
|
if not human_flag:
|
|
return '%12s' % num_bytes
|
|
|
|
num = float(num_bytes)
|
|
suffixes = [None] + list('KMGTPEZY')
|
|
for suffix in suffixes[:-1]:
|
|
if num <= 1023:
|
|
break
|
|
num /= 1024.0
|
|
else:
|
|
suffix = suffixes[-1]
|
|
|
|
if not suffix: # num_bytes must be < 1024
|
|
return '%4s' % num_bytes
|
|
elif num >= 10:
|
|
return '%3d%s' % (num, suffix)
|
|
else:
|
|
return '%.1f%s' % (num, suffix)
|
|
|
|
|
|
def generate_temp_url(path, seconds, key, method, absolute=False):
|
|
"""Generates a temporary URL that gives unauthenticated access to the
|
|
Swift object.
|
|
|
|
:param path: The full path to the Swift object. Example:
|
|
/v1/AUTH_account/c/o.
|
|
:param seconds: If absolute is False then this specifies the amount of time
|
|
in seconds for which the temporary URL will be valid. If absolute is
|
|
True then this specifies an absolute time at which the temporary URL
|
|
will expire.
|
|
:param key: The secret temporary URL key set on the Swift
|
|
cluster. To set a key, run 'swift post -m
|
|
"Temp-URL-Key: <substitute tempurl key here>"'
|
|
:param method: A HTTP method, typically either GET or PUT, to allow
|
|
for this temporary URL.
|
|
:param absolute: if True then the seconds parameter is interpreted as an
|
|
absolute Unix time, otherwise seconds is interpreted as a relative time
|
|
offset from current time.
|
|
:raises: ValueError if seconds is not a positive integer or path is not to
|
|
an object.
|
|
:raises: TypeError if seconds is not an integer
|
|
:return: the path portion of a temporary URL
|
|
"""
|
|
try:
|
|
seconds = int(seconds)
|
|
except ValueError:
|
|
raise TypeError('seconds must be an integer')
|
|
if seconds < 0:
|
|
raise ValueError('seconds must be a positive integer')
|
|
|
|
if isinstance(path, six.binary_type):
|
|
try:
|
|
path_for_body = path.decode('utf-8')
|
|
except UnicodeDecodeError:
|
|
raise ValueError('path must be representable as UTF-8')
|
|
else:
|
|
path_for_body = path
|
|
|
|
parts = path_for_body.split('/', 4)
|
|
if len(parts) != 5 or parts[0] or not all(parts[1:]):
|
|
raise ValueError('path must be full path to an object e.g. /v1/a/c/o')
|
|
|
|
standard_methods = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE']
|
|
if method.upper() not in standard_methods:
|
|
logger = logging.getLogger("swiftclient")
|
|
logger.warning('Non default HTTP method %s for tempurl specified, '
|
|
'possibly an error', method.upper())
|
|
|
|
if not absolute:
|
|
expiration = int(time.time() + seconds)
|
|
else:
|
|
expiration = seconds
|
|
hmac_body = u'\n'.join([method.upper(), str(expiration), path_for_body])
|
|
|
|
# Encode to UTF-8 for py3 compatibility
|
|
if not isinstance(key, six.binary_type):
|
|
key = key.encode('utf-8')
|
|
sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest()
|
|
|
|
temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format(
|
|
path=path_for_body, sig=sig, exp=expiration)
|
|
# Have return type match path from caller
|
|
if isinstance(path, six.binary_type):
|
|
return temp_url.encode('utf-8')
|
|
else:
|
|
return temp_url
|
|
|
|
|
|
def parse_api_response(headers, body):
|
|
if headers.get('content-encoding') == 'gzip':
|
|
with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz:
|
|
body = gz.read()
|
|
|
|
charset = 'utf-8'
|
|
# Swift *should* be speaking UTF-8, but check content-type just in case
|
|
content_type = headers.get('content-type', '')
|
|
if '; charset=' in content_type:
|
|
charset = content_type.split('; charset=', 1)[1].split(';', 1)[0]
|
|
|
|
return json.loads(body.decode(charset))
|
|
|
|
|
|
def report_traceback():
|
|
"""
|
|
Reports a timestamp and full traceback for a given exception.
|
|
|
|
:return: Full traceback and timestamp.
|
|
"""
|
|
try:
|
|
formatted_lines = traceback.format_exc()
|
|
now = time.time()
|
|
return formatted_lines, now
|
|
except AttributeError:
|
|
return None, None
|
|
|
|
|
|
class NoopMD5(object):
|
|
def __init__(self, *a, **kw):
|
|
pass
|
|
|
|
def update(self, *a, **kw):
|
|
pass
|
|
|
|
def hexdigest(self, *a, **kw):
|
|
return ''
|
|
|
|
|
|
class ReadableToIterable(object):
|
|
"""
|
|
Wrap a filelike object and act as an iterator.
|
|
|
|
It is recommended to use this class only on files opened in binary mode.
|
|
Due to the Unicode changes in Python 3, files are now opened using an
|
|
encoding not suitable for use with the md5 class and because of this
|
|
hit the exception on every call to next. This could cause problems,
|
|
especially with large files and small chunk sizes.
|
|
"""
|
|
|
|
def __init__(self, content, chunk_size=65536, md5=False):
|
|
"""
|
|
:param content: The filelike object that is yielded from.
|
|
:param chunk_size: The max size of each yielded item.
|
|
:param md5: Flag to enable calculating the MD5 of the content
|
|
as it is yielded.
|
|
"""
|
|
self.md5sum = hashlib.md5() if md5 else NoopMD5()
|
|
self.content = content
|
|
self.chunk_size = chunk_size
|
|
|
|
def get_md5sum(self):
|
|
return self.md5sum.hexdigest()
|
|
|
|
def __next__(self):
|
|
"""
|
|
Both ``__next__`` and ``next`` are provided to allow compatibility
|
|
with python 2 and python 3 and their use of ``iterable.next()``
|
|
and ``next(iterable)`` respectively.
|
|
"""
|
|
chunk = self.content.read(self.chunk_size)
|
|
if not chunk:
|
|
raise StopIteration
|
|
|
|
try:
|
|
self.md5sum.update(chunk)
|
|
except TypeError:
|
|
self.md5sum.update(chunk.encode())
|
|
|
|
return chunk
|
|
|
|
def next(self):
|
|
return self.__next__()
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
|
|
class LengthWrapper(object):
|
|
"""
|
|
Wrap a filelike object with a maximum length.
|
|
|
|
Fix for https://github.com/kennethreitz/requests/issues/1648.
|
|
It is recommended to use this class only on files opened in binary mode.
|
|
"""
|
|
def __init__(self, readable, length, md5=False):
|
|
"""
|
|
:param readable: The filelike object to read from.
|
|
:param length: The maximum amount of content that can be read from
|
|
the filelike object before it is simulated to be
|
|
empty.
|
|
:param md5: Flag to enable calculating the MD5 of the content
|
|
as it is read.
|
|
"""
|
|
self._md5 = md5
|
|
self._reset_md5()
|
|
self._length = self._remaining = length
|
|
self._readable = readable
|
|
self._can_reset = all(hasattr(readable, attr)
|
|
for attr in ('seek', 'tell'))
|
|
if self._can_reset:
|
|
self._start = readable.tell()
|
|
|
|
def __len__(self):
|
|
return self._length
|
|
|
|
def _reset_md5(self):
|
|
self.md5sum = hashlib.md5() if self._md5 else NoopMD5()
|
|
|
|
def get_md5sum(self):
|
|
return self.md5sum.hexdigest()
|
|
|
|
def read(self, size=-1):
|
|
if self._remaining <= 0:
|
|
return ''
|
|
|
|
to_read = self._remaining if size < 0 else min(size, self._remaining)
|
|
chunk = self._readable.read(to_read)
|
|
self._remaining -= len(chunk)
|
|
|
|
try:
|
|
self.md5sum.update(chunk)
|
|
except TypeError:
|
|
self.md5sum.update(chunk.encode())
|
|
|
|
return chunk
|
|
|
|
@property
|
|
def reset(self):
|
|
if self._can_reset:
|
|
return self._reset
|
|
raise AttributeError("%r object has no attribute 'reset'" %
|
|
type(self).__name__)
|
|
|
|
def _reset(self, *args, **kwargs):
|
|
if not self._can_reset:
|
|
raise TypeError('%r object cannot be reset; needs both seek and '
|
|
'tell methods' % type(self._readable).__name__)
|
|
self._readable.seek(self._start)
|
|
self._reset_md5()
|
|
self._remaining = self._length
|
|
|
|
|
|
def iter_wrapper(iterable):
|
|
for chunk in iterable:
|
|
if len(chunk) == 0:
|
|
# If we emit an empty chunk, requests will go ahead and send it,
|
|
# causing the server to close the connection
|
|
continue
|
|
yield chunk
|
|
|
|
|
|
def n_at_a_time(seq, n):
|
|
for i in range(0, len(seq), n):
|
|
yield seq[i:i + n]
|
|
|
|
|
|
def n_groups(seq, n):
|
|
items_per_group = ((len(seq) - 1) // n) + 1
|
|
return n_at_a_time(seq, items_per_group)
|